Merge branch 'script_gj' into 'master'

Script gj

See merge request !320
This commit is contained in:
2022-06-01 12:38:35 +08:00
18 changed files with 1379 additions and 53 deletions
@@ -27,7 +27,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
@@ -187,11 +187,22 @@ class BigdataCommunityBatch extends BaseBatch{
}else{
def jsonObj = null
if(dist.address){
jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
jsonObj = OpenApiClient.standard1Action(dist.city, dist.address)
}
if(jsonObj && jsonObj.searchAddress){
ret << jsonObj.searchAddress.street
ret << jsonObj.searchAddress.streetNumber
if(jsonObj && jsonObj.street){
if(jsonObj.street){
ret << jsonObj.street
}else{
ret << ""
}
if(jsonObj.streetNumber){
ret << jsonObj.streetNumber
}else{
ret << ""
}
}else{
ret << ""
ret << ""
@@ -0,0 +1,155 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
class CcbDataStat {
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new CcbDataStat()
// task.run()
task.tradeRun()
}
def tradeRun(){
def ccbCitys = CcbCommonUtils.citys;
def startDate = DateUtils.parseFromTightByPattern("2021-01-01","yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\成交案例统计_0531.xlsx", { d ->
if(d['A']){
if(ccbCitys.contains(d['A'])){
d['D'] = findOracleTotalCount(d['A'])
}else{
def totalcount = findAllTradeCount(d['A'], startDate, endDate)
def repeatCount = findRepeatCount(d['A'], startDate, endDate)
def matchTotalCount = findMatchTradeCount(d['A'], startDate, endDate)
def matchRepeatTotalCount = findMatchRepeatTradeCount(d['A'], startDate, endDate)
d['B'] = totalcount
d['C'] = repeatCount
d['D'] = matchTotalCount
d['E'] = matchRepeatTotalCount
}
}
})
}
def findRepeatCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def set = new HashSet()
for(def d : dbcursor){
def key = d.region+"##"+d.district_name+"##"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd")
if(!set.contains(key)){
set << key
}
}
return set.size()
}
def findMatchRepeatTradeCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("match_id", new BasicDBObject('$exists', true))
def dbcursor = DEAL_PRICE_COLL.find(q)
def set = new HashSet()
for(def d : dbcursor){
def key = d.match_id+"#"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd")
if(!set.contains(key)){
set << key
}
}
return set.size()
}
def findMatchTradeCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("match_id", new BasicDBObject('$exists', true))
def count = DEAL_PRICE_COLL.count(q)
return count
}
public static def findAllTradeCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
// q.put("match_id", new BasicDBObject('$exists', false))
def total = DEAL_PRICE_COLL.count(q)
return total
}
def findOracleTotalCount(city){
def sql = "select count(*) as count from bigdata_deal_gj where cityname='${city}' and dealdate>=to_date('2021-01-01','yyyy-MM-dd') and dealdate<to_date('2022-01-01','yyyy-MM-dd')";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
def run(){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\92个城市字段覆盖率情况.xlsx",{ d->
if(d['A']) {
def citycode = cityCodeDataMap.get(d['A'])
def total = findTotalCount(citycode)
def f_count = findDistFinishDate(citycode)
def b_count = findBuildTypeCount(citycode)
def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total)
def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total)
d['C'] = f_pre
d['D'] = b_pre
d['E'] = total
println d['A'] +" "+ total +" "+f_count+" "+b_count
}
})
}
def findTotalCount(citycode){
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
def findDistFinishDate(citycode){
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and BUILDYEAR>0";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
def findBuildTypeCount(citycode){
def sql = "select count(*) as count from (select commid, count(commid) as count from bigdata_commbuildtype_gj where cityid='${citycode}' and BUILDTYPE!='0199' group by commid)";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
}
@@ -137,7 +137,7 @@ ExcelUtils.write("d:\\
def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )"
println s
def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60"
//DingDingMessageUtil.sendTextMessage(access_token, s)
DingDingMessageUtil.sendTextMessage(access_token, s)
////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]);
@@ -164,6 +164,7 @@ class OpenApiUtils {
if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
return null;
}
address = address.trim().replace(" ","")
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all"
List<com.alibaba.fastjson.JSONObject> poiList = null;
@@ -18,44 +18,62 @@ class CcbCityPriceStat {
task.run()
}
def run(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","临汾","聊城","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","齐齐哈尔","眉山","郴州","营口","蚌埠","赤峰","大庆","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","朔州","随州","大理","通化","楚雄","巴彦淖尔","乌兰察布","松原","景德镇","新余","辽阳","阜新","黔南","酒泉","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","钦州","内江","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","鹤岗","萍乡","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","湘西","昭通","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
def rets = []
def startDate = DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")
for(def city : citys){
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30"))
def cityRets = []
for(def dist : districts){
def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01")
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
if(tradeAvgPrice && weekendPrice){
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
cityRets << row
println row.toString()
def matchTotalCount = findMatchTradeCount(city, startDate, endDate)
println city +" "+matchTotalCount
if(matchTotalCount){
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2022-05-29"))
def cityRets = []
for(def dist : districts){
def tradeAvgPrice = findTrades(dist, "2022-02-01", "2022-06-01")
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
if(tradeAvgPrice && weekendPrice){
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
cityRets << row
println row.toString()
}
}
print city +" "+ cityRets.size()
rets.addAll(cityRets)
}
print city +" "+ cityRets.size()
rets.addAll(cityRets)
}
println "总量:"+rets.size()
ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有")
ExcelUtils.write("d:\\数据\\234个城市价格数据.xlsx", rets, "所有")
}
def findMatchTradeCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("match_id", new BasicDBObject('$exists', true))
def count = DEAL_PRICE_COLL.count(q)
return count
}
def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
// q.put("city", dist.city)
// q.put("region", dist.region)
//
// if(dist.alias){
// if(!dist.alias.contains(dist.name)){
// dist.alias << dist.name
// }
// q.put("district_name", new BasicDBObject('$in', dist.alias))
// }else{
// q.put("district_name", dist.name)
// }
q.put("match_id", dist._id.toString())
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
@@ -0,0 +1,189 @@
package com.fangjia.script.gj.crawl
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.RegionData
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.CommonUtils
import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class AnjukeHouseCrawl {
def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
static main(args) {
def task = new AnjukeHouseCrawl();
task.run();
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
for(def i=24;i<50;i++){
def url = "https://chengde.anjuke.com/sale/shuangqiao/p${i}/?from=navigation"
def doc = getDocument(url, driver)
def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
println "页数:"+i+" 记录数:"+eles.size()
if(eles){
parser(eles, driver)
}
}
}
def parser(eles, driver){
def index = 0
for(def e : eles){
index ++
def href = e.attr("href")
def doc = getDocument(href, driver)
// parserHtml(doc)
def newUrl = href.substring(0, href.indexOf("?"))
sendMq(newUrl, doc)
println "[ "+index +"] "+ newUrl
}
}
def sendMq(href, doc){
JSON json = new JSONObject()
json.put("project", "sell_anjuke")
json.put("url", href)
JSON result = new JSONObject()
result.put("content", doc.toString())
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse", packedMsg)
}
def parserHtml(doc) {
String city = null
def result = [:]
result.siteCode = "安居客"
if (doc) {
result.title = JQueryUtils.selectText([".banner-title h1"], doc)
def heads = JQueryUtils.select(".crumbs a", doc)
if (heads && heads.size() >= 5) {
result.city = heads[0].text().replace("房产网", "")
def orgRegion = heads[2].text().replace("二手房", "")
result.region = RegionData.formatRegion(result.city, orgRegion.trim())
result.block = heads[3].text().replace("二手房", "")
result.district_name = heads[4].text()
} else if (heads && heads.size() >= 3) {
result.city = heads[0].text().replace("房产网", "")
result.district_name = heads[2].text()
def eles = JQueryUtils.select("li:contains(所在位置:) a", doc)
if (eles && eles.size() > 0) {
def orgRegion = eles[0].text()
result.region = RegionData.formatRegion(result.city, orgRegion.trim())
}
}
if (!result.district_name) {
println " [未解析出(城市-区域-名称)信息]"
}
println result
def priceStr = JQueryUtils.selectText(".maininfo-price-num", doc)
if (priceStr) {
if (priceStr.isNumber()) {
result.total_price = priceStr.toDouble() * 10000
}
}
def roomInfoStr = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-strong", doc)
if (roomInfoStr) {
Integer room = DataCrawlUtils.parseRoom(roomInfoStr)
Integer hall = DataCrawlUtils.parseHall(roomInfoStr)
Integer toilet = DataCrawlUtils.parseToilet(roomInfoStr)
if (room) {
result.room = room
}
if (hall) {
result.hall = hall
}
if (toilet) {
result.toilet = toilet
}
}
def areaStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-strong", doc).replace("㎡", "")
if (areaStr) {
result.area = areaStr.toDouble()
}
def directionStr = JQueryUtils.selectText(".maininfo-model-item-3 i", doc)
result.direction = DataCrawlUtils.parseDirection(directionStr)
def buildDateStr = JQueryUtils.selectText(".maininfo-model-item-3 .maininfo-model-weak", doc)
if (buildDateStr && buildDateStr.contains("/")) {
def arrs = buildDateStr.split("\\/")
if (arrs && arrs.size() == 2) {
result.build_date = arrs[0]
result.estate_type = arrs[1]
}
}
result.floor_info = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-weak", doc)
if (result.floor_info && result.floor_info.contains("共")) {
def height = result.floor_info
if (result.floor_info.contains("(")) {
height = height.substring(result.floor_info.indexOf("(") + 1, result.floor_info.indexOf(")"))
}
height = height.replace("共", "").replace("层", "")
if (height) {
result.height = height.toInteger()
}
}
if (result.area && result.total_price) {
result.avg_price = (result.total_price / result.area).toInteger()
}
def fitmitStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-weak", doc)
result.fitment = DataCrawlUtils.parseFitment(fitmitStr)
def dateStr = JQueryUtils.selectText(["td:contains(发布时间) .houseInfo-main-item-name"], doc)
}
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
Thread.sleep(5000)
def els = JQueryUtils.select(".list-main .list-left .list", doc)
if(!els){
els = JQueryUtils.select(".props-main", doc)
}
if(els && els.size()>0){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
}
@@ -0,0 +1,250 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.util.DateUtils
import org.apache.commons.lang.StringUtils
import java.util.regex.Matcher
import java.util.regex.Pattern
class DataCrawlUtils {
static final Map<Pattern, Closure> DEAL_DATE_RESOLVERS = [:]
static final Map<Pattern, Closure> ROOM_RESOLVERS = [:]
static final Map<Pattern, Closure> HALL_RESOLVERS = [:]
static final Map<Pattern, Closure> TOILET_RESOLVERS = [:]
static final Map<Pattern, Closure> TOTAL_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> UNIT_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> AREA_RESOLVERS = [:]
static final Map<Pattern, Closure> HEIGHT_RESOLVERS = [:]
static final Map<Pattern, Closure> FLOOR_RESOLVERS = [:]
static final List<String> DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static {
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
})
ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str ->
str = str.replaceAll("室|房", '').trim()
return str.toInteger()
})
HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str ->
str = str.replace("厅", '').trim()
return str.toInteger()
})
TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str ->
str = str.replace("卫", '').trim()
return str.toInteger()
})
TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str ->
str = str.replace("万", '').trim()
return (str.toDouble() * 10000).toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str ->
str = str.replace("元/㎡", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m?"), { String str ->
str = str.replace("元/m?", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str ->
str = str.replace("元/平", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str ->
str = str.replace("元", '').trim()
return str.toDouble().toInteger()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str ->
str = str.replace("平米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str ->
str = str.replace("平方米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str ->
str = str.replace("平", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str ->
str = str.replace("㎡", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m?"), { String str ->
str = str.replace("m?", '').trim()
return str.toDouble()
})
HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str ->
str = str.replace("共", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str ->
str = str.replace("总", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n2
}
return null
})
FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n1
}
return n1
})
FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str ->
str = str.replace("第", "").replaceAll("层|楼", "")
return str.toInteger()
})
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
/**
* 解析室数
* @param content
* @return
*/
static Integer parseRoom(String content) {
return parse(content, ROOM_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseHall(String content) {
return parse(content, HALL_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseToilet(String content) {
return parse(content, TOILET_RESOLVERS)
}
static Object parse(String content, Map<Pattern, Closure> resolvers) {
if (content != null) {
content = content.replaceAll("\\s", "")
}
for (def resolver : resolvers.entrySet()) {
String match = match(content, resolver.getKey())
if (StringUtils.isNotBlank(match)) {
match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2")
return resolver.getValue()(match)
}
}
return null
}
static String match(String str, Pattern pattern) {
if (str == null) {
return null
}
Matcher matcher = pattern.matcher(str)
if (matcher.find()) {
return matcher.group()
}
return null
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
}
@@ -0,0 +1,347 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.util.DateUtils
import org.apache.commons.lang.StringUtils
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* 成交数据解析工具类
*/
class DealPriceUtils {
static final Map<Pattern, Closure> DEAL_DATE_RESOLVERS = [:]
static final Map<Pattern, Closure> ROOM_RESOLVERS = [:]
static final Map<Pattern, Closure> HALL_RESOLVERS = [:]
static final Map<Pattern, Closure> TOILET_RESOLVERS = [:]
static final Map<Pattern, Closure> TOTAL_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> UNIT_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> AREA_RESOLVERS = [:]
static final Map<Pattern, Closure> HEIGHT_RESOLVERS = [:]
static final Map<Pattern, Closure> FLOOR_RESOLVERS = [:]
static final List<String> DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static {
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
})
ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str ->
str = str.replaceAll("室|房", '').trim()
return str.toInteger()
})
HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str ->
str = str.replace("厅", '').trim()
return str.toInteger()
})
TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str ->
str = str.replace("卫", '').trim()
return str.toInteger()
})
TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str ->
str = str.replace("万", '').trim()
return (str.toDouble() * 10000).toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str ->
str = str.replace("元/㎡", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m²"), { String str ->
str = str.replace("元/m²", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str ->
str = str.replace("元/平", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str ->
str = str.replace("元", '').trim()
return str.toDouble().toInteger()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str ->
str = str.replace("平米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str ->
str = str.replace("平方米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str ->
str = str.replace("平", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str ->
str = str.replace("㎡", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m²"), { String str ->
str = str.replace("m²", '').trim()
return str.toDouble()
})
HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str ->
str = str.replace("共", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str ->
str = str.replace("总", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n2
}
return null
})
FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n1
}
return n1
})
FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str ->
str = str.replace("第", "").replaceAll("层|楼", "")
return str.toInteger()
})
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
/**
* 解析成交日期
* @param content
* @return
*/
static Date parseDealDate(String content) {
return parse(content, DEAL_DATE_RESOLVERS)
}
/**
* 解析室数
* @param content
* @return
*/
static Integer parseRoom(String content) {
return parse(content, ROOM_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseHall(String content) {
return parse(content, HALL_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseToilet(String content) {
return parse(content, TOILET_RESOLVERS)
}
/**
* 解析总价
* @param content
* @return
*/
static Integer parseTotalPrice(String content) {
return parse(content, TOTAL_PRICE_RESOLVERS)
}
/**
* 解析单价
* @param content
* @return
*/
static Integer parseUnitPrice(String content) {
return parse(content, UNIT_PRICE_RESOLVERS)
}
/**
* 解析面积
* @param content
* @return
*/
static Double parseArea(String content) {
return parse(content, AREA_RESOLVERS)
}
/**
* 解析楼层
* @param content
* @return
*/
static Integer parseFloor(String content) {
return parse(content, FLOOR_RESOLVERS)
}
/**
* 解析楼高
* @param content
* @return
*/
static Integer parseHeight(String content) {
return parse(content, HEIGHT_RESOLVERS)
}
/**
* 格式化小区名
* @param name
* @return
*/
static String formatName(String name) {
name = name.replace("-商住", "").replace("-别墅", "")
//汇福家园平安里(迁)2015-2017
name = name.replaceAll("\\d\\d\\d\\d-\\d\\d\\d\\d", "")
//国贸天悦一期2017
name = name.replaceAll("\\d\\d\\d\\d\$", "")
if (name ==~ /.+(\(|()[\d一二三四五六七八九十东南西北|\w]+(期|区)(\)|))$/) {
name = name.replaceAll("\\(|(|\\)|)", '')
} else if (name ==~ /.+(\(|()别墅(\)|))$/) {
name = name.replaceAll("\\(|(|\\)|)", '')
} else if (name ==~ /.+(\(|().+(\)|))$/) {
name = name.replaceAll("(\\(|().+(\\)|))", '')
} else if (name.contains("·") || (name.contains("-") && !(name ==~ /.*\d+-\d+.*/))) {
println("---------------------------->" + name)
name = name.replaceAll("·|-", '')
println(name)
}
if (name ==~ /.{2,}路.{3,}$/ && !(name ==~ /.{2,}路.+号/) && !(name ==~ /.{2,}路.+弄/) && !(name ==~ /.*小区$/)) {
name = name.replaceAll("^.{2,}路", "")
}
return name
}
static Object parse(String content, Map<Pattern, Closure> resolvers) {
if (content != null) {
content = content.replaceAll("\\s", "")
}
for (def resolver : resolvers.entrySet()) {
String match = match(content, resolver.getKey())
if (StringUtils.isNotBlank(match)) {
match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2")
return resolver.getValue()(match)
}
}
return null
}
/**
* 匹配符合条件的字符串
*
* @param str
* @param pattern
* @return
*/
static String match(String str, Pattern pattern) {
if (str == null) {
return null
}
Matcher matcher = pattern.matcher(str)
if (matcher.find()) {
return matcher.group()
}
return null
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
static String parsePrice(String priceStr) {
def pattern =Pattern.compile("\\d+(\\.\\d+)?万")
String match = match(priceStr, pattern)
}
static void main(String[] args) {
println parsePrice("顶秀欣园西苑2室1厅1卫492.3万")
}
}
@@ -0,0 +1,167 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.thirdPriceCrawl.Utils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class ZhugeFangDistCrawl {
public static final SHORT_DATE = Utils.getLastMonth();
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static main(args) {
def task = new ZhugeFangDistCrawl();
task.run();
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
def url = "https://fz.esfxiaoqu.zhuge.com/fuqing/"
WebDriver driver = new ChromeDriver()
driver.manage().deleteAllCookies()
getRegions(driver, url)
}
def getRegions(driver, url) {
for(def i=1;i<=11;i++){
def newUrl = url+"page${i}/"
println newUrl
def doc = getDocument(newUrl, driver)
def total = parseList(doc)
// def next = JQueryUtils.select("a:contains(下一页)", doc)
// if(next){
// driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
// }
}
}
def parseList(doc){
def city= "福州"
def els = JQueryUtils.select("#listTableBox li", doc)
if (els) {
for(def e : els){
def name = e.select(".house-name").text()
def str = e.select(".house-adr").text()
def address = null
if(str){
if(str.contains("]")){
address = str.substring(str.indexOf("]")+1).trim()
}else{
address = str
}
}
def regionStr = null
if(str && str.contains("]") && str.contains("[")){
regionStr = str.substring(str.indexOf("[")+1, str.indexOf("]"))
}
def region = null
def block = null
if(regionStr){
def arrs = regionStr.split("-")
if(arrs && arrs.size()>0){
if(arrs.size()==2){
block = arrs[1].trim()
}
if(arrs.size()>=1){
region = arrs[0].trim()
}
}
}
def links = e.select(".house-name a")
def href = null
if(links){
href = links[0].attr("href")
}
def priceStr = e.select(".price-number").text()
def price = null
if(priceStr && !priceStr.contains("暂无")){
if(priceStr.contains("元")){
priceStr = priceStr.substring(0, priceStr.indexOf("元")).trim()
}
if(priceStr && priceStr.isNumber()){
price = priceStr.toInteger()
}
}
def obj = findByParams(city, region, name)
if(obj){
if(price){
DBObject udata2 = new BasicDBObject()
udata2.put("avg_price", price)
udata2.put("m_date", new Date())
println "[upeate] ${region} "+name
thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2));
}
}else{
def udata = new BasicDBObject()
udata.put("city", city)
udata.put("region", region)
udata.put("block", block)
udata.put("url", href)
udata.put("address", address)
udata.put("name", name)
udata.put("avg_price", price)
udata.put("s_date", SHORT_DATE)
udata.put("c_date", new Date())
udata.put("m_date", new Date())
udata.put("source", "诸葛找房")
thirdPartyColl.save(udata)
println "[save] ${region} "+name+" "+priceStr
}
}
}
return els.size()
}
def findByParams(city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("source", "诸葛找房")
q.put("s_date", SHORT_DATE)
return thirdPartyColl.findOne(q);
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
Thread.sleep(10000)
def els = JQueryUtils.select("#listTableBox li", doc)
if(els && els.size()>0){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
}
@@ -0,0 +1,123 @@
package com.fangjia.script.gj.location
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
class DistrictMapMatch {
static main(args) {
def task = new DistrictMapMatch()
task.mapMatch()
}
def mapMatch(){
def index = 0
ExcelUtils.modify("D:\\数据\\重庆农商行\\小区\\待审核小区(地图匹配).xlsx",{ d->
if(!d['H']){
index++
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
// if (!map || !map.lat) {
// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
// }
if(map){
d['F'] = map.id
if(map.adname){
d['G'] = map.adname
}else if(map.region){
d['G'] = map.region
}
d['H'] = map.name
d['I'] = map.address
if(map.type){
d['J'] = map.type
}else if(map.estateType){
d['J'] = map.estateType
}
d['K'] = map.lng
d['L'] = map.lat
d['M'] = map.score
d['N'] = map.source
}
}
},0)
}
def findStandardObject2(def city, def region, def name, def address, def rets) {
if (rets) {
def bestObj = [:]
if (rets.size() >= 1) {
for (def mapObj : rets) {
def score = 0
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
score = 0.2
}
if(mapObj.name && name){
def newName = mapObj.name;
def names = []
if(mapObj.name && mapObj.name.indexOf("(")>-1){
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
names << name1
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
names << name2
}else{
names << newName
}
def maxNameScore = 0
for(def n : names){
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
if(score1 && score1>0){
def nameScore = (score1 * 4)/10
if(nameScore > maxNameScore){
maxNameScore = nameScore
}
}
}
if(maxNameScore){
try{
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
}catch(Exception e){
maxNameScore = 0
}
}
score = maxNameScore + score;
}
if(mapObj.address && address){
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
if(addressScore && addressScore>0){
try{
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
}catch(Exception e){
addressScore = 0
}
score = addressScore*4/10 + score
}
}
if(!bestObj){
mapObj.score = score
bestObj = mapObj
}else{
if(score > bestObj.score){
mapObj.score = score
bestObj = mapObj
}
}
}
}
return bestObj
}
return null
}
}
@@ -945,20 +945,20 @@ class CZDataBatch extends BaseBatch{
if(price){
return price.avg_price;
}else{
//样例估价
def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false);
try{
JSONObject jsonObject = JSONObject.fromObject(s);
def map = DataHandleUtils.JsonToMap(jsonObject);
if(map.code==200){
if(map.result){
return map.result.avgPrice;
}
}
}catch(Exception e){
}
// //样例估价
// def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false);
// try{
// JSONObject jsonObject = JSONObject.fromObject(s);
// def map = DataHandleUtils.JsonToMap(jsonObject);
// if(map.code==200){
// if(map.result){
// return map.result.avgPrice;
// }
//
// }
// }catch(Exception e){
//
// }
//分期找总期
@@ -0,0 +1,28 @@
package com.fangjia.script.temp.shenjian.didata
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
/**
* 从excel中匹配出
*/
ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\地数\\数据建设\\待审核小区(地图匹配).xlsx", { row ->
def city = row['A']
def region = row['B']
def district_name = row['C']
def gaodeRegion = row['H']
def gaodeName = row['I']
def jsonObject = OpenApiClient.addressMatch(city, district_name, "district", false, region, district_name, "strict")
if (jsonObject != null && jsonObject.precise != -1) {
row['D'] = jsonObject.get("name")
}
if (gaodeRegion && gaodeName) {
def jsonObject2 = OpenApiClient.addressMatch(city, gaodeName, "district", false, gaodeRegion, gaodeName, "strict")
if (jsonObject2 != null && jsonObject2.precise != -1) {
row['J'] = jsonObject2.get("name")
}
}
});
@@ -2,6 +2,8 @@ package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.statistics.poi.AmapPoiDetail
import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
import com.fangjia.script.temp.shenjian.util.ExcelUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
@@ -14,13 +16,15 @@ import java.util.concurrent.atomic.AtomicLong
class GetPoiFromExcel {
static final def URL = "http://192.168.0.216:38080/FangJiaDataService/saveAmapPoi"
static final ExecutorService executorService = Executors.newFixedThreadPool(10)
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail"
// 住宅类
static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304";
static void main(String[] args) {
AtomicLong atomicLong = new AtomicLong(0)
//List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\POI"), null, false);
List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\高德反查无ID"), null, false);
List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区"), null, false);
for (File file : fileList) {
List<List<String>> seaweeds = ExcelUtils.read(file.getPath())
for (List<String> infoTemp : seaweeds) {
@@ -31,9 +35,10 @@ class GetPoiFromExcel {
String city = info.get(0)
String region = info.get(1)
String seaweedName = info.get(2)
List<AmapPoi> pois = GaoDeUtils.searchPoi(city, seaweedName, region, "120300|120301|120302");
List<AmapPoi> pois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes);
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
List<AmapPoiDetail> detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
String jsonStr = JSONObject.toJSONString(detailList)
HttpUtils.postForJson(URL, jsonStr)
atomicLong.incrementAndGet()
}
@@ -81,6 +81,9 @@ class SeaweedPoiDetail {
if (CollectionUtils.isEmpty(pois)) {
pois = GaoDeUtils.searchPoi(city, name, region, typecodes)
}
if (CollectionUtils.isEmpty(pois)) {
pois = GaoDeUtils.searchPoi(city, address, region, typecodes)
}
List<AmapPoiDetail> detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
// for (AmapPoi amapPoi : pois) {
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger
class GaoDeUtils {
// 新获取key
def static GAO_DE_AK = ["4ccfe3c898a9489884bd2bcb1ef6dcde", "4e9592208336433ee1b7aa4d1d434418"]
def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418"]
def static AtomicInteger keyUsed = new AtomicInteger(0)
def static codeNameMap = new HashMap<String, String>(){{
@@ -30,7 +30,7 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.regex.Matcher
import java.util.regex.Patter
import java.util.regex.Pattern
/**
* 第三方数据标准化处理
@@ -514,6 +514,7 @@ class DataStandardBatch extends BaseBatch{
def filterSpacialChar(def s){
def pattern = ~/[()!;\·]\[【】()]/
def matcher = pattern.matcher(s)
def count = matcher.getCount()
if(count && count>0){
return matcher.replaceAll('_')
@@ -169,7 +169,7 @@ public class CommonUtils {
static void sendMessage(String queueName, byte[] bytes){
ConnectionFactory factory = new ConnectionFactory();
factory.setUri("amqp://guest:guest@192.168.0.19:45672/%2f");
factory.setUri("amqp://guest:guest@192.168.0.100:35677/%2f");
Connection conn = factory.newConnection();
Channel channel = conn.createChannel();
channel.queueDeclare(queueName, true, false, false, null);
@@ -348,6 +348,34 @@ public class OpenApiClient {
}
public static JSONObject standard1Action(String city, String address) {
try {
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("address", address);
params.put("token", TOKEN);
params.put("__refresh_cache","1")
String content = new HttpRequest().get("https://open.fangjia.com/address/standard", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/address/standard", params, "utf-8")
println("${city} ${address} 程序调用异常")
Thread.sleep(6000)
}
if (result.getInt("code") == 200) {
return result.getJSONObject("result");
} else if (result.getInt("code") !=0) {
println("${city} ${address}")
}
} catch (Exception e) {
println("${city} ${address} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject standardize(String city, String address) {
try {
Map<String, String> params = new HashMap<String, String>();