diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java index 392a896a4..28f780026 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java @@ -467,11 +467,15 @@ public class AddrSvcController { } else { map.put("addrVerify", "地址有效"); map.put("typeName", areaMap.get(map.get("typeCode"))); + map.put("addrArea", addrArea(map, type)); + map.put("countyLevelCity", countyLevelCity(map)); basicModel.setResult(map); } } else { map.put("addrVerify", "地址有效"); map.put("typeName", areaMap.get(map.get("typeCode"))); + map.put("addrArea", addrArea(map, type)); + map.put("countyLevelCity", countyLevelCity(map)); basicModel.setResult(map); } @@ -498,6 +502,45 @@ public class AddrSvcController { } return mav; } + + /** + * 判断县级市 + */ + public String countyLevelCity(Map map) { + if (map.get("city") == null) { + return ""; + } + if (map.get("city").endsWith("区")) { + return "否"; + } + if (map.get("city").equals("东莞市") || map.get("city").equals("嘉峪关市") || map.get("city").equals("中山市") || map.get("city").equals("儋州市") ) { + return "否"; + } + return "是"; + } + + /** + * 判断所属区域 + * + * @param map + * @return + */ + public String addrArea(Map map, String type) { + if (map.get("verifyFlag") == null ) { + return ""; + } + if (map.get("verifyFlag").equals("非农")) { + return "城市区域"; + } + if (map.get("verifyFlag").equals("农")) { + return "农村区域"; + } + if ("2".equals(type) && "是".equals(map.get("countyGov"))) { + return "县城城区"; + } + return ""; + } + /** * 涉农识别-新规则-上海征信 * @@ -565,11 +608,15 @@ public class AddrSvcController { basicModel.setResult(result); } else { map.put("addrVerify", "地址有效"); + map.put("addrArea", addrArea(map, type)); + map.put("countyLevelCity", countyLevelCity(map)); map.put("typeName", areaMap.get(map.get("typeCode"))); basicModel.setResult(map); } } else { map.put("addrVerify", "地址有效"); + map.put("addrArea", addrArea(map, type)); + map.put("countyLevelCity", countyLevelCity(map)); map.put("typeName", areaMap.get(map.get("typeCode"))); basicModel.setResult(map); } @@ -703,11 +750,15 @@ public class AddrSvcController { basicModel.setResult(result); } else { map.put("addrVerify", "地址有效"); + map.put("addrArea", addrArea(map, type)); + map.put("countyLevelCity", countyLevelCity(map)); map.put("typeName", areaMap.get(map.get("typeCode"))); basicModel.setResult(map); } } else { map.put("addrVerify", "地址有效"); + map.put("addrArea", addrArea(map, type)); + map.put("countyLevelCity", countyLevelCity(map)); map.put("typeName", areaMap.get(map.get("typeCode"))); basicModel.setResult(map); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy new file mode 100644 index 000000000..d1ec16934 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy @@ -0,0 +1,40 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 将国信达及成交对标数据加入到第三方均价表,做为均价参考 + */ +class AddThirdPriceSample { + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new AddThirdPriceSample() + task.run() + } + def run(){ + for(def i=1;i<19;i++){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d-> + if(d['A'] && d['C']){ + def dist = CommonDatautils.findDistrictById(d['A'].toString().trim()) + if(dist){ + def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407) + def udata = new BasicDBObject("zhongyuan_esf_price", d['C'].toString().toDouble().toInteger()) + + AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false) + }else{ + println d['A'] +" 没查到" + } + } + }, i) + + println(i) + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index db19c3e41..8a924e652 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -70,12 +70,12 @@ class BigdataDealBatch extends BaseBatch{ //664b73800000000000000000 5-21 def citys = CcbCommonUtils.getRunningCitys(args) -// println "curDate:"+curDate -// def allCitys = CommonDatautils.findAllCity() -// for(def city : allCitys){ -// addDealPriceMatch(city); -// } -// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月 + println "curDate:"+curDate + def allCitys = CommonDatautils.findAllCity() + for(def city : allCitys){ + addDealPriceMatch(city); + } + updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月 def rows = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 13a46482e..0d36523e6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -612,9 +612,9 @@ def findAllData(sqlInstance236, citycode){ ////},0) // -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d-> +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\删除小区.xlsx",{d-> // if(d['A']){ -// def dbo = CommonDatautils.findDistrictById(d['D']) +// def dbo = CommonDatautils.findDistrictById(d['A']) // if(dbo){ // def udata = new BasicDBObject() //// udata.put("name_cat", 11) @@ -630,6 +630,38 @@ def findAllData(sqlInstance236, citycode){ // } //},0) + +ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d-> + if(d['A'] && d['B']){ + def dbo = CommonDatautils.findDistrictById(d['A']) + if(dbo){ + def alias = null + if(!dbo.alias){ + alias = [] + }else{ + alias = dbo.alias + } + if(!alias.contains(d['B'])){ + alias << d['B'] + } + if(alias){ + def udata = new BasicDBObject() + udata.put("alias", alias) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", dbo._id) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + }else{ + println d['A']+" "+"别名已存在" + } + + }else{ + println dbo._id.toString() +" 没找到" + } + } +}) + //def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] //for(def c : citys){ // def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"]) @@ -1299,34 +1331,34 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx" - -ExcelUtils.multiThreadingModify(fileName, {d-> - if(d['J']){ -// def region = RegionData.formatRegion(d['A'], d['J']) -// if(region){ -// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") +//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx" +// +//ExcelUtils.multiThreadingModify(fileName, {d-> +// if(d['J']){ +//// def region = RegionData.formatRegion(d['A'], d['J']) +//// if(region){ +//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") +//// if(dist){ +//// d['B'] = dist.region +//// d['C'] = dist.name +//// d['L'] = dist.status-dist.name_cat +//// } +//// } +//// if(d['J'] instanceof String){ +// def dist = CommonDatautils.findDistrictById(d['J']) // if(dist){ +// d['A'] = dist.city // d['B'] = dist.region // d['C'] = dist.name -// d['L'] = dist.status-dist.name_cat +// d['K'] = dist.estate_type2 +// d['L'] = dist.status +// d['M'] = dist.name_cat +// +//// d['H'] = dist.name // } -// } -// if(d['J'] instanceof String){ - def dist = CommonDatautils.findDistrictById(d['J']) - if(dist){ - d['A'] = dist.city - d['B'] = dist.region - d['C'] = dist.name - d['K'] = dist.estate_type2 - d['L'] = dist.status - d['M'] = dist.name_cat - -// d['H'] = dist.name - } -// } - } -},0) +//// } +// } +//},0) //def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 52fd3ddb2..5f5a53f33 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -257,6 +257,9 @@ class ThirdPartyPriceBatch extends BaseBatch { if(third.get("5i5j_esf_price")){ price_map.wiwj_avg_price = third.get("5i5j_esf_price") } + if(third.get("fccs_esf_price")){ + price_map.fccs_avg_price = third.get("fccs_esf_price") + } dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") allRows << dataMap } @@ -303,6 +306,8 @@ class ThirdPartyPriceBatch extends BaseBatch { web = "房多多" }else if(k=="wiwj_avg_price"){ web = "我爱我家" + }else if(k=="fccs_avg_price"){ + web = "房产超市" } ret << sourceMap.get(web) @@ -584,6 +589,10 @@ class ThirdPartyPriceBatch extends BaseBatch { if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){ dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price")) } + + if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){ + dataMap.put("fccs_esf_price", db.get("fccs_esf_price")) + } // if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){ // map.put(key, db) // } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index d36b853dc..b9b6bd4bf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1212,6 +1212,7 @@ class CommonDatautils { public static def findAllCount(String cat){ DBObject q = new BasicDBObject(); q.put("cat", cat); + q.put("status", 0) return SEAWEED_MONGO_COLL.count(q) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BaiduBaikeTown.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BaiduBaikeTown.groovy new file mode 100644 index 000000000..77356d988 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BaiduBaikeTown.groovy @@ -0,0 +1,102 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.google.gson.JsonObject +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.jsoup.Jsoup + +class BaiduBaikeTown { + + DBCollection dzb_COLL = MongoUtils.getCollectionByName("crawl", "development_zone_bdbk", "192.168.0.206",27020) + + + static main(args) { + def task = new BaiduBaikeTown() + task.run() + } + + def run(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",{row-> + if(!row['E'] && row['A']=="郑州经济技术开发区"){ + def q = new BasicDBObject() + q.put("desc",new BasicDBObject('$ne', 'null')) + q.put("name",row['A']) + def d = dzb_COLL.findOne(q) + def townMap = parse(d.desc, d.name) + row['D'] = townMap.town_str1 + row['F'] = townMap.town_str2 + +// def ret = [] +// ret << ["名称", "基本信息", "目录", "全文"] +// for(def d : dbcursor){ +// def townMap = parse(d.desc, d.name) +// if(townMap.town){ +// println d.name+" "+townMap.town +// } +// ret << [d.name, townMap.town, townMap.towns, townMap.town_str1, townMap.town_str2] +// } + } + + }) + +// ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx", ret, "数据") + } + + def parse(desc, name){ + def map = [:] + if(desc){ + def doc = Jsoup.parse(desc) + def town = JQueryUtils.selectText(".basicInfoBlock_Ex6IU div:contains(下辖地区) dd", doc) + + def s = doc.toString() + def dataStr = DataHandleUtils.findMatchPattern(s,"PAGE_DATA=.*").replace("PAGE_DATA= ","").replace("","") + def jsonObj = JSONObject.fromObject(dataStr) + + def str = DataHandleUtils.findMatchPattern(doc.text(),"下辖.*[街道|镇]、?和?") + def towns = [] + if(jsonObj && jsonObj.catalog){ + def catalogs = jsonObj.catalog + def index = null + for(def log : catalogs){ + def curIndex = log.index + if(log.title=="行政区划"){ + index = log.index + } + if(index && curIndex.toString().startsWith(index+"-")){ + towns << log.title + } + } + } + + + + map.town_str1 = str + if(town){ + map.town = town + }else{ + map.town = "" + } + + if(towns){ + map.towns = towns.join(",") + }else{ + map.towns = "" + } + + + def str2 = DataHandleUtils.findMatchPattern(doc.text(),"开发区辖.*[,|。]") + map.town_str2 = str2 + return map + } + map.town_str1 = "" + map.town_str2 = "" + map.town = "" + map.towns = "" + return map + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 772bbaf62..411c7e92e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -25,9 +25,11 @@ class DevProcess { def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) - DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) + DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) + DBCollection DEV_MAPPING_CONN2 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping_2", MONGO100_URL) + DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") @@ -54,7 +56,7 @@ class DevProcess { // task.createDevMapping() // task.deleteAddr() // task.deleteAddr2() -// task.synchDevTo43() +// task.synchDevTo100() // task.addNewDev2() // task.addAlias() @@ -71,6 +73,8 @@ class DevProcess { // task.devToPark() //开发区下辖园区 // task.checkTown() +// task.repeatSave() + } @@ -692,13 +696,13 @@ class DevProcess { }); } - def synchDevTo43(){ - def dbcursor = development_zone_CONN.find() + def synchDevTo100(){ + def dbcursor = DEVELOPMENT_ZONE_43_COLL.find() def rets = [] for(def r : dbcursor){ rets << r } - DEVELOPMENT_ZONE_43_COLL.insert(rets) + development_zone_CONN.insert(rets) } @@ -898,66 +902,89 @@ class DevProcess { } def devToPark(){ - def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\开发区下辖街道-处理后表格.xlsx",0, 1) + def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",0, 1) def rets = [] rets << ["开发区名称", "下辖街道", "城市", "区域"] for(def d : datas){ if(d['E']){ - def arrs = d['E'].split(",") - def q = new BasicDBObject().append("name", d['B']) + def arrs = d['E'].split("、") + def q = new BasicDBObject().append("name", d['A']) def obj = development_zone_CONN.findOne(q) if(obj){ for(def park : arrs){ - rets << [d['B'], park, obj.city, obj.district] + rets << [d['A'], park, obj.city, obj.district] } } } } - ExcelUtils.write("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx", rets, "下辖街道") + ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx", rets, "下辖街道") } def checkTown(){ - ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx",{d-> if(!d['E']){ if(d['A']){ -// if(d['A']=="长春高新技术产业开发区"){ - def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['A']) - if(jsonObject1) { - if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') { - def township = jsonObject1.get("township") - def district = jsonObject1.get("district") - d['E'] = "接口" - if(district && district!='null'){ - d['F'] = district - } - if(township && township!='null'){ - d['G'] = township - } - - d['H'] = jsonObject1.get("townCode") - if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){ - d['I'] = jsonObject1.get("provinceCode") - } - } -// } - } - -// def q = new BasicDBObject() -// q.put("city", d['C']) -//// q.put("district", d['D']) -// q.put("mzb_district_name", d['D']) -// q.put("type", "乡镇街道") -// q.put("street", d['B']) -// def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q) -// if(obj){ -// d['E'] = "可入库2" -// d['F'] = obj.district +// def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B']) +// if(jsonObject1) { +// if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') { +// def township = jsonObject1.get("township") +// def district = jsonObject1.get("district") +// d['E'] = "接口" +// if(district && district!='null'){ +// d['F'] = district +// } +// if(township && township!='null'){ +// d['G'] = township +// } +// +// d['H'] = jsonObject1.get("townCode") +// if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){ +// d['I'] = jsonObject1.get("provinceCode") +// } +// } // } + + def q = new BasicDBObject() + q.put("city", d['C']) +// q.put("district", d['D']) +// q.put("mzb_district_name", d['D']) + q.put("type", "乡镇街道") + q.put('street',new BasicDBObject('$regex', Pattern.compile(d['B'].toString().trim()+'.*$', Pattern.CASE_INSENSITIVE))) +// q.put("street", d['B']) +// q.put('alias', new BasicDBObject('$in',[d['B']])) + def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q) + if(obj){ + d['E'] = "可入库-忽略区域模糊2" + d['F'] = obj.district + d['G'] = obj.street + } } } - },1) + },0) + } + + def repeatSave(){ + def dbcursor = DEV_MAPPING_CONN.find(new BasicDBObject()) + def list = [] + for(def map : dbcursor){ + def columns = ["_id","province","city","district","name","type","address","dev_district", "township","park"] + def udata = new BasicDBObject() + map.each{k,v-> + if(columns.contains(k)){ + if(v instanceof String){ + if(k=="_id"){ + udata.put("_id", new ObjectId(v)) + }else{ + udata.put(k, v) + } + } + } + } + list << udata + } + DEV_MAPPING_CONN2.insert(list) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index 405fdbde8..052f2d046 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -29,7 +29,7 @@ class DevUpdateTestBatch { static main(args) { def task = new DevUpdateTestBatch() - def path = "D:\\数据\\数据建设\\开发区\\文心一言获取开发区下辖园区及街道.xlsx"//开发区-园区.xlsx + def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx // task.addDev(path) task.addDevAddress(path) @@ -137,26 +137,27 @@ class DevUpdateTestBatch { DEV_MAPPING_100_CONN.save(o) - o.id = o._id.toString() - o.remove("_id") - o.remove("c_date") - addEs(o, "192.168.0.89", "dev_mapping") - try{ - addEs(o, "192.168.0.52", "dev_mapping") - }catch(Exception e){ - println "52 es error: "+e.printStackTrace() - } +// o.id = o._id.toString() +// o.remove("_id") +// o.remove("c_date") +// addEs(o, "192.168.0.89", "dev_mapping") +// try{ +// addEs(o, "192.168.0.52", "dev_mapping") +// }catch(Exception e){ +// println "52 es error: "+e.printStackTrace() +// } d['D'] = "已完成" }else{ - d['D'] = "存在" + d['D'] = "重复" println d['A'] +" "+ d['C']+" 重复" } }else{ + d['D'] = "不存在" println d['A'] +" 不存在" } } - },2) + },1) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy index f814cceb2..61ed0ea7b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy @@ -40,11 +40,11 @@ class GXDdistrictMatch { def addOldMatch(){ //老匹配 - def fileName = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"//11城重新匹配.xlsx + def fileName = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"//11城重新匹配.xlsx ExcelUtils.multiThreadingModify(fileName, { d -> println d['C'] if(!d['AJ'] && d['I']){ - def jsonObject = OpenApiClient.addressMatch(d['C'], d['I'], "district", false,null, d['D']) + def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D']) if (jsonObject != null && !jsonObject.containsKey("result")) { d['AJ'] = jsonObject.get("id") d['AK'] = jsonObject.get("city") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy new file mode 100644 index 000000000..327e48a2e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy @@ -0,0 +1,230 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.regex.Pattern + +/** + * 名录集 + */ +class MinglujiData { + + DBCollection mlj_COLL = MongoUtils.getCollectionByName("crawl", "kfq_mingluji_com", "192.168.0.206",27020) + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + + DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) + DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) + + static main(args) { + def task = new MinglujiData() + task.run() + } + + def run(){ +// exportDatas("湖州") +// matchDev("浙江省","湖州") + matchCommunity("湖州") + } + def exportDatas(city){ + def datas = findMinglujiDatas(city) + ExcelUtils.write("D:\\数据\\数据建设\\名录集\\${city}.xlsx", datas, "数据") + } + def matchDev(prov, city){ + def devs = loadAllDevs(prov) + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d-> + if(d['E'] && !d['N'] ){ + def obj = findDevByName(d['E']) + if(!obj){ + obj = findDevByAlias(d['E']) + if(!obj){ + obj = findDebBySimlar(devs, d['E'], d['B']) + if(!obj){ + obj = findDevByPark(d['A'], d['B'], d['E']) + if (!obj) { + }else{ + d['N'] = "园区" + } + }else{ + d['N'] = "相似" + } + }else{ + d['N'] = "别名" + } + }else{ + d['N'] = "相同" + } + if(obj){ + d['M'] = obj.name + d['O'] = obj.city + d['P'] = obj.district + } + } + }) + } + + def matchCommunity(city){ + ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}-没配居委1.xlsx",{d-> + if(d['A'] && d['B'] && d['L'] && !d['Q']){ + def list = findCommunityByName(d['A'], d['B'], d['L']) + if(!list){ + list = findCommunityBySimlar(d['A'], d['B'], d['L']) + if(!list){ + + }else{ + d['T'] = "包含" + } + }else{ + d['T'] = "相同" + } + if(list){ + if(list.size()==1){ + def obj = list[0] + d['Q'] = obj.name + d['R'] = obj.street_name + }else if(list.size()>=2){ + for(def l : list){ + if(d['C']==l.area_name){ + d['Q'] = l.name + d['R'] = l.street_name + break; + } + } + d['S'] = list.size() + } + } + } + }, 0) + } + + def loadAllDevs(prov){ + def q = new BasicDBObject() + q.put("province", prov) + return development_zone_CONN.find(q).toList() + } + + def findDebBySimlar(def devs, def name, def city){ + name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园") + def last = null + for(def d : devs){ + def alias1 = null + if(!d.alias){ + alias1 = [] + }else{ + alias1 = d.alias + } + if(!alias1.contains(d.name)){ + alias1 << d.name + } + if(alias1){ + try { + for(def n : alias1){ + n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园") + if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){ + last = d + break + } + } + }catch(exception){ + println alias1 + } + + } + if(last){ + break + } + } + return last + } + def findDevByName(name){ + def q = new BasicDBObject() + q.put("name", name) + return development_zone_CONN.findOne(q) + } + + def findDevByAlias(name){ + def q = new BasicDBObject() + q.put('alias', new BasicDBObject('$in',[name])) + return development_zone_CONN.findOne(q) + } + + def findDevBySimlar(name){ + def q = new BasicDBObject() + q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 + return development_zone_CONN.findOne(q) + } + + def findDevByPark(prov, city, name){ + def q = new BasicDBObject() + q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 + q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 + q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 + q.put("type", "园区") + return dev_mapping_CONN.findOne(q) + } + + def findCommunityByName(province, city, name){ + def q = new BasicDBObject() + q.put("province_name", province+"省") + q.put("city_name", city+"市") + q.put("name", name) + def dbcursor = community_100_CONN.find(q) + def list = [] + for(def d : dbcursor){ + list << d + } + return list + } + + def findCommunityBySimlar(province, city, name){ + def q = new BasicDBObject() + q.put("province_name", province+"省") + q.put("city_name", city+"市") + q.put("name", new BasicDBObject('$regex', Pattern.compile('^'+name+'.*$', Pattern.CASE_INSENSITIVE))) + def dbcursor = community_100_CONN.find(q) + def list = [] + for(def d : dbcursor){ + list << d + } + return list + } + + def findMinglujiDatas(city){ + def keys = new BasicDBObject() + keys.put("c_address",1) + keys.put("c_city",1) + keys.put("c_committee",1) + keys.put("c_community",1) + keys.put("c_company",1) + keys.put("c_district",1) + keys.put("c_street",1) + keys.put("kind",1) + keys.put("name",1) + keys.put("province",1) + def datas = mlj_COLL.find(new BasicDBObject("c_city", city), keys) + def rets = [] + rets << ["省","城市","区域","街道","开发区名","类型","公司名","社区","居委","地址","拆分街道","拆分居委","匹配开发区名"] + for(def d : datas){ + def street = d.c_street + def splitStreet = "" + if(street){ + if(street.contains(d.c_district)){ + def start = street.toString().lastIndexOf(d.c_district)+d.c_district.size() + def end = street.toString().size() + splitStreet = street.toString().substring(start, end) + } + } + def splitCommunity = "" + if(d.c_committee){ + if(d.c_committee.contains(d.c_district)){ + splitCommunity = d.c_committee.substring(d.c_committee.lastIndexOf(d.c_district)+d.c_district.size(), d.c_committee.toString().size()) + } + } + rets << [d.province, d.c_city, d.c_district, d.c_street, d.name, d.kind, d.c_company, d.c_community, d.c_committee, d.c_address, splitStreet, splitCommunity] + } + return rets + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy index 16e99a043..f649b98ff 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy @@ -25,7 +25,7 @@ class FjwBigdataCommbuildtypeBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) //235为房价网内部计算均价的库 - def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy index 0291a1e8c..e01c76779 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy @@ -33,7 +33,7 @@ class FjwBigdataCommunityBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) //235为房价网内部计算均价的库 - def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy index 3234ddbbd..936382db2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy @@ -40,7 +40,7 @@ class FjwBigdataDealBatch extends BaseBatch{ static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") - def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy index afcedfa60..1cb508851 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy @@ -67,7 +67,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{ // def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", // "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") @@ -492,9 +492,9 @@ class FjwBigdataForsaleBatch extends BaseBatch{ def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){ - def date = "20200101"//DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd") - def startMdate = DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd") - def endMdate = DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd") + def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd") + def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd") + def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd") QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy index d7246795a..dae99b102 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy @@ -34,7 +34,7 @@ class FjwGuiDingPriceBatch extends BaseBatch{ def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15"; - def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy index f8a140633..eed97969b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy @@ -13,7 +13,7 @@ class FjwPgsqlCommUtils { public static def database = "fangjia" - public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/"+database+"?stringtype=unspecified", + public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/"+database+"?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") public static def findAllDistrictIdsByOracle(def cityid){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy index daa08b17c..baae234f8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy @@ -36,7 +36,7 @@ class FjwThirdPartyPriceBatch extends BaseBatch { "房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"] - def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") static main(args) { @@ -265,6 +265,10 @@ class FjwThirdPartyPriceBatch extends BaseBatch { if(third.get("5i5j_esf_price")){ price_map.wiwj_avg_price = third.get("5i5j_esf_price") } + + if(third.get("fccs_esf_price")){ + price_map.fccs_avg_price = third.get("fccs_esf_price") + } dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") allRows << dataMap } @@ -311,6 +315,8 @@ class FjwThirdPartyPriceBatch extends BaseBatch { web = "房多多" }else if(k=="wiwj_avg_price"){ web = "我爱我家" + }else if(k=="fccs_avg_price"){ + web = "房产超市" } ret << sourceMap.get(web) @@ -459,6 +465,11 @@ class FjwThirdPartyPriceBatch extends BaseBatch { if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){ dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price")) } + + if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){ + dataMap.put("fccs_esf_price", db.get("fccs_esf_price")) + } + // if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){ // map.put(key, db) // } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index 2dd8a91a7..14b6a5c26 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,11 +33,11 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx" + def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx" // addGXDPrice(filePath) -// addCcbPrice(filePath) + addCcbPrice(filePath) // addThirdPrice(filePath) - addFjwPrice(filePath) +// addFjwPrice(filePath) // exportPkPrice() // addRegion() // addTradePrice(filePath) @@ -169,9 +169,9 @@ class AddPriceTest { def addCcbPrice(filepath){ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() ExcelUtils.multiThreadingModify(filepath,{ d-> - if(d['AJ']){ - def cityId = cityCodeDataMap.get(d['AK']) - def map = findAvgPrice(d['AJ'], cityId) + if(d['U']){ + def cityId = cityCodeDataMap.get(d['V']) + def map = findAvgPrice(d['U'], cityId) if(map){ d['F'] = map.avg_price // d['P'] = map.calc_type @@ -183,7 +183,7 @@ class AddPriceTest { def findAvgPrice(def commid, def cityid){ - def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202406 order by gujiabds_clc_yrmo_val desc limit 1" + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1" def map = [:] pgsqlInstance.eachRow(sql, { r -> map.avg_price = r.gujia_bds_clc_avprc diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy index b6fea9896..c2f11e6f8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy @@ -27,7 +27,7 @@ class QiliangTest { // task.run() def data = [ - 'address': '江阴市澄杨路77号', + 'address': '景县开发区东区', 'type': '1' //1:对公 0:对私 ] def s = task.getProxyResult(data); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 024a20faf..b8d657c89 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -15,6 +15,8 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.ProvinceData import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.google.common.hash.BloomFilter +import com.google.common.hash.Funnels import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl @@ -29,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders +import java.nio.charset.StandardCharsets import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -164,10 +167,17 @@ class GaoDePoiDataProcess extends BaseBatch{ '陵水黎族自治县', '随州市', '雅安市', '青岛市', '鞍山市', '韶关市', '马鞍山市', '驻马店市', '鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市', '黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市'] + + //"丽水市","杭州市", + def runbuducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市"] for(def city : buducitys){ + if(runbuducitys.contains(city)){ + continue + } def pois = findAllVillagePoiMap(city) def rows = createVillageData(city, pois) + save(city, rows) // saveToMG(city, rows) println city +" "+pois.size()+" add:"+rows.size() @@ -182,8 +192,7 @@ class GaoDePoiDataProcess extends BaseBatch{ final CountDownLatch latch = new CountDownLatch(pois.size()) // def townMap = findTownRange(city) def townMap = CommonDatautils.queryTownByCity(city) - def poiKeys = findEsPoiByCity(city) - + def poiKeysFilter = findEsPoiByCity(city) for(def i=0;i filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001); ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi_single", queryBuilder, { String id, Map d-> - rets << d.city +"##"+d.district+"##"+d.name +// rets << d.city +"##"+d.district+"##"+d.name + filter.put(d.city +"##"+d.district+"##"+d.name) }) - return rets + return filter } def queryMapPoi(city){ @@ -314,7 +325,6 @@ class GaoDePoiDataProcess extends BaseBatch{ // retMap.small_type = "村庄级地名" retMap.cat = map.std_tag - def nameArr = map.std_tag.split(";|:") if(nameArr.size()>=1){ retMap.type = nameArr[0] @@ -408,6 +418,9 @@ class GaoDePoiDataProcess extends BaseBatch{ } def findAllVillagePoiMap(city){ + def q = new BasicDBObject("city", city); + q.put("std_tag", new BasicDBObject('$exists', true)) + def keys = new BasicDBObject() keys.put("city", 1) keys.put("district", 1) @@ -416,7 +429,7 @@ class GaoDePoiDataProcess extends BaseBatch{ keys.put("gcj02_x", 1) keys.put("gcj02_y", 1) keys.put("std_tag", 1) - def list = BAIDU_POI_COLL.find(new BasicDBObject("city", city), keys).toList() + def list = BAIDU_POI_COLL.find(q, keys).toList() return list; }