From e9adefff80fc0d996417782d0303ec04bc0667f6 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Fri, 16 Aug 2024 10:17:58 +0800 Subject: [PATCH 01/10] update --- .../script/commData/CommonDatautils.groovy | 1 + .../gj/data_handle/BaiduBaikeTown.groovy | 102 ++++++++++++++++++ .../script/gj/data_handle/DevProcess.groovy | 83 +++++++------- .../gj/data_handle/DevUpdateTestBatch.groovy | 25 ++--- .../gj/data_handle/GXDdistrictMatch.groovy | 4 +- .../fangjia/script/gj/poc/AddPriceTest.groovy | 14 +-- 6 files changed, 167 insertions(+), 62 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BaiduBaikeTown.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index d36b853dc..b9b6bd4bf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1212,6 +1212,7 @@ class CommonDatautils { public static def findAllCount(String cat){ DBObject q = new BasicDBObject(); q.put("cat", cat); + q.put("status", 0) return SEAWEED_MONGO_COLL.count(q) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BaiduBaikeTown.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BaiduBaikeTown.groovy new file mode 100644 index 000000000..77356d988 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BaiduBaikeTown.groovy @@ -0,0 +1,102 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.google.gson.JsonObject +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.jsoup.Jsoup + +class BaiduBaikeTown { + + DBCollection dzb_COLL = MongoUtils.getCollectionByName("crawl", "development_zone_bdbk", "192.168.0.206",27020) + + + static main(args) { + def task = new BaiduBaikeTown() + task.run() + } + + def run(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",{row-> + if(!row['E'] && row['A']=="郑州经济技术开发区"){ + def q = new BasicDBObject() + q.put("desc",new BasicDBObject('$ne', 'null')) + q.put("name",row['A']) + def d = dzb_COLL.findOne(q) + def townMap = parse(d.desc, d.name) + row['D'] = townMap.town_str1 + row['F'] = townMap.town_str2 + +// def ret = [] +// ret << ["名称", "基本信息", "目录", "全文"] +// for(def d : dbcursor){ +// def townMap = parse(d.desc, d.name) +// if(townMap.town){ +// println d.name+" "+townMap.town +// } +// ret << [d.name, townMap.town, townMap.towns, townMap.town_str1, townMap.town_str2] +// } + } + + }) + +// ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx", ret, "数据") + } + + def parse(desc, name){ + def map = [:] + if(desc){ + def doc = Jsoup.parse(desc) + def town = JQueryUtils.selectText(".basicInfoBlock_Ex6IU div:contains(下辖地区) dd", doc) + + def s = doc.toString() + def dataStr = DataHandleUtils.findMatchPattern(s,"PAGE_DATA=.*").replace("PAGE_DATA= ","").replace("","") + def jsonObj = JSONObject.fromObject(dataStr) + + def str = DataHandleUtils.findMatchPattern(doc.text(),"下辖.*[街道|镇]、?和?") + def towns = [] + if(jsonObj && jsonObj.catalog){ + def catalogs = jsonObj.catalog + def index = null + for(def log : catalogs){ + def curIndex = log.index + if(log.title=="行政区划"){ + index = log.index + } + if(index && curIndex.toString().startsWith(index+"-")){ + towns << log.title + } + } + } + + + + map.town_str1 = str + if(town){ + map.town = town + }else{ + map.town = "" + } + + if(towns){ + map.towns = towns.join(",") + }else{ + map.towns = "" + } + + + def str2 = DataHandleUtils.findMatchPattern(doc.text(),"开发区辖.*[,|。]") + map.town_str2 = str2 + return map + } + map.town_str1 = "" + map.town_str2 = "" + map.town = "" + map.towns = "" + return map + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 772bbaf62..69a4318b0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -37,7 +37,7 @@ class DevProcess { def task = new DevProcess(); // task.run() // task.parseCityAndDist() - task.devSplitRoad() +// task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() // task.matchDev() @@ -69,7 +69,7 @@ class DevProcess { // task.addKfc() // task.devToPark() //开发区下辖园区 -// task.checkTown() + task.checkTown() } @@ -898,66 +898,67 @@ class DevProcess { } def devToPark(){ - def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\开发区下辖街道-处理后表格.xlsx",0, 1) + def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",0, 1) def rets = [] rets << ["开发区名称", "下辖街道", "城市", "区域"] for(def d : datas){ if(d['E']){ - def arrs = d['E'].split(",") - def q = new BasicDBObject().append("name", d['B']) + def arrs = d['E'].split("、") + def q = new BasicDBObject().append("name", d['A']) def obj = development_zone_CONN.findOne(q) if(obj){ for(def park : arrs){ - rets << [d['B'], park, obj.city, obj.district] + rets << [d['A'], park, obj.city, obj.district] } } } } - ExcelUtils.write("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx", rets, "下辖街道") + ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx", rets, "下辖街道") } def checkTown(){ - ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx",{d-> if(!d['E']){ if(d['A']){ -// if(d['A']=="长春高新技术产业开发区"){ - def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['A']) - if(jsonObject1) { - if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') { - def township = jsonObject1.get("township") - def district = jsonObject1.get("district") - d['E'] = "接口" - if(district && district!='null'){ - d['F'] = district - } - if(township && township!='null'){ - d['G'] = township - } - - d['H'] = jsonObject1.get("townCode") - if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){ - d['I'] = jsonObject1.get("provinceCode") - } - } -// } - } - -// def q = new BasicDBObject() -// q.put("city", d['C']) -//// q.put("district", d['D']) -// q.put("mzb_district_name", d['D']) -// q.put("type", "乡镇街道") -// q.put("street", d['B']) -// def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q) -// if(obj){ -// d['E'] = "可入库2" -// d['F'] = obj.district +// def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B']) +// if(jsonObject1) { +// if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') { +// def township = jsonObject1.get("township") +// def district = jsonObject1.get("district") +// d['E'] = "接口" +// if(district && district!='null'){ +// d['F'] = district +// } +// if(township && township!='null'){ +// d['G'] = township +// } +// +// d['H'] = jsonObject1.get("townCode") +// if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){ +// d['I'] = jsonObject1.get("provinceCode") +// } +// } // } + + def q = new BasicDBObject() + q.put("city", d['C']) +// q.put("district", d['D']) +// q.put("mzb_district_name", d['D']) + q.put("type", "乡镇街道") + q.put('street',new BasicDBObject('$regex', Pattern.compile(d['B'].toString().trim()+'.*$', Pattern.CASE_INSENSITIVE))) +// q.put("street", d['B']) +// q.put('alias', new BasicDBObject('$in',[d['B']])) + def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q) + if(obj){ + d['E'] = "可入库-忽略区域模糊2" + d['F'] = obj.district + d['G'] = obj.street + } } } - },1) + },0) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index 405fdbde8..8af68c5fa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -29,7 +29,7 @@ class DevUpdateTestBatch { static main(args) { def task = new DevUpdateTestBatch() - def path = "D:\\数据\\数据建设\\开发区\\文心一言获取开发区下辖园区及街道.xlsx"//开发区-园区.xlsx + def path = "D:\\数据\\数据建设\\开发区\\开发区下辖园区及街道\\前瞻网开发区下辖园区入库.xlsx"//开发区-园区.xlsx // task.addDev(path) task.addDevAddress(path) @@ -137,26 +137,27 @@ class DevUpdateTestBatch { DEV_MAPPING_100_CONN.save(o) - o.id = o._id.toString() - o.remove("_id") - o.remove("c_date") - addEs(o, "192.168.0.89", "dev_mapping") - try{ - addEs(o, "192.168.0.52", "dev_mapping") - }catch(Exception e){ - println "52 es error: "+e.printStackTrace() - } +// o.id = o._id.toString() +// o.remove("_id") +// o.remove("c_date") +// addEs(o, "192.168.0.89", "dev_mapping") +// try{ +// addEs(o, "192.168.0.52", "dev_mapping") +// }catch(Exception e){ +// println "52 es error: "+e.printStackTrace() +// } d['D'] = "已完成" }else{ - d['D'] = "存在" + d['D'] = "重复" println d['A'] +" "+ d['C']+" 重复" } }else{ + d['D'] = "不存在" println d['A'] +" 不存在" } } - },2) + },0) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy index f814cceb2..61ed0ea7b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy @@ -40,11 +40,11 @@ class GXDdistrictMatch { def addOldMatch(){ //老匹配 - def fileName = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"//11城重新匹配.xlsx + def fileName = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"//11城重新匹配.xlsx ExcelUtils.multiThreadingModify(fileName, { d -> println d['C'] if(!d['AJ'] && d['I']){ - def jsonObject = OpenApiClient.addressMatch(d['C'], d['I'], "district", false,null, d['D']) + def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D']) if (jsonObject != null && !jsonObject.containsKey("result")) { d['AJ'] = jsonObject.get("id") d['AK'] = jsonObject.get("city") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index 2dd8a91a7..14b6a5c26 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,11 +33,11 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx" + def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx" // addGXDPrice(filePath) -// addCcbPrice(filePath) + addCcbPrice(filePath) // addThirdPrice(filePath) - addFjwPrice(filePath) +// addFjwPrice(filePath) // exportPkPrice() // addRegion() // addTradePrice(filePath) @@ -169,9 +169,9 @@ class AddPriceTest { def addCcbPrice(filepath){ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() ExcelUtils.multiThreadingModify(filepath,{ d-> - if(d['AJ']){ - def cityId = cityCodeDataMap.get(d['AK']) - def map = findAvgPrice(d['AJ'], cityId) + if(d['U']){ + def cityId = cityCodeDataMap.get(d['V']) + def map = findAvgPrice(d['U'], cityId) if(map){ d['F'] = map.avg_price // d['P'] = map.calc_type @@ -183,7 +183,7 @@ class AddPriceTest { def findAvgPrice(def commid, def cityid){ - def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202406 order by gujiabds_clc_yrmo_val desc limit 1" + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1" def map = [:] pgsqlInstance.eachRow(sql, { r -> map.avg_price = r.gujia_bds_clc_avprc From df68744bcf579ccfca0acb4547203f64c833eedb Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Fri, 16 Aug 2024 14:14:55 +0800 Subject: [PATCH 02/10] =?UTF-8?q?5432-=E3=80=8B5433?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy | 2 +- .../script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy | 2 +- .../script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy | 2 +- .../script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy | 2 +- .../script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy | 2 +- .../fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy | 2 +- .../script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy index 16e99a043..f649b98ff 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy @@ -25,7 +25,7 @@ class FjwBigdataCommbuildtypeBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) //235为房价网内部计算均价的库 - def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy index 0291a1e8c..e01c76779 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy @@ -33,7 +33,7 @@ class FjwBigdataCommunityBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) //235为房价网内部计算均价的库 - def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy index 3234ddbbd..936382db2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy @@ -40,7 +40,7 @@ class FjwBigdataDealBatch extends BaseBatch{ static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") - def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy index afcedfa60..5f43810f6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy @@ -67,7 +67,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{ // def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", // "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy index d7246795a..dae99b102 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy @@ -34,7 +34,7 @@ class FjwGuiDingPriceBatch extends BaseBatch{ def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15"; - def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy index f8a140633..eed97969b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy @@ -13,7 +13,7 @@ class FjwPgsqlCommUtils { public static def database = "fangjia" - public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/"+database+"?stringtype=unspecified", + public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/"+database+"?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") public static def findAllDistrictIdsByOracle(def cityid){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy index daa08b17c..52d65e101 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy @@ -36,7 +36,7 @@ class FjwThirdPartyPriceBatch extends BaseBatch { "房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"] - def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") static main(args) { From 6faa15235d4c137fd9d504d3513322e7c1799617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 19 Aug 2024 13:22:37 +0800 Subject: [PATCH 03/10] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddrSvcController.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java index 392a896a4..28f780026 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java @@ -467,11 +467,15 @@ public class AddrSvcController { } else { map.put("addrVerify", "地址有效"); map.put("typeName", areaMap.get(map.get("typeCode"))); + map.put("addrArea", addrArea(map, type)); + map.put("countyLevelCity", countyLevelCity(map)); basicModel.setResult(map); } } else { map.put("addrVerify", "地址有效"); map.put("typeName", areaMap.get(map.get("typeCode"))); + map.put("addrArea", addrArea(map, type)); + map.put("countyLevelCity", countyLevelCity(map)); basicModel.setResult(map); } @@ -498,6 +502,45 @@ public class AddrSvcController { } return mav; } + + /** + * 判断县级市 + */ + public String countyLevelCity(Map map) { + if (map.get("city") == null) { + return ""; + } + if (map.get("city").endsWith("区")) { + return "否"; + } + if (map.get("city").equals("东莞市") || map.get("city").equals("嘉峪关市") || map.get("city").equals("中山市") || map.get("city").equals("儋州市") ) { + return "否"; + } + return "是"; + } + + /** + * 判断所属区域 + * + * @param map + * @return + */ + public String addrArea(Map map, String type) { + if (map.get("verifyFlag") == null ) { + return ""; + } + if (map.get("verifyFlag").equals("非农")) { + return "城市区域"; + } + if (map.get("verifyFlag").equals("农")) { + return "农村区域"; + } + if ("2".equals(type) && "是".equals(map.get("countyGov"))) { + return "县城城区"; + } + return ""; + } + /** * 涉农识别-新规则-上海征信 * @@ -565,11 +608,15 @@ public class AddrSvcController { basicModel.setResult(result); } else { map.put("addrVerify", "地址有效"); + map.put("addrArea", addrArea(map, type)); + map.put("countyLevelCity", countyLevelCity(map)); map.put("typeName", areaMap.get(map.get("typeCode"))); basicModel.setResult(map); } } else { map.put("addrVerify", "地址有效"); + map.put("addrArea", addrArea(map, type)); + map.put("countyLevelCity", countyLevelCity(map)); map.put("typeName", areaMap.get(map.get("typeCode"))); basicModel.setResult(map); } @@ -703,11 +750,15 @@ public class AddrSvcController { basicModel.setResult(result); } else { map.put("addrVerify", "地址有效"); + map.put("addrArea", addrArea(map, type)); + map.put("countyLevelCity", countyLevelCity(map)); map.put("typeName", areaMap.get(map.get("typeCode"))); basicModel.setResult(map); } } else { map.put("addrVerify", "地址有效"); + map.put("addrArea", addrArea(map, type)); + map.put("countyLevelCity", countyLevelCity(map)); map.put("typeName", areaMap.get(map.get("typeCode"))); basicModel.setResult(map); } From 8d819cef951c280f93afdf6af2c6d9e408695da6 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Mon, 19 Aug 2024 13:27:27 +0800 Subject: [PATCH 04/10] =?UTF-8?q?=E5=86=85=E9=83=A8=E6=88=BF=E6=BA=90?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E4=BF=AE=E6=94=B9=E5=8F=8A=E7=99=BE=E5=BA=A6?= =?UTF-8?q?=E5=85=A5=E5=BA=93=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/data_handle/DevProcess.groovy | 38 +++++-- .../gj/data_handle/DevUpdateTestBatch.groovy | 4 +- .../FjwBigdataForsaleBatch.groovy | 6 +- .../script/gj/village/QiliangTest.groovy | 2 +- .../gj/xiamen/GaoDePoiDataProcess.groovy | 100 ++++++++++-------- 5 files changed, 92 insertions(+), 58 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 69a4318b0..9b4d34ff1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -25,9 +25,11 @@ class DevProcess { def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) - DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) + DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) + DBCollection DEV_MAPPING_CONN2 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping_2", MONGO100_URL) + DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") @@ -54,7 +56,7 @@ class DevProcess { // task.createDevMapping() // task.deleteAddr() // task.deleteAddr2() -// task.synchDevTo43() +// task.synchDevTo100() // task.addNewDev2() // task.addAlias() @@ -69,7 +71,9 @@ class DevProcess { // task.addKfc() // task.devToPark() //开发区下辖园区 - task.checkTown() +// task.checkTown() + +// task.repeatSave() } @@ -692,13 +696,13 @@ class DevProcess { }); } - def synchDevTo43(){ - def dbcursor = development_zone_CONN.find() + def synchDevTo100(){ + def dbcursor = DEVELOPMENT_ZONE_43_COLL.find() def rets = [] for(def r : dbcursor){ rets << r } - DEVELOPMENT_ZONE_43_COLL.insert(rets) + development_zone_CONN.insert(rets) } @@ -961,4 +965,26 @@ class DevProcess { },0) } + + def repeatSave(){ + def dbcursor = DEV_MAPPING_CONN.find(new BasicDBObject()) + def list = [] + for(def map : dbcursor){ + def columns = ["_id","province","city","district","name","type","address","dev_district", "township","park"] + def udata = new BasicDBObject() + map.each{k,v-> + if(columns.contains(k)){ + if(v instanceof String){ + if(k=="_id"){ + udata.put("_id", new ObjectId(v)) + }else{ + udata.put(k, v) + } + } + } + } + list << udata + } + DEV_MAPPING_CONN2.insert(list) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index 8af68c5fa..052f2d046 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -29,7 +29,7 @@ class DevUpdateTestBatch { static main(args) { def task = new DevUpdateTestBatch() - def path = "D:\\数据\\数据建设\\开发区\\开发区下辖园区及街道\\前瞻网开发区下辖园区入库.xlsx"//开发区-园区.xlsx + def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx // task.addDev(path) task.addDevAddress(path) @@ -157,7 +157,7 @@ class DevUpdateTestBatch { println d['A'] +" 不存在" } } - },0) + },1) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy index 5f43810f6..1cb508851 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy @@ -492,9 +492,9 @@ class FjwBigdataForsaleBatch extends BaseBatch{ def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){ - def date = "20200101"//DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd") - def startMdate = DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd") - def endMdate = DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd") + def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd") + def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd") + def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd") QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy index b6fea9896..c2f11e6f8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy @@ -27,7 +27,7 @@ class QiliangTest { // task.run() def data = [ - 'address': '江阴市澄杨路77号', + 'address': '景县开发区东区', 'type': '1' //1:对公 0:对私 ] def s = task.getProxyResult(data); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 024a20faf..a4ff32dad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -15,6 +15,8 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.ProvinceData import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.google.common.hash.BloomFilter +import com.google.common.hash.Funnels import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl @@ -29,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders +import java.nio.charset.StandardCharsets import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -123,51 +126,54 @@ class GaoDePoiDataProcess extends BaseBatch{ } def sysnchVillage(){ - def buducitys = [ '七台河市', '上海市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市', - '东营市', '中卫市', '中山市', '临夏回族自治州', '临汾市', '临沂市', '临沧市', '临高县', '丹东市', - '丽水市', '丽江市', '乌兰察布市', '乌海市', '乌鲁木齐市', '乐东黎族自治县', '乐山市', '九江市', - '云浮市', '五家渠市', '五指山市', '亳州市', '仙桃市', '伊春市', '伊犁哈萨克自治州', '佛山市', - '佳木斯市', '保亭黎族苗族自治县', '保定市', '保山市', '信阳市', '儋州市', '克孜勒苏柯尔克孜自治州', - '克拉玛依市', '六安市', '六盘水市', '兰州市', '兴安盟', '内江市', '包头市', '北京市', '北海市', - '十堰市', '南京市', '南充市', '南宁市', '南平市', '南昌市', '南通市', '南阳市', - '博尔塔拉蒙古自治州', '厦门市', '双鸭山市', '台州市', '合肥市', '吉安市', '吉林市', '吐鲁番地区', - '吕梁市', '吴忠市', '周口市', '呼伦贝尔市', '呼和浩特市', '和田地区', '咸宁市', '咸阳市', - '哈密地区', '哈尔滨市', '唐山市', '商丘市', '商洛市', '喀什地区', '嘉兴市', '嘉峪关市', '四平市', - '固原市', '图木舒克市', '塔城地区', '大兴安岭地区', '大同市', '大庆市', '大理白族自治州', '大连市', - '天水市', '天津市', '天门市', '太原市', '威海市', '娄底市', '孝感市', '宁德市', '宁波市', - '安庆市', '安康市', '安阳市', '安顺市', '定安县', '定西市', '宜宾市', '宜昌市', '宜春市', - '宝鸡市', '宣城市', '宿州市', '宿迁市', '屯昌县', '山南地区', '岳阳市', '崇左市', '巴中市', - '巴彦淖尔市', '巴音郭楞蒙古自治州', '常州市', '常德市', '平凉市', '平顶山市', '广元市', '广安市', - '广州市', '庆阳市', '廊坊市', '延安市', '延边朝鲜族自治州', '开封市', '张家口市', '张家界市', - '张掖市', '徐州市', '德州市', '德阳市', '忻州市', '怀化市', '恩施土家族苗族自治州', '惠州市', - '成都市', '扬州市', '承德市', '抚州市', '抚顺市', '拉萨市', '揭阳市', '攀枝花市', '文昌市', - '新乡市', '新余市', '无锡市', '日喀则地区', '日照市', '昆明市', '昌吉回族自治州', '昌江黎族自治县', - '昌都地区', '昭通市', '晋中市', '晋城市', '普洱市', '景德镇市', '曲靖市', '朔州市', '朝阳市', - '本溪市', '来宾市', '杭州市', '松原市', '林芝地区', '枣庄市', '柳州市', '株洲市', '桂林市', - '梅州市', '梧州市', '楚雄彝族自治州', '榆林市', '武威市', '武汉市', '毕节地区', '永州市', '汉中市', - '汕头市', '汕尾市', '江门市', '池州市', '沈阳市', '沧州市', '河池市', '河源市', '泉州市', - '泰安市', '泰州市', '泸州市', '洛阳市', '济南市', '济宁市', '济源市', '海东地区', '海口市', - '淄博市', '淮北市', '淮南市', '淮安市', '深圳市', '清远市', '温州市', '渭南市', '湖州市', - '湘潭市', '湘西土家族苗族自治州', '湛江市', '滁州市', '滨州市', '漯河市', '漳州市', '潍坊市', - '潜江市', '潮州市', '澄迈县', '濮阳市', '烟台市', '焦作市', '牡丹江市', '玉林市', '玉溪市', - '珠海市', '琼中黎族苗族自治县', '琼海市', '甘南藏族自治州', '白城市', '白山市', '白沙黎族自治县', - '白银市', '百色市', '益阳市', '盐城市', '盘锦市', '眉山市', '石嘴山市', '石家庄市', '石河子市', - '神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绍兴市', '绥化市', '绵阳市', '聊城市', - '肇庆市', '自贡市', '舟山市', '芜湖市', '苏州市', '茂名市', '荆州市', '荆门市', '莆田市', - '菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市', '衢州市', '襄阳市', - '西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市', - '辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市', - '邢台市', '那曲地区', '邯郸市', '邵阳市', '郑州市', '郴州市', '鄂尔多斯市', '鄂州市', '酒泉市', - '重庆市', '金华市', '金昌市', '钦州市', '铁岭市', '铜仁地区', '铜川市', '铜陵市', '银川市', - '锡林郭勒盟', '锦州市', '镇江市', '长春市', '长沙市', '长治市', '阜新市', '阜阳市', '防城港市', - '阳江市', '阳泉市', '阿克苏地区', '阿勒泰地区', '阿拉善盟', '阿拉尔市', '阿里地区', '陇南市', - '陵水黎族自治县', '随州市', '雅安市', '青岛市', '鞍山市', '韶关市', '马鞍山市', '驻马店市', - '鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市', - '黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市'] +// def buducitys = [ '七台河市', '上海市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市', +// '东营市', '中卫市', '中山市', '临夏回族自治州', '临汾市', '临沂市', '临沧市', '临高县', '丹东市', +// '丽水市', '丽江市', '乌兰察布市', '乌海市', '乌鲁木齐市', '乐东黎族自治县', '乐山市', '九江市', +// '云浮市', '五家渠市', '五指山市', '亳州市', '仙桃市', '伊春市', '伊犁哈萨克自治州', '佛山市', +// '佳木斯市', '保亭黎族苗族自治县', '保定市', '保山市', '信阳市', '儋州市', '克孜勒苏柯尔克孜自治州', +// '克拉玛依市', '六安市', '六盘水市', '兰州市', '兴安盟', '内江市', '包头市', '北京市', '北海市', +// '十堰市', '南京市', '南充市', '南宁市', '南平市', '南昌市', '南通市', '南阳市', +// '博尔塔拉蒙古自治州', '厦门市', '双鸭山市', '台州市', '合肥市', '吉安市', '吉林市', '吐鲁番地区', +// '吕梁市', '吴忠市', '周口市', '呼伦贝尔市', '呼和浩特市', '和田地区', '咸宁市', '咸阳市', +// '哈密地区', '哈尔滨市', '唐山市', '商丘市', '商洛市', '喀什地区', '嘉兴市', '嘉峪关市', '四平市', +// '固原市', '图木舒克市', '塔城地区', '大兴安岭地区', '大同市', '大庆市', '大理白族自治州', '大连市', +// '天水市', '天津市', '天门市', '太原市', '威海市', '娄底市', '孝感市', '宁德市', '宁波市', +// '安庆市', '安康市', '安阳市', '安顺市', '定安县', '定西市', '宜宾市', '宜昌市', '宜春市', +// '宝鸡市', '宣城市', '宿州市', '宿迁市', '屯昌县', '山南地区', '岳阳市', '崇左市', '巴中市', +// '巴彦淖尔市', '巴音郭楞蒙古自治州', '常州市', '常德市', '平凉市', '平顶山市', '广元市', '广安市', +// '广州市', '庆阳市', '廊坊市', '延安市', '延边朝鲜族自治州', '开封市', '张家口市', '张家界市', +// '张掖市', '徐州市', '德州市', '德阳市', '忻州市', '怀化市', '恩施土家族苗族自治州', '惠州市', +// '成都市', '扬州市', '承德市', '抚州市', '抚顺市', '拉萨市', '揭阳市', '攀枝花市', '文昌市', +// '新乡市', '新余市', '无锡市', '日喀则地区', '日照市', '昆明市', '昌吉回族自治州', '昌江黎族自治县', +// '昌都地区', '昭通市', '晋中市', '晋城市', '普洱市', '景德镇市', '曲靖市', '朔州市', '朝阳市', +// '本溪市', '来宾市', '杭州市', '松原市', '林芝地区', '枣庄市', '柳州市', '株洲市', '桂林市', +// '梅州市', '梧州市', '楚雄彝族自治州', '榆林市', '武威市', '武汉市', '毕节地区', '永州市', '汉中市', +// '汕头市', '汕尾市', '江门市', '池州市', '沈阳市', '沧州市', '河池市', '河源市', '泉州市', +// '泰安市', '泰州市', '泸州市', '洛阳市', '济南市', '济宁市', '济源市', '海东地区', '海口市', +// '淄博市', '淮北市', '淮南市', '淮安市', '深圳市', '清远市', '温州市', '渭南市', '湖州市', +// '湘潭市', '湘西土家族苗族自治州', '湛江市', '滁州市', '滨州市', '漯河市', '漳州市', '潍坊市', +// '潜江市', '潮州市', '澄迈县', '濮阳市', '烟台市', '焦作市', '牡丹江市', '玉林市', '玉溪市', +// '珠海市', '琼中黎族苗族自治县', '琼海市', '甘南藏族自治州', '白城市', '白山市', '白沙黎族自治县', +// '白银市', '百色市', '益阳市', '盐城市', '盘锦市', '眉山市', '石嘴山市', '石家庄市', '石河子市', +// '神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绍兴市', '绥化市', '绵阳市', '聊城市', +// '肇庆市', '自贡市', '舟山市', '芜湖市', '苏州市', '茂名市', '荆州市', '荆门市', '莆田市', +// '菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市', '衢州市', '襄阳市', +// '西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市', +// '辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市', +// '邢台市', '那曲地区', '邯郸市', '邵阳市', '郑州市', '郴州市', '鄂尔多斯市', '鄂州市', '酒泉市', +// '重庆市', '金华市', '金昌市', '钦州市', '铁岭市', '铜仁地区', '铜川市', '铜陵市', '银川市', +// '锡林郭勒盟', '锦州市', '镇江市', '长春市', '长沙市', '长治市', '阜新市', '阜阳市', '防城港市', +// '阳江市', '阳泉市', '阿克苏地区', '阿勒泰地区', '阿拉善盟', '阿拉尔市', '阿里地区', '陇南市', +// '陵水黎族自治县', '随州市', '雅安市', '青岛市', '鞍山市', '韶关市', '马鞍山市', '驻马店市', +// '鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市', +// '黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市'] + + def buducitys = ["丽水市","杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市"] for(def city : buducitys){ def pois = findAllVillagePoiMap(city) def rows = createVillageData(city, pois) + save(city, rows) // saveToMG(city, rows) println city +" "+pois.size()+" add:"+rows.size() @@ -182,7 +188,7 @@ class GaoDePoiDataProcess extends BaseBatch{ final CountDownLatch latch = new CountDownLatch(pois.size()) // def townMap = findTownRange(city) def townMap = CommonDatautils.queryTownByCity(city) - def poiKeys = findEsPoiByCity(city) + def poiKeysFilter = findEsPoiByCity(city) for(def i=0;i filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001); ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi_single", queryBuilder, { String id, Map d-> - rets << d.city +"##"+d.district+"##"+d.name +// rets << d.city +"##"+d.district+"##"+d.name + filter.put(d.city +"##"+d.district+"##"+d.name) }) - return rets + return filter } def queryMapPoi(city){ From a7bb5f2890643f8c0f750531619cab17308dfbd6 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Mon, 19 Aug 2024 18:48:25 +0800 Subject: [PATCH 05/10] =?UTF-8?q?=E6=B9=96=E5=B7=9E=E6=94=BE=E5=89=8D?= =?UTF-8?q?=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/data_handle/MinglujiData.groovy | 221 ++++++++++++++++++ .../gj/xiamen/GaoDePoiDataProcess.groovy | 10 +- 2 files changed, 227 insertions(+), 4 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy new file mode 100644 index 000000000..e68ef59b1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy @@ -0,0 +1,221 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.regex.Pattern + +/** + * 名录集 + */ +class MinglujiData { + + DBCollection mlj_COLL = MongoUtils.getCollectionByName("crawl", "kfq_mingluji_com", "192.168.0.206",27020) + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + + DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) + DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) + + static main(args) { + def task = new MinglujiData() + task.run() + } + + def run(){ +// exportDatas("湖州") +// matchDev("浙江省","湖州") + matchCommunity("湖州") + } + def exportDatas(city){ + def datas = findMinglujiDatas(city) + ExcelUtils.write("D:\\数据\\数据建设\\名录集\\${city}.xlsx", datas, "数据") + } + def matchDev(prov, city){ + def devs = loadAllDevs(prov) + ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d-> + if(d['E'] && !d['N'] ){ + def obj = findDevByName(d['E']) + if(!obj){ + obj = findDevByAlias(d['E']) + if(!obj){ + obj = findDebBySimlar(devs, d['E'], d['B']) + if(!obj){ + obj = findDevByPark(d['A'], d['B'], d['E']) + if (!obj) { + }else{ + d['N'] = "园区" + } + }else{ + d['N'] = "相似" + } + }else{ + d['N'] = "别名" + } + }else{ + d['N'] = "相同" + } + if(obj){ + d['M'] = obj.name + d['O'] = obj.city + d['P'] = obj.district + } + } + }) + } + + def matchCommunity(city){ + ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d-> + if(d['A'] && d['B'] && d['L'] && !d['Q'] && d['L']=="东村委会"){ + def list = findCommunityByName(d['A'], d['B'], d['L']) + if(!list){ + list = findCommunityBySimlar(d['A'], d['B'], d['L']) + if(!list){ + + }else{ + d['T'] = "包含" + } + }else{ + d['T'] = "相同" + } + if(list){ + if(list.size()==1){ + def obj = list[0] + d['Q'] = obj.name + d['R'] = obj.street_name + }else if(list.size()>=2){ + for(def l : list){ + if(d['C']==l.area_name){ + d['Q'] = l.name + d['R'] = l.street_name + break; + } + } + d['S'] = list.size() + } + } + } + }) + } + + def loadAllDevs(prov){ + def q = new BasicDBObject() + q.put("province", prov) + return development_zone_CONN.find(q).toList() + } + + def findDebBySimlar(def devs, def name, def city){ + name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园") + def last = null + for(def d : devs){ + def alias = null + if(!d.alias){ + alias = [] + }else{ + alias = d.alias + } + alias << d.name + for(def n : alias){ + n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园") + if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){ + last = d + break + } + } + if(last){ + break + } + } + return last + } + def findDevByName(name){ + def q = new BasicDBObject() + q.put("name", name) + return development_zone_CONN.findOne(q) + } + + def findDevByAlias(name){ + def q = new BasicDBObject() + q.put('alias', new BasicDBObject('$in',[name])) + return development_zone_CONN.findOne(q) + } + + def findDevBySimlar(name){ + def q = new BasicDBObject() + q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 + return development_zone_CONN.findOne(q) + } + + def findDevByPark(prov, city, name){ + def q = new BasicDBObject() + q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 + q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 + q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 + q.put("type", "园区") + return dev_mapping_CONN.findOne(q) + } + + def findCommunityByName(province, city, name){ + def q = new BasicDBObject() + q.put("province_name", province+"省") + q.put("city_name", city+"市") + q.put("name", name) + def dbcursor = community_100_CONN.find(q) + def list = [] + for(def d : dbcursor){ + list << d + } + return list + } + + def findCommunityBySimlar(province, city, name){ + def q = new BasicDBObject() + q.put("province_name", province+"省") + q.put("city_name", city+"市") + q.put("name", new BasicDBObject('$regex', Pattern.compile(name+'.*$', Pattern.CASE_INSENSITIVE))) + def dbcursor = community_100_CONN.find(q) + def list = [] + for(def d : dbcursor){ + list << d + } + return list + } + + def findMinglujiDatas(city){ + def keys = new BasicDBObject() + keys.put("c_address",1) + keys.put("c_city",1) + keys.put("c_committee",1) + keys.put("c_community",1) + keys.put("c_company",1) + keys.put("c_district",1) + keys.put("c_street",1) + keys.put("kind",1) + keys.put("name",1) + keys.put("province",1) + def datas = mlj_COLL.find(new BasicDBObject("c_city", city), keys) + def rets = [] + rets << ["省","城市","区域","街道","开发区名","类型","公司名","社区","居委","地址","拆分街道","拆分居委","匹配开发区名"] + for(def d : datas){ + def street = d.c_street + def splitStreet = "" + if(street){ + if(street.contains(d.c_district)){ + def start = street.toString().lastIndexOf(d.c_district)+d.c_district.size() + def end = street.toString().size() + splitStreet = street.toString().substring(start, end) + } + } + def splitCommunity = "" + if(d.c_committee){ + if(d.c_committee.contains(d.c_district)){ + splitCommunity = d.c_committee.substring(d.c_committee.lastIndexOf(d.c_district)+d.c_district.size(), d.c_committee.toString().size()) + } + } + rets << [d.province, d.c_city, d.c_district, d.c_street, d.name, d.kind, d.c_company, d.c_community, d.c_committee, d.c_address, splitStreet, splitCommunity] + } + return rets + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index a4ff32dad..06ddf6d39 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -168,7 +168,8 @@ class GaoDePoiDataProcess extends BaseBatch{ // '鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市', // '黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市'] - def buducitys = ["丽水市","杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市"] + //"丽水市","杭州市", + def buducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市"] for(def city : buducitys){ def pois = findAllVillagePoiMap(city) def rows = createVillageData(city, pois) @@ -189,7 +190,6 @@ class GaoDePoiDataProcess extends BaseBatch{ // def townMap = findTownRange(city) def townMap = CommonDatautils.queryTownByCity(city) def poiKeysFilter = findEsPoiByCity(city) - for(def i=0;i=1){ retMap.type = nameArr[0] @@ -416,6 +415,9 @@ class GaoDePoiDataProcess extends BaseBatch{ } def findAllVillagePoiMap(city){ + def q = new BasicDBObject("city", city); + q.put("std_tag", new BasicDBObject('$exists', true)) + def keys = new BasicDBObject() keys.put("city", 1) keys.put("district", 1) @@ -424,7 +426,7 @@ class GaoDePoiDataProcess extends BaseBatch{ keys.put("gcj02_x", 1) keys.put("gcj02_y", 1) keys.put("std_tag", 1) - def list = BAIDU_POI_COLL.find(new BasicDBObject("city", city), keys).toList() + def list = BAIDU_POI_COLL.find(q, keys).toList() return list; } From 657d058394f97aa54bbdae827d08724dd0cbe04f Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 20 Aug 2024 16:47:41 +0800 Subject: [PATCH 06/10] =?UTF-8?q?=E5=8A=A0=E6=88=BF=E4=BA=A7=E8=B6=85?= =?UTF-8?q?=E5=B8=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/AddThirdPriceSample.groovy | 40 +++++++++ .../fangjia/script/ccbevaluation/Test.groovy | 84 +++++++++++++------ .../ccbevaluation/ThirdPartyPriceBatch.groovy | 9 ++ .../script/gj/data_handle/MinglujiData.groovy | 35 +++++--- 4 files changed, 129 insertions(+), 39 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy new file mode 100644 index 000000000..d1ec16934 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy @@ -0,0 +1,40 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 将国信达及成交对标数据加入到第三方均价表,做为均价参考 + */ +class AddThirdPriceSample { + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new AddThirdPriceSample() + task.run() + } + def run(){ + for(def i=1;i<19;i++){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d-> + if(d['A'] && d['C']){ + def dist = CommonDatautils.findDistrictById(d['A'].toString().trim()) + if(dist){ + def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407) + def udata = new BasicDBObject("zhongyuan_esf_price", d['C'].toString().toDouble().toInteger()) + + AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false) + }else{ + println d['A'] +" 没查到" + } + } + }, i) + + println(i) + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 13a46482e..0d36523e6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -612,9 +612,9 @@ def findAllData(sqlInstance236, citycode){ ////},0) // -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d-> +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\删除小区.xlsx",{d-> // if(d['A']){ -// def dbo = CommonDatautils.findDistrictById(d['D']) +// def dbo = CommonDatautils.findDistrictById(d['A']) // if(dbo){ // def udata = new BasicDBObject() //// udata.put("name_cat", 11) @@ -630,6 +630,38 @@ def findAllData(sqlInstance236, citycode){ // } //},0) + +ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d-> + if(d['A'] && d['B']){ + def dbo = CommonDatautils.findDistrictById(d['A']) + if(dbo){ + def alias = null + if(!dbo.alias){ + alias = [] + }else{ + alias = dbo.alias + } + if(!alias.contains(d['B'])){ + alias << d['B'] + } + if(alias){ + def udata = new BasicDBObject() + udata.put("alias", alias) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", dbo._id) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + }else{ + println d['A']+" "+"别名已存在" + } + + }else{ + println dbo._id.toString() +" 没找到" + } + } +}) + //def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] //for(def c : citys){ // def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"]) @@ -1299,34 +1331,34 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx" - -ExcelUtils.multiThreadingModify(fileName, {d-> - if(d['J']){ -// def region = RegionData.formatRegion(d['A'], d['J']) -// if(region){ -// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") +//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx" +// +//ExcelUtils.multiThreadingModify(fileName, {d-> +// if(d['J']){ +//// def region = RegionData.formatRegion(d['A'], d['J']) +//// if(region){ +//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") +//// if(dist){ +//// d['B'] = dist.region +//// d['C'] = dist.name +//// d['L'] = dist.status-dist.name_cat +//// } +//// } +//// if(d['J'] instanceof String){ +// def dist = CommonDatautils.findDistrictById(d['J']) // if(dist){ +// d['A'] = dist.city // d['B'] = dist.region // d['C'] = dist.name -// d['L'] = dist.status-dist.name_cat +// d['K'] = dist.estate_type2 +// d['L'] = dist.status +// d['M'] = dist.name_cat +// +//// d['H'] = dist.name // } -// } -// if(d['J'] instanceof String){ - def dist = CommonDatautils.findDistrictById(d['J']) - if(dist){ - d['A'] = dist.city - d['B'] = dist.region - d['C'] = dist.name - d['K'] = dist.estate_type2 - d['L'] = dist.status - d['M'] = dist.name_cat - -// d['H'] = dist.name - } -// } - } -},0) +//// } +// } +//},0) //def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 52fd3ddb2..5f5a53f33 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -257,6 +257,9 @@ class ThirdPartyPriceBatch extends BaseBatch { if(third.get("5i5j_esf_price")){ price_map.wiwj_avg_price = third.get("5i5j_esf_price") } + if(third.get("fccs_esf_price")){ + price_map.fccs_avg_price = third.get("fccs_esf_price") + } dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") allRows << dataMap } @@ -303,6 +306,8 @@ class ThirdPartyPriceBatch extends BaseBatch { web = "房多多" }else if(k=="wiwj_avg_price"){ web = "我爱我家" + }else if(k=="fccs_avg_price"){ + web = "房产超市" } ret << sourceMap.get(web) @@ -584,6 +589,10 @@ class ThirdPartyPriceBatch extends BaseBatch { if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){ dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price")) } + + if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){ + dataMap.put("fccs_esf_price", db.get("fccs_esf_price")) + } // if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){ // map.put(key, db) // } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy index e68ef59b1..0ae018d53 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy @@ -35,7 +35,7 @@ class MinglujiData { } def matchDev(prov, city){ def devs = loadAllDevs(prov) - ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d-> if(d['E'] && !d['N'] ){ def obj = findDevByName(d['E']) if(!obj){ @@ -67,8 +67,8 @@ class MinglujiData { } def matchCommunity(city){ - ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d-> - if(d['A'] && d['B'] && d['L'] && !d['Q'] && d['L']=="东村委会"){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\${city}-5.xlsx",{d-> + if(d['A'] && d['B'] && d['L'] && !d['Q']){ def list = findCommunityByName(d['A'], d['B'], d['L']) if(!list){ list = findCommunityBySimlar(d['A'], d['B'], d['L']) @@ -97,7 +97,7 @@ class MinglujiData { } } } - }) + }, 0) } def loadAllDevs(prov){ @@ -110,19 +110,28 @@ class MinglujiData { name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园") def last = null for(def d : devs){ - def alias = null + def alias1 = null if(!d.alias){ - alias = [] + alias1 = [] }else{ - alias = d.alias + alias1 = d.alias } - alias << d.name - for(def n : alias){ - n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园") - if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){ - last = d - break + if(!alias1.contains(d.name)){ + alias1 << d.name + } + if(alias1){ + try { + for(def n : alias1){ + n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园") + if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){ + last = d + break + } + } + }catch(exception){ + println alias1 } + } if(last){ break From 30444f6d8458e8f60510a96ea0cdf3b43c9435ce Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 20 Aug 2024 18:03:30 +0800 Subject: [PATCH 07/10] =?UTF-8?q?=E6=88=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/BigdataDealBatch.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index db19c3e41..8a924e652 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -70,12 +70,12 @@ class BigdataDealBatch extends BaseBatch{ //664b73800000000000000000 5-21 def citys = CcbCommonUtils.getRunningCitys(args) -// println "curDate:"+curDate -// def allCitys = CommonDatautils.findAllCity() -// for(def city : allCitys){ -// addDealPriceMatch(city); -// } -// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月 + println "curDate:"+curDate + def allCitys = CommonDatautils.findAllCity() + for(def city : allCitys){ + addDealPriceMatch(city); + } + updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月 def rows = [] From 2b34b9d610eab1d60700f3ef02929e9af032eb60 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 20 Aug 2024 18:16:23 +0800 Subject: [PATCH 08/10] update --- .../com/fangjia/script/gj/data_handle/DevProcess.groovy | 2 +- .../com/fangjia/script/gj/data_handle/MinglujiData.groovy | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 9b4d34ff1..411c7e92e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -39,7 +39,7 @@ class DevProcess { def task = new DevProcess(); // task.run() // task.parseCityAndDist() -// task.devSplitRoad() + task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() // task.matchDev() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy index 0ae018d53..327e48a2e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy @@ -67,7 +67,7 @@ class MinglujiData { } def matchCommunity(city){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\${city}-5.xlsx",{d-> + ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}-没配居委1.xlsx",{d-> if(d['A'] && d['B'] && d['L'] && !d['Q']){ def list = findCommunityByName(d['A'], d['B'], d['L']) if(!list){ @@ -183,7 +183,7 @@ class MinglujiData { def q = new BasicDBObject() q.put("province_name", province+"省") q.put("city_name", city+"市") - q.put("name", new BasicDBObject('$regex', Pattern.compile(name+'.*$', Pattern.CASE_INSENSITIVE))) + q.put("name", new BasicDBObject('$regex', Pattern.compile('^'+name+'.*$', Pattern.CASE_INSENSITIVE))) def dbcursor = community_100_CONN.find(q) def list = [] for(def d : dbcursor){ From f1325affa7e139c1ffc14640fca4867d60214bbe Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 21 Aug 2024 09:55:30 +0800 Subject: [PATCH 09/10] =?UTF-8?q?=E5=85=A8=E5=9B=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/xiamen/GaoDePoiDataProcess.groovy | 87 ++++++++++--------- 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 06ddf6d39..b8d657c89 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -126,51 +126,54 @@ class GaoDePoiDataProcess extends BaseBatch{ } def sysnchVillage(){ -// def buducitys = [ '七台河市', '上海市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市', -// '东营市', '中卫市', '中山市', '临夏回族自治州', '临汾市', '临沂市', '临沧市', '临高县', '丹东市', -// '丽水市', '丽江市', '乌兰察布市', '乌海市', '乌鲁木齐市', '乐东黎族自治县', '乐山市', '九江市', -// '云浮市', '五家渠市', '五指山市', '亳州市', '仙桃市', '伊春市', '伊犁哈萨克自治州', '佛山市', -// '佳木斯市', '保亭黎族苗族自治县', '保定市', '保山市', '信阳市', '儋州市', '克孜勒苏柯尔克孜自治州', -// '克拉玛依市', '六安市', '六盘水市', '兰州市', '兴安盟', '内江市', '包头市', '北京市', '北海市', -// '十堰市', '南京市', '南充市', '南宁市', '南平市', '南昌市', '南通市', '南阳市', -// '博尔塔拉蒙古自治州', '厦门市', '双鸭山市', '台州市', '合肥市', '吉安市', '吉林市', '吐鲁番地区', -// '吕梁市', '吴忠市', '周口市', '呼伦贝尔市', '呼和浩特市', '和田地区', '咸宁市', '咸阳市', -// '哈密地区', '哈尔滨市', '唐山市', '商丘市', '商洛市', '喀什地区', '嘉兴市', '嘉峪关市', '四平市', -// '固原市', '图木舒克市', '塔城地区', '大兴安岭地区', '大同市', '大庆市', '大理白族自治州', '大连市', -// '天水市', '天津市', '天门市', '太原市', '威海市', '娄底市', '孝感市', '宁德市', '宁波市', -// '安庆市', '安康市', '安阳市', '安顺市', '定安县', '定西市', '宜宾市', '宜昌市', '宜春市', -// '宝鸡市', '宣城市', '宿州市', '宿迁市', '屯昌县', '山南地区', '岳阳市', '崇左市', '巴中市', -// '巴彦淖尔市', '巴音郭楞蒙古自治州', '常州市', '常德市', '平凉市', '平顶山市', '广元市', '广安市', -// '广州市', '庆阳市', '廊坊市', '延安市', '延边朝鲜族自治州', '开封市', '张家口市', '张家界市', -// '张掖市', '徐州市', '德州市', '德阳市', '忻州市', '怀化市', '恩施土家族苗族自治州', '惠州市', -// '成都市', '扬州市', '承德市', '抚州市', '抚顺市', '拉萨市', '揭阳市', '攀枝花市', '文昌市', -// '新乡市', '新余市', '无锡市', '日喀则地区', '日照市', '昆明市', '昌吉回族自治州', '昌江黎族自治县', -// '昌都地区', '昭通市', '晋中市', '晋城市', '普洱市', '景德镇市', '曲靖市', '朔州市', '朝阳市', -// '本溪市', '来宾市', '杭州市', '松原市', '林芝地区', '枣庄市', '柳州市', '株洲市', '桂林市', -// '梅州市', '梧州市', '楚雄彝族自治州', '榆林市', '武威市', '武汉市', '毕节地区', '永州市', '汉中市', -// '汕头市', '汕尾市', '江门市', '池州市', '沈阳市', '沧州市', '河池市', '河源市', '泉州市', -// '泰安市', '泰州市', '泸州市', '洛阳市', '济南市', '济宁市', '济源市', '海东地区', '海口市', -// '淄博市', '淮北市', '淮南市', '淮安市', '深圳市', '清远市', '温州市', '渭南市', '湖州市', -// '湘潭市', '湘西土家族苗族自治州', '湛江市', '滁州市', '滨州市', '漯河市', '漳州市', '潍坊市', -// '潜江市', '潮州市', '澄迈县', '濮阳市', '烟台市', '焦作市', '牡丹江市', '玉林市', '玉溪市', -// '珠海市', '琼中黎族苗族自治县', '琼海市', '甘南藏族自治州', '白城市', '白山市', '白沙黎族自治县', -// '白银市', '百色市', '益阳市', '盐城市', '盘锦市', '眉山市', '石嘴山市', '石家庄市', '石河子市', -// '神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绍兴市', '绥化市', '绵阳市', '聊城市', -// '肇庆市', '自贡市', '舟山市', '芜湖市', '苏州市', '茂名市', '荆州市', '荆门市', '莆田市', -// '菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市', '衢州市', '襄阳市', -// '西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市', -// '辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市', -// '邢台市', '那曲地区', '邯郸市', '邵阳市', '郑州市', '郴州市', '鄂尔多斯市', '鄂州市', '酒泉市', -// '重庆市', '金华市', '金昌市', '钦州市', '铁岭市', '铜仁地区', '铜川市', '铜陵市', '银川市', -// '锡林郭勒盟', '锦州市', '镇江市', '长春市', '长沙市', '长治市', '阜新市', '阜阳市', '防城港市', -// '阳江市', '阳泉市', '阿克苏地区', '阿勒泰地区', '阿拉善盟', '阿拉尔市', '阿里地区', '陇南市', -// '陵水黎族自治县', '随州市', '雅安市', '青岛市', '鞍山市', '韶关市', '马鞍山市', '驻马店市', -// '鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市', -// '黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市'] + def buducitys = [ '七台河市', '上海市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市', + '东营市', '中卫市', '中山市', '临夏回族自治州', '临汾市', '临沂市', '临沧市', '临高县', '丹东市', + '丽水市', '丽江市', '乌兰察布市', '乌海市', '乌鲁木齐市', '乐东黎族自治县', '乐山市', '九江市', + '云浮市', '五家渠市', '五指山市', '亳州市', '仙桃市', '伊春市', '伊犁哈萨克自治州', '佛山市', + '佳木斯市', '保亭黎族苗族自治县', '保定市', '保山市', '信阳市', '儋州市', '克孜勒苏柯尔克孜自治州', + '克拉玛依市', '六安市', '六盘水市', '兰州市', '兴安盟', '内江市', '包头市', '北京市', '北海市', + '十堰市', '南京市', '南充市', '南宁市', '南平市', '南昌市', '南通市', '南阳市', + '博尔塔拉蒙古自治州', '厦门市', '双鸭山市', '台州市', '合肥市', '吉安市', '吉林市', '吐鲁番地区', + '吕梁市', '吴忠市', '周口市', '呼伦贝尔市', '呼和浩特市', '和田地区', '咸宁市', '咸阳市', + '哈密地区', '哈尔滨市', '唐山市', '商丘市', '商洛市', '喀什地区', '嘉兴市', '嘉峪关市', '四平市', + '固原市', '图木舒克市', '塔城地区', '大兴安岭地区', '大同市', '大庆市', '大理白族自治州', '大连市', + '天水市', '天津市', '天门市', '太原市', '威海市', '娄底市', '孝感市', '宁德市', '宁波市', + '安庆市', '安康市', '安阳市', '安顺市', '定安县', '定西市', '宜宾市', '宜昌市', '宜春市', + '宝鸡市', '宣城市', '宿州市', '宿迁市', '屯昌县', '山南地区', '岳阳市', '崇左市', '巴中市', + '巴彦淖尔市', '巴音郭楞蒙古自治州', '常州市', '常德市', '平凉市', '平顶山市', '广元市', '广安市', + '广州市', '庆阳市', '廊坊市', '延安市', '延边朝鲜族自治州', '开封市', '张家口市', '张家界市', + '张掖市', '徐州市', '德州市', '德阳市', '忻州市', '怀化市', '恩施土家族苗族自治州', '惠州市', + '成都市', '扬州市', '承德市', '抚州市', '抚顺市', '拉萨市', '揭阳市', '攀枝花市', '文昌市', + '新乡市', '新余市', '无锡市', '日喀则地区', '日照市', '昆明市', '昌吉回族自治州', '昌江黎族自治县', + '昌都地区', '昭通市', '晋中市', '晋城市', '普洱市', '景德镇市', '曲靖市', '朔州市', '朝阳市', + '本溪市', '来宾市', '杭州市', '松原市', '林芝地区', '枣庄市', '柳州市', '株洲市', '桂林市', + '梅州市', '梧州市', '楚雄彝族自治州', '榆林市', '武威市', '武汉市', '毕节地区', '永州市', '汉中市', + '汕头市', '汕尾市', '江门市', '池州市', '沈阳市', '沧州市', '河池市', '河源市', '泉州市', + '泰安市', '泰州市', '泸州市', '洛阳市', '济南市', '济宁市', '济源市', '海东地区', '海口市', + '淄博市', '淮北市', '淮南市', '淮安市', '深圳市', '清远市', '温州市', '渭南市', '湖州市', + '湘潭市', '湘西土家族苗族自治州', '湛江市', '滁州市', '滨州市', '漯河市', '漳州市', '潍坊市', + '潜江市', '潮州市', '澄迈县', '濮阳市', '烟台市', '焦作市', '牡丹江市', '玉林市', '玉溪市', + '珠海市', '琼中黎族苗族自治县', '琼海市', '甘南藏族自治州', '白城市', '白山市', '白沙黎族自治县', + '白银市', '百色市', '益阳市', '盐城市', '盘锦市', '眉山市', '石嘴山市', '石家庄市', '石河子市', + '神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绍兴市', '绥化市', '绵阳市', '聊城市', + '肇庆市', '自贡市', '舟山市', '芜湖市', '苏州市', '茂名市', '荆州市', '荆门市', '莆田市', + '菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市', '衢州市', '襄阳市', + '西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市', + '辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市', + '邢台市', '那曲地区', '邯郸市', '邵阳市', '郑州市', '郴州市', '鄂尔多斯市', '鄂州市', '酒泉市', + '重庆市', '金华市', '金昌市', '钦州市', '铁岭市', '铜仁地区', '铜川市', '铜陵市', '银川市', + '锡林郭勒盟', '锦州市', '镇江市', '长春市', '长沙市', '长治市', '阜新市', '阜阳市', '防城港市', + '阳江市', '阳泉市', '阿克苏地区', '阿勒泰地区', '阿拉善盟', '阿拉尔市', '阿里地区', '陇南市', + '陵水黎族自治县', '随州市', '雅安市', '青岛市', '鞍山市', '韶关市', '马鞍山市', '驻马店市', + '鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市', + '黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市'] //"丽水市","杭州市", - def buducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市"] + def runbuducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市"] for(def city : buducitys){ + if(runbuducitys.contains(city)){ + continue + } def pois = findAllVillagePoiMap(city) def rows = createVillageData(city, pois) From bdbac0e02bd90dbe7111f445f566cbf877baf7e6 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 21 Aug 2024 10:05:31 +0800 Subject: [PATCH 10/10] =?UTF-8?q?=E5=86=85=E9=83=A8=E5=9D=87=E4=BB=B7?= =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy index 52d65e101..baae234f8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy @@ -265,6 +265,10 @@ class FjwThirdPartyPriceBatch extends BaseBatch { if(third.get("5i5j_esf_price")){ price_map.wiwj_avg_price = third.get("5i5j_esf_price") } + + if(third.get("fccs_esf_price")){ + price_map.fccs_avg_price = third.get("fccs_esf_price") + } dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") allRows << dataMap } @@ -311,6 +315,8 @@ class FjwThirdPartyPriceBatch extends BaseBatch { web = "房多多" }else if(k=="wiwj_avg_price"){ web = "我爱我家" + }else if(k=="fccs_avg_price"){ + web = "房产超市" } ret << sourceMap.get(web) @@ -459,6 +465,11 @@ class FjwThirdPartyPriceBatch extends BaseBatch { if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){ dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price")) } + + if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){ + dataMap.put("fccs_esf_price", db.get("fccs_esf_price")) + } + // if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){ // map.put(key, db) // }