diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index d36b853dc..b9b6bd4bf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1212,6 +1212,7 @@ class CommonDatautils { public static def findAllCount(String cat){ DBObject q = new BasicDBObject(); q.put("cat", cat); + q.put("status", 0) return SEAWEED_MONGO_COLL.count(q) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BaiduBaikeTown.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BaiduBaikeTown.groovy new file mode 100644 index 000000000..77356d988 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BaiduBaikeTown.groovy @@ -0,0 +1,102 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.google.gson.JsonObject +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.jsoup.Jsoup + +class BaiduBaikeTown { + + DBCollection dzb_COLL = MongoUtils.getCollectionByName("crawl", "development_zone_bdbk", "192.168.0.206",27020) + + + static main(args) { + def task = new BaiduBaikeTown() + task.run() + } + + def run(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",{row-> + if(!row['E'] && row['A']=="郑州经济技术开发区"){ + def q = new BasicDBObject() + q.put("desc",new BasicDBObject('$ne', 'null')) + q.put("name",row['A']) + def d = dzb_COLL.findOne(q) + def townMap = parse(d.desc, d.name) + row['D'] = townMap.town_str1 + row['F'] = townMap.town_str2 + +// def ret = [] +// ret << ["名称", "基本信息", "目录", "全文"] +// for(def d : dbcursor){ +// def townMap = parse(d.desc, d.name) +// if(townMap.town){ +// println d.name+" "+townMap.town +// } +// ret << [d.name, townMap.town, townMap.towns, townMap.town_str1, townMap.town_str2] +// } + } + + }) + +// ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx", ret, "数据") + } + + def parse(desc, name){ + def map = [:] + if(desc){ + def doc = Jsoup.parse(desc) + def town = JQueryUtils.selectText(".basicInfoBlock_Ex6IU div:contains(下辖地区) dd", doc) + + def s = doc.toString() + def dataStr = DataHandleUtils.findMatchPattern(s,"PAGE_DATA=.*").replace("PAGE_DATA= ","").replace("","") + def jsonObj = JSONObject.fromObject(dataStr) + + def str = DataHandleUtils.findMatchPattern(doc.text(),"下辖.*[街道|镇]、?和?") + def towns = [] + if(jsonObj && jsonObj.catalog){ + def catalogs = jsonObj.catalog + def index = null + for(def log : catalogs){ + def curIndex = log.index + if(log.title=="行政区划"){ + index = log.index + } + if(index && curIndex.toString().startsWith(index+"-")){ + towns << log.title + } + } + } + + + + map.town_str1 = str + if(town){ + map.town = town + }else{ + map.town = "" + } + + if(towns){ + map.towns = towns.join(",") + }else{ + map.towns = "" + } + + + def str2 = DataHandleUtils.findMatchPattern(doc.text(),"开发区辖.*[,|。]") + map.town_str2 = str2 + return map + } + map.town_str1 = "" + map.town_str2 = "" + map.town = "" + map.towns = "" + return map + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 772bbaf62..69a4318b0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -37,7 +37,7 @@ class DevProcess { def task = new DevProcess(); // task.run() // task.parseCityAndDist() - task.devSplitRoad() +// task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() // task.matchDev() @@ -69,7 +69,7 @@ class DevProcess { // task.addKfc() // task.devToPark() //开发区下辖园区 -// task.checkTown() + task.checkTown() } @@ -898,66 +898,67 @@ class DevProcess { } def devToPark(){ - def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\开发区下辖街道-处理后表格.xlsx",0, 1) + def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",0, 1) def rets = [] rets << ["开发区名称", "下辖街道", "城市", "区域"] for(def d : datas){ if(d['E']){ - def arrs = d['E'].split(",") - def q = new BasicDBObject().append("name", d['B']) + def arrs = d['E'].split("、") + def q = new BasicDBObject().append("name", d['A']) def obj = development_zone_CONN.findOne(q) if(obj){ for(def park : arrs){ - rets << [d['B'], park, obj.city, obj.district] + rets << [d['A'], park, obj.city, obj.district] } } } } - ExcelUtils.write("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx", rets, "下辖街道") + ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx", rets, "下辖街道") } def checkTown(){ - ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx",{d-> if(!d['E']){ if(d['A']){ -// if(d['A']=="长春高新技术产业开发区"){ - def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['A']) - if(jsonObject1) { - if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') { - def township = jsonObject1.get("township") - def district = jsonObject1.get("district") - d['E'] = "接口" - if(district && district!='null'){ - d['F'] = district - } - if(township && township!='null'){ - d['G'] = township - } - - d['H'] = jsonObject1.get("townCode") - if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){ - d['I'] = jsonObject1.get("provinceCode") - } - } -// } - } - -// def q = new BasicDBObject() -// q.put("city", d['C']) -//// q.put("district", d['D']) -// q.put("mzb_district_name", d['D']) -// q.put("type", "乡镇街道") -// q.put("street", d['B']) -// def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q) -// if(obj){ -// d['E'] = "可入库2" -// d['F'] = obj.district +// def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B']) +// if(jsonObject1) { +// if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') { +// def township = jsonObject1.get("township") +// def district = jsonObject1.get("district") +// d['E'] = "接口" +// if(district && district!='null'){ +// d['F'] = district +// } +// if(township && township!='null'){ +// d['G'] = township +// } +// +// d['H'] = jsonObject1.get("townCode") +// if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){ +// d['I'] = jsonObject1.get("provinceCode") +// } +// } // } + + def q = new BasicDBObject() + q.put("city", d['C']) +// q.put("district", d['D']) +// q.put("mzb_district_name", d['D']) + q.put("type", "乡镇街道") + q.put('street',new BasicDBObject('$regex', Pattern.compile(d['B'].toString().trim()+'.*$', Pattern.CASE_INSENSITIVE))) +// q.put("street", d['B']) +// q.put('alias', new BasicDBObject('$in',[d['B']])) + def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q) + if(obj){ + d['E'] = "可入库-忽略区域模糊2" + d['F'] = obj.district + d['G'] = obj.street + } } } - },1) + },0) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index 405fdbde8..8af68c5fa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -29,7 +29,7 @@ class DevUpdateTestBatch { static main(args) { def task = new DevUpdateTestBatch() - def path = "D:\\数据\\数据建设\\开发区\\文心一言获取开发区下辖园区及街道.xlsx"//开发区-园区.xlsx + def path = "D:\\数据\\数据建设\\开发区\\开发区下辖园区及街道\\前瞻网开发区下辖园区入库.xlsx"//开发区-园区.xlsx // task.addDev(path) task.addDevAddress(path) @@ -137,26 +137,27 @@ class DevUpdateTestBatch { DEV_MAPPING_100_CONN.save(o) - o.id = o._id.toString() - o.remove("_id") - o.remove("c_date") - addEs(o, "192.168.0.89", "dev_mapping") - try{ - addEs(o, "192.168.0.52", "dev_mapping") - }catch(Exception e){ - println "52 es error: "+e.printStackTrace() - } +// o.id = o._id.toString() +// o.remove("_id") +// o.remove("c_date") +// addEs(o, "192.168.0.89", "dev_mapping") +// try{ +// addEs(o, "192.168.0.52", "dev_mapping") +// }catch(Exception e){ +// println "52 es error: "+e.printStackTrace() +// } d['D'] = "已完成" }else{ - d['D'] = "存在" + d['D'] = "重复" println d['A'] +" "+ d['C']+" 重复" } }else{ + d['D'] = "不存在" println d['A'] +" 不存在" } } - },2) + },0) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy index f814cceb2..61ed0ea7b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy @@ -40,11 +40,11 @@ class GXDdistrictMatch { def addOldMatch(){ //老匹配 - def fileName = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"//11城重新匹配.xlsx + def fileName = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"//11城重新匹配.xlsx ExcelUtils.multiThreadingModify(fileName, { d -> println d['C'] if(!d['AJ'] && d['I']){ - def jsonObject = OpenApiClient.addressMatch(d['C'], d['I'], "district", false,null, d['D']) + def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D']) if (jsonObject != null && !jsonObject.containsKey("result")) { d['AJ'] = jsonObject.get("id") d['AK'] = jsonObject.get("city") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index 2dd8a91a7..14b6a5c26 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,11 +33,11 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx" + def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx" // addGXDPrice(filePath) -// addCcbPrice(filePath) + addCcbPrice(filePath) // addThirdPrice(filePath) - addFjwPrice(filePath) +// addFjwPrice(filePath) // exportPkPrice() // addRegion() // addTradePrice(filePath) @@ -169,9 +169,9 @@ class AddPriceTest { def addCcbPrice(filepath){ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() ExcelUtils.multiThreadingModify(filepath,{ d-> - if(d['AJ']){ - def cityId = cityCodeDataMap.get(d['AK']) - def map = findAvgPrice(d['AJ'], cityId) + if(d['U']){ + def cityId = cityCodeDataMap.get(d['V']) + def map = findAvgPrice(d['U'], cityId) if(map){ d['F'] = map.avg_price // d['P'] = map.calc_type @@ -183,7 +183,7 @@ class AddPriceTest { def findAvgPrice(def commid, def cityid){ - def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202406 order by gujiabds_clc_yrmo_val desc limit 1" + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1" def map = [:] pgsqlInstance.eachRow(sql, { r -> map.avg_price = r.gujia_bds_clc_avprc