diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 7978ddbeb..019c22494 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1289,52 +1289,54 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达价格不作为指导价录入5-20.xlsx" -// -//ExcelUtils.multiThreadingModify(fileName, {d-> -// if(d['J']){ -//// def region = RegionData.formatRegion(d['B'], d['C']) -//// if(region){ -//// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") -//// if(dist){ -//// d['G'] = dist.region -//// d['H'] = dist._id.toString() -//// } -//// } -// if(!d['A']){ -// def dist = CommonDatautils.findDistrictById(d['J']) +def fileName = "D:\\数据\\建行\\房产估值\\4月分析级联badcase汇总.xlsx" + +ExcelUtils.multiThreadingModify(fileName, {d-> + if(d['G']){ +// def region = RegionData.formatRegion(d['B'], d['C']) +// if(region){ +// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") // if(dist){ +// d['G'] = dist.region +// d['H'] = dist._id.toString() +// } +// } + if(!d['H'] && d['G'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['G']) + if(dist){ // d['A'] = dist.city // d['B'] = dist.region // d['C'] = dist.name // d['K'] = dist.estate_type2 // d['L'] = dist.status // d['M'] = dist.name_cat -// } -// } -// } -//},0) - -def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx" -ExcelUtils.multiThreadingModify(filePath, { d -> - if(d['B']){ - - def obj = CommonDatautils.findDistrictById(d['B']) - if(obj){ - def markMap = obj.mark_map - if(!obj.mark_map){ - markMap = [:] + d['H'] = dist.name } - def data = new BasicDBObject() - markMap.put("exclusive", "是") - data.put("mark_map", markMap) - data.put("m_date", new Date()) - DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data)) } } +},0) -}) + +//def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx" +//ExcelUtils.multiThreadingModify(filePath, { d -> +// if(d['B']){ +// +// def obj = CommonDatautils.findDistrictById(d['B']) +// if(obj){ +// def markMap = obj.mark_map +// if(!obj.mark_map){ +// markMap = [:] +// } +// def data = new BasicDBObject() +// markMap.put("exclusive", "是") +// data.put("mark_map", markMap) +// data.put("m_date", new Date()) +// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data)) +// } +// } +// +//}) //def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 89a12c9d2..7c1a06a58 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -29,10 +29,10 @@ class DevProcess { def task = new DevProcess(); // task.run() // task.parseCityAndDist() -// task.devSplitRoad() + task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() - task.matchDev() +// task.matchDev() // task.calAddrDistince() // task.updateCityAndDist() @@ -46,6 +46,9 @@ class DevProcess { // task.deleteAddr() // task.deleteAddr2() // task.synchDevTo43() +// task.addNewDev2() + +// task.addAlias() } @@ -165,10 +168,10 @@ class DevProcess { def devSplitRoad(){ def i = 0; - while(i<1){ - def fileName = "/home/goojia/gj/company-${i}.xlsx" + while(i<14){ + def fileName = "/home/goojia/gj/zhejiang-${i}.xlsx" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\数据建设\\开发区\\56万企业地址\\省没出值地方.xlsx" + fileName = "D:\\数据\\数据建设\\开发区\\浙江\\zhejiang_0.xlsx" } ExcelUtils.multiThreadingModify(fileName,{d-> /*if(d['C'] && !d['V']){ @@ -194,7 +197,7 @@ class DevProcess { } } }*/ - if(d['C'] && d['B']!='河北省' && !d['J']){ + if(d['C'] && !d['J']){ def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C']) if(jsonObject){ def street = jsonObject.get("street") @@ -292,10 +295,10 @@ class DevProcess { } def matchDev(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\统计局区域的开发区匹配省级以上.xlsx",{d-> if(!d['F']){ def q = new BasicDBObject() - q.put("name", d['D'].replace("河北", "").replace("浙江","")) + q.put("name", d['C']) def obj = development_zone_CONN.findOne(q) if(obj){ d['F'] = obj._id.toString() @@ -507,6 +510,27 @@ class DevProcess { }) } + def addNewDev2(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d-> + if(d['F'] && d['F']=="新增"){ + def udata = new BasicDBObject() + udata.create_time = new Date() + udata.province = d['G'] + udata.city = d['I'] + udata.district = d['J'] + udata.level = "省级" + udata.name = d['D'] + + def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name)) + if(!obj){ + development_zone_CONN.save(udata) + }else{ + println udata.name +" 已存在" + } + } + }) + } + def createDevMapping(){ def dbcursor = development_zone_CONN.find(new BasicDBObject()) for(def d : dbcursor){ @@ -550,15 +574,15 @@ class DevProcess { } def deleteAddr2(){ - ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北开发区删除地址.xlsx",{d-> - if(d['E']){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\56万中待删除地址.xlsx",{d-> + if(d['Q'] && d['M']){ def q = new BasicDBObject() - q.put("_id", new ObjectId(d['E'])) + q.put("_id", new ObjectId(d['Q'])) def obj = development_zone_CONN.findOne(q) if(obj){ def q2 = new BasicDBObject() q2.put("name", obj.name) - q2.put("address", d['A']+d['B']) + q2.put("address", d['I']+d['J']) def sameObj = DEV_MAPPING_CONN.findOne(q2) if(sameObj){ DEV_MAPPING_CONN.remove(new BasicDBObject("_id", sameObj._id)) @@ -580,17 +604,19 @@ class DevProcess { def matchDevAndAddAddress(){ - ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\56万企业地址\\56万需修改地址.xlsx",{d-> - if(d['L'] && d['L']=="路正常" && d['K']){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\统计局区域的开发区匹配省级以上(49个对应名称1个新增).xlsx",{d-> + if(d['G']){ def q = new BasicDBObject() - q.put("name", d['K']) + q.put("name", d['E']) def obj = development_zone_CONN.findOne(q) if(obj){ - d['Q'] = obj._id.toString() - d['R'] = obj.name - def address = d['I']+d['J'] - q.put("address", address) + d['N'] = obj._id.toString() + d['O'] = obj.name +// def address = d['F']//d['I']+d['J'] +// q.put("address", address) + def street = d['G'] + q.put("township", street) def sameObj = DEV_MAPPING_CONN.findOne(q) if(!sameObj){ def udata = new BasicDBObject() @@ -603,13 +629,39 @@ class DevProcess { } udata.put("name",obj.name) - udata.put("type", "地址") - udata.put("address", address) +// udata.put("type", "地址") + udata.put("type", "乡镇街道") + udata.put("township", street) DEV_MAPPING_CONN.save(udata) - println obj.name +" "+address + println obj.name +" "+street } } } },0); } + + def addAlias(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区别名.xlsx",{d-> + def q = new BasicDBObject() + q.put("name", d['A']) + def obj = development_zone_CONN.findOne(q) + if(obj){ + d['D'] = obj._id.toString() + def arrs = d['B'].split("、") + def alias = obj.alias + if(!alias){ + alias = [] + } + for(def s : arrs){ + if(!alias.contains(s)){ + alias << s + } + } + + def udata = new BasicDBObject() + udata.put("alias", alias) + development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + } + }) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy index 1c67e8706..99b913307 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy @@ -13,7 +13,7 @@ class MsbPocTest { task.run(); } def run(){ - def inputfile = "D:\\数据\\建行\\房产估值\\202311广州.xlsx" + def inputfile = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx" // def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx // standard(inputfile) // match(inputfile) @@ -69,21 +69,21 @@ class MsbPocTest { def matchOld(path){ ExcelUtils.multiThreadingModify(path, { row -> - if(!row['S'] && row['D']){ + if(!row['Y'] && row['G']){ def city = row['C'] - def region = row['D'] - def name = row['F'] - def address = row['H'] + def region = null //row['D'] + def name = row['E'] + def address = row['G'] println "[match] city:"+city+" region:"+region+" address:"+address - def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name) + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name) if(jsonObject != null && jsonObject.precise!=-1) { - row['S'] = jsonObject.get("id") - row['T'] = jsonObject.get("city") - row['U'] = jsonObject.get("district") - row['V'] = jsonObject.get("name") - row['W'] = jsonObject.get("propertyType") - row['X'] = jsonObject.get("credit") + row['Y'] = jsonObject.get("id") + row['Z'] = jsonObject.get("city") + row['AA'] = jsonObject.get("district") + row['AB'] = jsonObject.get("name") + row['AC'] = jsonObject.get("propertyType") +// row['X'] = jsonObject.get("credit") } } }) @@ -111,18 +111,18 @@ class MsbPocTest { def match(file){ ExcelUtils.multiThreadingModify(file, { row -> - if(!row['N']){ - def city = row['A'] - def region = row['B'] - def address = row['C'] //row['G']//row['F'] + if(!row['Y']){ + def city = row['C'] + def region = null //row['B'] + def address = row['G'] //row['G']//row['F'] println "[match] city:"+city+" region:"+region+" address:"+address - def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false) + def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false) if(jsonObject != null) { - row['N'] = jsonObject.get("id") - row['O'] = jsonObject.get("city") - row['P'] = jsonObject.get("district") - row['Q'] = jsonObject.get("name") + row['Y'] = jsonObject.get("id") + row['Z'] = jsonObject.get("city") + row['AA'] = jsonObject.get("district") + row['AB'] = jsonObject.get("name") } } }) @@ -169,17 +169,17 @@ class MsbPocTest { def addCCBPrice(path){ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() ExcelUtils.multiThreadingModify(path, { row -> - if(!row['H']){ + if(!row['V']){ // def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district") // if(dist){ - if(row['D'] instanceof String){ - def d = CommonDatautils.findDistrictById(row['D']) + if(row['Y'] instanceof String){ + def d = CommonDatautils.findDistrictById(row['Y']) if(d){ def cityId = cityCodeDataMap.get(d.city) def map = CcbCommonUtils.findAvgPrice(d._id.toString(), cityId) if(map){ - row['H'] = map.avg_price - row['I'] = map.calc_type + row['V'] = map.avg_price +// row['I'] = map.calc_type } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index dc9e9b1c6..dbbf916d8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,14 +33,14 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达.xlsx" + def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx" // addGXDPrice(filePath) // addCcbPrice(filePath) -// addThirdPrice(filePath) + addThirdPrice(filePath) // addFjwPrice(filePath) // exportPkPrice() // addRegion() - addTradePrice(filePath) +// addTradePrice(filePath) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 6d41f48cd..0682ee62c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.xiamen import com.fangjia.core.RegionData import com.fangjia.core.util.GeoUtils +import com.fangjia.core.util.HttpRequest import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -18,6 +19,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl +import net.sf.json.JSONObject import org.bson.types.ObjectId import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkResponse @@ -39,7 +41,7 @@ import java.util.concurrent.atomic.AtomicLong @BatchOwner(Watcher.GaoJuan) class GaoDePoiDataProcess extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) public static def version = new Date().getTime() @@ -50,7 +52,7 @@ class GaoDePoiDataProcess extends BaseBatch{ def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; - DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_new", MAP_POI_URL) + DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL) DBCollection MAPPOI_CONN_TEST = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL) @@ -78,29 +80,38 @@ class GaoDePoiDataProcess extends BaseBatch{ // synchNetwork() // println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime() // sysnchPoi() -// sysnchVillage() + sysnchVillage() - addTownship() +// addTownship() } def addTownship(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\湖州小区.xlsx",{d-> - if(d['J']){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江省小区.xlsx",{d-> + if(d['E']){ // def gaodeMap = getTownshipInfo(d['F']+","+d['E']) // if(gaodeMap && gaodeMap.township){ -// d['H'] = gaodeMap.city -// d['I'] = gaodeMap.district -// d['J'] = gaodeMap.township -// d['K'] = gaodeMap.format_addr +// if(gaodeMap.city && gaodeMap.city!="[]"){ +// d['H'] = gaodeMap.city +// } +// if( gaodeMap.district && gaodeMap.district!='[]'){ +// d['I'] = gaodeMap.district +// } +// if(gaodeMap.township && gaodeMap.township!='[]'){ +// d['J'] = gaodeMap.township +// } +// if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){ +// d['K'] = gaodeMap.format_addr +// } +// // } def q = new BasicDBObject() q.put("city", d['H']) q.put("district", d['I']) - q.put("street",d['G']) + q.put("street",d['J']) def obj = ADMINISTRATIVE_MAPPING_COLL_100.findOne(q) if(obj){ - d['O'] = "是" + d['L'] = "是" } } }) @@ -108,107 +119,161 @@ class GaoDePoiDataProcess extends BaseBatch{ } def sysnchVillage(){ - def pois = findAllVillagePoiMap() + def buducitys = ["乌兰察布市","巴彦淖尔市","渭南市","宝鸡市","枣庄市","日照市","东营市","威海市","太原市","文山壮族苗族自治州","中卫市","临夏回族自治州","辽源市","抚顺市","阿坝藏族羌族自治州","宜宾市","中山市","亳州市","滁州市","宣城市","廊坊市","宁德市","龙岩市","厦门市","莆田市","天水市","清远市","湛江市","阳江市","河源市","潮州市","来宾市","百色市","防城港市","铜仁地区","毕节地区","承德市","衡水市","濮阳市","开封市","焦作市","三门峡市","平顶山市","信阳市","鹤壁市","十堰市","荆门市","武汉市","常德市","岳阳市","娄底市","株洲市","盐城市","苏州市","景德镇市","抚州市","本溪市","盘锦市","包头市","阿拉善盟","榆林市","铜川市","西安市","临沂市","滨州市","青岛市","朔州市","晋中市","巴中市","绵阳市","广安市","资阳市","固原市","甘南藏族自治州","内江市","曲靖市","淮南市","巢湖市","黄山市","淮北市","三明市","漳州市","陇南市","广州市","云浮市","揭阳市","贺州市","南宁市","遵义市","安顺市","张家口市","唐山市","邢台市","安阳市","郑州市","驻马店市","宜昌市","黄冈市","益阳市","邵阳市","湘西土家族苗族自治州","郴州市","泰州市","宿迁市","宜春市","鹰潭市","朝阳市","营口市","丹东市","鄂尔多斯市","延安市","商洛市","济宁市","潍坊市","济南市","上海市","晋城市","南充市","北海市","赤峰市","六安市","池州市","福州市","惠州市","江门市","汕头市","梧州市","柳州市","黔南布依族苗族自治州","保定市","周口市","南阳市","孝感市","黄石市","张家界市","湘潭市","永州市","南京市","徐州市","无锡市","吉安市","葫芦岛市","鞍山市","呼和浩特市","吴忠市","咸阳市","安康市","泰安市","烟台市","吕梁市","运城市","广元市","遂宁市","泸州市","天津市","石嘴山市","昭通市","铜陵市","肇庆市","汕尾市","深圳市","贵港市","黔东南苗族侗族自治州","黔西南布依族苗族自治州","漯河市","扬州市","连云港市","常州市","九江市","萍乡市","辽阳市","汉中市","菏泽市","淄博市","大同市","长治市","阳泉市","马鞍山市","平凉市","银川市","玉林市","咸宁市","怀化市","上饶市","赣州市","聊城市","忻州市","临汾市","达州市","宿州市","随州市","德州市","恩施土家族苗族自治州","阿拉尔市","石河子市","五家渠市","图木舒克市","定安县","儋州市","万宁市","保亭黎族苗族自治县","西沙群岛","济源市","潜江市","中沙群岛","南沙群岛","屯昌县","昌江黎族自治县","陵水黎族自治县","五指山市","仙桃市","琼中黎族苗族自治县","乐东黎族自治县","临高县","琼海市","白沙黎族自治县","东方市","天门市","神农架林区","澄迈县","文昌市","澳门特别行政区","香港特别行政区","桃园市","台北市","南投县","嘉义市","彰化县","新竹县","澎湖县","台东县","宜兰县","新北市","基隆市","屏东县","嘉义县","云林县","花莲县","台南市","台中市","新竹市","高雄市","苗栗县"] + for(def city : buducitys){ + def pois = findAllVillagePoiMap(city) + def rows = createVillageData(city, pois) + save(city, rows) + saveToMG(city, rows) + println city +" "+pois.size()+" add:"+rows.size() + } + + } + + def createVillageData(city, pois){ def rets = new CopyOnWriteArrayList() def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(pois.size()) - - + def townMap = findTownRange(city) for(def i=0;i + if(row['Q'] && row['Q']=='是'){ + def udata = new BasicDBObject() + udata.put("township", row['J']) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(row['A'])), new BasicDBObject('$set', udata)) + } + }) + + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index f14bedadd..96bbbbfcd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -92,7 +92,7 @@ class ElasticUtils { HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { httpAsyncClientBuilder.disableAuthCaching(); CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));//goojia123456 + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456 httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); return httpAsyncClientBuilder; }