diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 5072828fd..023cc157d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -389,7 +389,7 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ // ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水" // def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all - def citys = ["上海","广州"]//CcbCommonUtils.citys_all + def citys = CcbCommonUtils.citys_all def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 5c735e4c3..a151ad003 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-21") + def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-22") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index d527b08dc..3b8eb91c4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1289,7 +1289,7 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240607人工指导价.xlsx" +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达指导价6-21.xlsx" ExcelUtils.multiThreadingModify(fileName, {d-> if(d['J']){ @@ -1301,7 +1301,7 @@ ExcelUtils.multiThreadingModify(fileName, {d-> // d['H'] = dist._id.toString() // } // } - if(!d['A'] && d['J'] instanceof String){ + if(d['J'] instanceof String){ def dist = CommonDatautils.findDistrictById(d['J']) if(dist){ d['A'] = dist.city @@ -1318,6 +1318,19 @@ ExcelUtils.multiThreadingModify(fileName, {d-> },0) +//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx" +//ExcelUtils.multiThreadingModify(file, { d -> +// if(d['A']){ +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// d['C'] = dist.city +// d['D'] = dist.region +// d['E'] = dist.name +// def random = RandomUtil.random(-300,50) +// d['F'] = random + d['B'] +// } +// } +//}) //def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx" //ExcelUtils.multiThreadingModify(filePath, { d -> // if(d['B']){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 9326a356e..eb1ffe3ab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1846,39 +1846,6 @@ class CommonDatautils { } return priceMap; } - - - public static def queryDistrictPrice(city, region, name, weekend, cat){ - DBObject queryCondition = new BasicDBObject() - queryCondition.put("city", city); - queryCondition.put("region", region); - queryCondition.put("district_name", name); - queryCondition.put("weekend", weekend); - - if(cat == "district"){ - BasicDBObject queryObj = new BasicDBObject(); - queryObj.putAt(QueryOperators.IN, ["小区","别墅"]); - queryCondition.put("estate_type", queryObj); - }else if(cat == "office"){ - queryCondition.put("estate_type", "写字楼"); - }else if(cat == "shop"){ - queryCondition.put("estate_type", "商铺"); - } - - - DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)) - cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) - def price = 0; - while(cur.hasNext()) { - DBObject dbo = cur.next() - price = dbo.get("avg_price"); - if(price){ - break; - } - } - - return price; - } public static def queryFirstDistrictPrice(city, region, name){ @@ -1917,6 +1884,39 @@ class CommonDatautils { } + public static def queryDistrictPrice(city, region, name, weekend, cat){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("region", region); + queryCondition.put("district_name", name); + queryCondition.put("weekend", weekend); + + if(cat == "district"){ + BasicDBObject queryObj = new BasicDBObject(); + queryObj.putAt(QueryOperators.IN, ["小区","别墅"]); + queryCondition.put("estate_type", queryObj); + }else if(cat == "office"){ + queryCondition.put("estate_type", "写字楼"); + }else if(cat == "shop"){ + queryCondition.put("estate_type", "商铺"); + } + + + DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def price = 0; + while(cur.hasNext()) { + DBObject dbo = cur.next() + price = dbo.get("avg_price"); + if(price){ + break; + } + } + + return price; + } + + public static def queryDistrictPrice(city, region, name){ DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); @@ -4679,7 +4679,7 @@ class CommonDatautils { def q = new BasicDBObject() q.put("type","乡镇街道") - q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省'])) +// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省'])) def keys = new BasicDBObject() keys.put("province",1) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 4dbf6a57c..de279cbea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -39,7 +39,7 @@ class DevProcess { // task.calAddrDistince() // task.updateCityAndDist() - task.addDevAddres1() //按添加地址中包含开发区名称的地址 +// task.addDevAddres1() //按添加地址中包含开发区名称的地址 // task.addDevAddres2()//添加相差2公里范围内的地址 // task.matchDevAndAddAddress() @@ -48,7 +48,7 @@ class DevProcess { // task.createDevMapping() // task.deleteAddr() // task.deleteAddr2() -// task.synchDevTo43() + task.synchDevTo43() // task.addNewDev2() // task.addAlias() @@ -644,16 +644,16 @@ class DevProcess { def matchDevAndAddAddress(){ - ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建去重地址入库.xlsx",{d-> - if(d['J']){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d-> + if(d['E']){ def q = new BasicDBObject() - q.put("name", d['J']) + q.put("name", d['E']) def obj = development_zone_CONN.findOne(q) if(obj){ - d['P'] = obj._id.toString() - d['Q'] = obj.name - def address = d['L']+d['M'] + d['F'] = obj._id.toString() + d['G'] = obj.name + def address = d['D'] q.put("address", address) // def street = d['G'] // q.put("township", street) @@ -685,14 +685,14 @@ class DevProcess { def matchDevTownship(){ - ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d-> - if(d['C']){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d-> + if(d['D']){ def q = new BasicDBObject() - q.put("name", d['C']) + q.put("name", d['D']) def obj = development_zone_CONN.findOne(q) if(obj){ - d['E'] = obj._id.toString() - def street = d['D'] + d['F'] = obj._id.toString() + def street = d['E'] q.put("township", street) def sameObj = DEV_MAPPING_CONN.findOne(q) if(!sameObj){ @@ -706,11 +706,11 @@ class DevProcess { } udata.put("name",obj.name) -// udata.put("type", "乡镇街道") -// udata.put("township", street) + udata.put("type", "乡镇街道") + udata.put("township", street) - udata.put("type", "区域") - udata.put("dev_district", street) +// udata.put("type", "区域") +// udata.put("dev_district", street) udata.put("c_date",new Date()) DEV_MAPPING_CONN.save(udata) println obj.name +" "+street diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy index 76ed1b323..8f2333705 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy @@ -39,15 +39,15 @@ class PoiLocation { } def run(){ def citys = ["温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市", "南京市","苏州市","无锡市","扬州市","常州市","徐州市","镇江市","南通市","泰州市","连云港市","盐城市","淮安市","宿迁市","济南市","青岛市","烟台市","威海市","潍坊市","淄博市","临沂市","泰安市","菏泽市","东营市","德州市","济宁市","日照市","聊城市","滨州市","枣庄市"] - for(def city : citys){ - findPois(city) - } +// for(def city : citys){ + findPois() +// } } - def findPois(def city){ + def findPois(){ def q = new BasicDBObject() - q.put("city", city) +// q.put("city", city) q.put("run_status", new BasicDBObject('$exists', false)) def keys = new BasicDBObject() @@ -115,7 +115,9 @@ class PoiLocation { def regionFlag = null if(d.district){ for(def rs : regions){ - regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates)) + if(rs.shape && rs.shape.coordinates){ + regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates)) + } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy index f3d00c82c..540fc1ffa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy @@ -50,7 +50,7 @@ class TownMiddleProcess { //修改乡镇街道别名 // updateTownAlias() //中间表同步到100 -// synchTown() + synchTown() //添加地级市标志 // addCityFlag() // addPinyin() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy index f6e830017..12012bc8e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy @@ -3,9 +3,11 @@ package com.fangjia.script.gj.data_handle import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject import com.mongodb.DBCollection import java.util.concurrent.CopyOnWriteArrayList @@ -27,7 +29,9 @@ class TownshipStrandUpdate { static main(args) { def task = new TownshipStrandUpdate() - task.process() +// task.process() + task.updateTown() +// task.stat() } @@ -42,13 +46,14 @@ class TownshipStrandUpdate { def citys = CommonDatautils.findAllCity() citys = citys.reverse() for(def city : citys){ - if(["中山"].contains(city)){ +// + if(["中山","东莞"].contains(city)){ continue } - def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx" + def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx" File file=new File(fileName); if(!file.exists()){ - def dists = CommonDatautils.findAllDistrictsFromMg(city, "district", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"]) + def dists = CommonDatautils.findAllDistrictsFromMg(city, "shop", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"]) dists = checkData(dists) def datas = createDatas(dists) ExcelUtils.write(fileName, datas, "${city}") @@ -56,6 +61,77 @@ class TownshipStrandUpdate { } } +// 相同且为统计局乡镇 +// 更正 +// 不同且不为统计局乡镇 +// 区域不同 +// 地图无乡镇街道 +// 城市不同 + + def stat(){ + def citys = CommonDatautils.findAllCity() + citys = citys.reverse() + def rets = [] + rets << ["城市", "小区量", "更正量","相同且为统计局乡镇","不同且不为统计局乡镇","区域不同","地图无乡镇街道","城市不同","无值"] + for(def city : citys){ + def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx" + File file=new File(fileName); + def map = [:] + def datas = null + if(file.exists()){ + datas = ExcelUtils.read(fileName,0, 1) + for(def d : datas){ + def key = "空值" + if(d['S']){ + key = d['S'] + } + def count = map.get(key) + if(!count){ + count = 0 + } + map.put(key, count+1) + } + } + + println city + map.each {k,v-> + println k+" "+v + } + println "--------------------------------------------------------------------------------------------" + if(map){ + rets << [city, datas.size(), map.get("更正"),map.get("相同且为统计局乡镇"),map.get("不同且不为统计局乡镇"),map.get("区域不同"),map.get("地图无乡镇街道"),map.get("城市不同"),map.get("空值")] + } + + } + ExcelUtils.write("D:\\数据\\数据建设\\乡镇街道\\乡镇街道更新结果统计.xlsx", rets, "商铺") + } + + def updateTown(){ + def citys = CommonDatautils.findAllCity() + citys = citys.reverse() + for(def city : citys){ + def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx" + File file=new File(fileName); + if(file.exists()){ + println fileName + ExcelUtils.multiThreadingModify(fileName,{d-> + if(d['S']!='相同且为统计局乡镇' && d['S']){ + def q = new BasicDBObject() + q.put("city", d['A']) + q.put("region", d['B']) + q.put("name", d['C']) + q.put("status", 0) + q.put("cat","district") + + def udata = new BasicDBObject() + udata.put("township", d['R']) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + } + }) + } + } + } + def checkData(def dists){ def totalCount = new AtomicLong() def rets = new CopyOnWriteArrayList() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index f0c4282e1..025a68f58 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,11 +33,11 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\神经网络计算.xlsx" - addGXDPrice(filePath) + def filePath = "D:\\数据\\建行\\房产估值\\0621.xlsx" +// addGXDPrice(filePath) // addCcbPrice(filePath) // addThirdPrice(filePath) -// addFjwPrice(filePath) + addFjwPrice(filePath) // exportPkPrice() // addRegion() // addTradePrice(filePath) @@ -205,7 +205,7 @@ class AddPriceTest { def addThirdPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> if (!d['X']) { - def dist = CommonDatautils.findDistrictById(d['Y']) + def dist = CommonDatautils.findDistrictById(d['A']) if(dist){ def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name) d['X'] = thirdPrice @@ -220,8 +220,8 @@ class AddPriceTest { q.put("city", city) q.put("region", region) q.put("name", name) -// q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404)) - q.put("s_date", 202404) + q.put("s_date", new BasicDBObject('$gte', 202403)) +// q.put("s_date", 202404) def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def price = null for(def db : dbcursor){ @@ -271,13 +271,16 @@ class AddPriceTest { def addFjwPrice(path){ ExcelUtils.multiThreadingModify(path,{d-> if(!d['L']){ - if(d['B'] instanceof String){ - def dist = CommonDatautils.findDistrictById(d['B']) + if(d['A'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['A']) if(dist){ + d['A'] = dist.city + d['B'] = dist.region + d['C'] = dist.name // d['N'] = dist.region def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) if(dbo){ -// d['W'] = dbo.avg_price + d['W'] = dbo.avg_price // d['X'] = dbo.cal_type d['L'] = dbo.avg_price_2b }