diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 97f0a5bee..e92c8b656 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -15,6 +15,7 @@ import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl import groovy.sql.Sql import java.util.concurrent.ConcurrentHashMap @@ -32,6 +33,9 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ + def startDate = 202206 + def endDate = 202208 + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] @@ -41,6 +45,7 @@ class BigdataDealBatch extends BaseBatch{ DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -102,6 +107,9 @@ class BigdataDealBatch extends BaseBatch{ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(rets.size()) +// def lianjiaPriceMap = findThirdPartyPrice("上海") + def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd') + def lianjiaPriceMap = getDistrictStatByCity("上海", startDate) for(def i=0;i 3000) { + price = db.lianjia_esf_price + } + if (!price && db.beike_esf_price && db.beike_esf_price > 3000) { + price = db.beike_esf_price + } + if(price){ + map.put(db.region+"##"+db.name, price) + } + + } + return map + } + + def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def lianjiaPriceMap){ def ret = [] def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district") if(!obj){ return ret; } - ret << cityCodeDatas.get(trade.city) - ret << trade._id.toString() - ret << trade.city - def data = distCodeMap.get(trade.fjRegion) - if(data){ - ret << data.district_code - ret << data.district - }else{ - ret << "" - ret << "" - } - def street = "" - def streetNumber = "" - if(trade.address){ - if(trade.address.toString().contains("弄")){ - String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) - if(number){ - street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) - streetNumber = number + def price = lianjiaPriceMap.get(obj.region+"##"+obj.name) + if(price){ + def mod = java.lang.Math.abs((trade.price-price)*100/price) + if(mod<=10){ + ret << cityCodeDatas.get(trade.city) + ret << trade._id.toString() + ret << trade.city + def data = distCodeMap.get(trade.fjRegion) + if(data){ + ret << data.district_code + ret << data.district + }else{ + ret << "" + ret << "" } - } - if(!street){ - def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name) - if(jsonObj && jsonObj.searchAddress){ - if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){ + def street = "" + def streetNumber = "" + if(trade.address){ + if(trade.address.toString().contains("弄")){ String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) if(number){ street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) streetNumber = number } - }else{ - street = jsonObj.searchAddress.street - streetNumber = jsonObj.searchAddress.streetNumber } + if(!street){ + def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name) + if(jsonObj && jsonObj.searchAddress){ + if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){ + String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) + if(number){ + street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) + streetNumber = number + } + }else{ + street = jsonObj.searchAddress.street + streetNumber = jsonObj.searchAddress.streetNumber + } + } + } } - } - } - if(!street){ - if(obj.address){ - if(obj.address.toString().contains("弄")){ - String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄")) - if(number){ - street = obj.address.toString().substring(0, obj.address.toString().indexOf(number)) - streetNumber = number + if(!street){ + if(obj.address){ + if(obj.address.toString().contains("弄")){ + String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄")) + if(number){ + street = obj.address.toString().substring(0, obj.address.toString().indexOf(number)) + streetNumber = number + } + }else{ + def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name) + if(jsonObj && jsonObj.searchAddress){ + street = jsonObj.searchAddress.street + streetNumber = jsonObj.searchAddress.streetNumber + } + } } + } + + ret << street + ret << streetNumber + + + if(obj){ + ret << obj._id.toString() + ret << trade.fjName + } + + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属 + ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life) + ret << ""//期数 + String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室")) + if(roomNum && roomNum.contains("0")){ + ret << roomNum.substring(0, roomNum.lastIndexOf("0")) }else{ - def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name) - if(jsonObj && jsonObj.searchAddress){ - street = jsonObj.searchAddress.street - streetNumber = jsonObj.searchAddress.streetNumber - } + ret << trade.floor } + + ret << "" + ret << trade.height + def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型 + def buildType = '' + if(arrs){ + buildType = arrs[0] + } + ret << buildType + ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure) + ret << trade.area + ret << "" + ret << getHouseRoom(buildType, trade.room, trade.hall) + ret << trade.room + ret << trade.hall + ret << "" + ret << trade.toilet + ret << CcbCodeUtils.parseHouseUse(obj.estate_type2) + ret << "" + ret << CcbCodeUtils.getDirectionCode(trade.direction) + ret << CcbCodeUtils.getFitmentCode(trade.fitment) + ret << "" + ret << trade.price + ret << trade.totalPrice * 10000 + ret << "" + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd") + ret << "02" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "99" + + ret << "" + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + ret << trade.totalPrice * 10000 + ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") } } - ret << street - ret << streetNumber - - - if(obj){ - ret << obj._id.toString() - ret << trade.fjName - } - - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属 - ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life) - ret << ""//期数 - String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室")) - if(roomNum && roomNum.contains("0")){ - ret << roomNum.substring(0, roomNum.lastIndexOf("0")) - }else{ - ret << trade.floor - } - - ret << "" - ret << trade.height - def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型 - def buildType = '' - if(arrs){ - buildType = arrs[0] - } - ret << buildType - ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure) - ret << trade.area - ret << "" - ret << getHouseRoom(buildType, trade.room, trade.hall) - ret << trade.room - ret << trade.hall - ret << "" - ret << trade.toilet - ret << CcbCodeUtils.parseHouseUse(obj.estate_type2) - ret << "" - ret << CcbCodeUtils.getDirectionCode(trade.direction) - ret << CcbCodeUtils.getFitmentCode(trade.fitment) - ret << "" - ret << trade.price - ret << trade.totalPrice * 10000 - ret << "" - ret << "" - ret << "" - ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd") - ret << "02" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "99" - - ret << "" - ret << "" - ret << "" - ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") - ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") - ret << "FJW" - ret << trade.totalPrice * 10000 - ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") return ret; } @@ -543,11 +581,14 @@ class BigdataDealBatch extends BaseBatch{ def getDistrictStatByCity(city, weekend){ def q = new BasicDBObject() q.put("city", city) - q.put("weekend", weekend) - def dbcursor = DIST_SELL_STAT.find(q) + q.put("weekend", new BasicDBObject('$gte', weekend)) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("weekend",-1)) def map = new ConcurrentHashMap() for(def d : dbcursor){ - map.put(d.region+"@"+d.district_name, d.avg_price) + def price = map.get(d.region+"##"+d.district_name) + if(!price && d.value_type==2){ + map.put(d.region+"##"+d.district_name, d.avg_price) + } } return map } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index f839de5f5..666909b2e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-07-22", "2022-08-24") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-08-24", "2022-09-21") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index bd932d6fd..44494cad2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -45,935 +45,971 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) -//def url = "https://sh.anjuke.com" -//def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com") -//println s - -//def districts = CommonDatautils.findAllFiledDistrict("district", "南宁") -//for(def d : districts){ -// println d.name -//} - -//def dists = CommonDatautils.findAllDistrictByCity("南宁", "district", ["city","region","name"]) -//for(def d : dists){ -// def o = CommonDatautils.findDistrictById(d._id.toString()) -// println o._id.toString() + " "+o.name -//} - -//def citys = CcbCommonUtils.citys; -//def rows = [] +////def url = "https://sh.anjuke.com" +////def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com") +////println s // -//for(def city : citys) { -// def data = CcbCommonUtils.findCityCodeValue() -// def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'"; +////def districts = CommonDatautils.findAllFiledDistrict("district", "南宁") +////for(def d : districts){ +//// println d.name +////} // -// def count = 0 -// sqlInstance.execute(sql,{ -// count = it.count -// }); -// println city +" "+count +////def dists = CommonDatautils.findAllDistrictByCity("南宁", "district", ["city","region","name"]) +////for(def d : dists){ +//// def o = CommonDatautils.findDistrictById(d._id.toString()) +//// println o._id.toString() + " "+o.name +////} +// +////def citys = CcbCommonUtils.citys; +////def rows = [] +//// +////for(def city : citys) { +//// def data = CcbCommonUtils.findCityCodeValue() +//// def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'"; +//// +//// def count = 0 +//// sqlInstance.execute(sql,{ +//// count = it.count +//// }); +//// println city +" "+count +////} +// +////def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927(1).xlsx" +////def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1) +////def dealMap = [:] +////for(def d : datas){ +//// dealMap.put(d['A'], d['B']) +////} +////ExcelUtils.multiThreadingModify(fileName,{d-> +//// if(d['C']){ +//// d['Q'] = dealMap.get(d['C']) +//// } +////},2) +// +// +// +// +////def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0910.xlsx" +// +////def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) +////def dealMap = [:] +////for(def d : datas){ +//// def map = [:] +//// map.put("land_area",d['D']) +//// map.put("build_area",d['F']) +//// map.put("greeningrate_area",d['H']) +//// map.put("parkingcnt_area",d['J']) +//// map.put("zts_area",d['L']) +//// map.put("buildyear_area",d['N']) +//// map.put("plotratio",d['P']) +//// +//// dealMap.put(d['A'], map) +////} +// +////def cityCodeDatas = CcbCommonUtils.findCityCodeValue() +//////小区量 +////ExcelUtils.multiThreadingModify(fileName, { d -> +//// def city = d['B'] +//// if (city) { +//// def cityid = cityCodeDatas.get(city) +//// def total = findCount(cityid, sqlInstance) +//// d['P'] = total +//// } +////},0) +// +//////小区字段覆盖率 +////ExcelUtils.multiThreadingModify(fileName,{d-> +//// def city = d['A'] +//// if(city){ +//// def cityid = cityCodeDatas.get(city) +//// def total = findCount(cityid, sqlInstance) +//// d['AA'] = total +//// +//// def a = findEstateType(cityid, sqlInstance) +//// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) +//// +//// def b = findFiledCount(cityid, 'buildyear', sqlInstance) +//// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) +//// +//// def c = findFiledCount(cityid, 'address', sqlInstance) +//// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) +//// +//// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) +//// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) +//// +//// def e = findFiledCount(cityid, 'landarea', sqlInstance) +//// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) +//// +//// def f = findFiledCount(cityid, 'buildarea', sqlInstance) +//// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) +//// +//// def g = findFiledCount(cityid, 'plotratio', sqlInstance) +//// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) +//// +//// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) +//// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) +//// +//// def i = findFiledCount(cityid, 'latitude', sqlInstance) +//// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) +//// +//// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) +//// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) +//// println d +//// } +////},1) +// +////def fileName = "D:\\数据\\建行\\房产估值\\上海成交.xlsx" +////def datas =ExcelUtils.read(fileName,0,1) +////def index = 0 +////def rows = [] +////for(def d : datas) { +////// def count = findDealDetail(sqlInstance, d['A'], d['B'], d['C'], d['P'], d['I'].toInteger(), d['K']) +//// def count = findAvmData(sqlInstance, d['B'], d['C'], d['P'], d['I'].toInteger(), d['K']) +//// if (count == 0) { +//// index++ +//// rows << d +//// println index +" "+d +//// } +////// if(rows.size()>20){ +////// break +////// } +////// def dists = CommonDatautils.findDistrictsByName(d['A'],d['C'],"district") +////// if(dists.size()==1){ +////// def dist = dists[0] +////// def count = youdaDeal(youda_esfhouse_coll, d['A'], dist.region, d['C'], d['P'], d['I'].toInteger(), d['K']) +////// if (count > 0) { +////// index++ +////// println index +" "+d +////// } +////// }else{ +////// for(def r :dists){ +////// if(d['B'].contains(r.region)){ +////// def count = youdaDeal(youda_esfhouse_coll, d['A'], r.region, d['C'], d['P'], d['I'].toInteger(), d['K']) +////// if (count > 0) { +////// index++ +////// println index +" "+d +////// } +////// } +////// } +////// } +////} +////ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交本次未使用的记录.xlsx", rows, "数据") +////println "共查到:"+index +// +// +//def index = 0 +////def commIdThirdMap = findAllThird(202207, sqlInstance236); +////def cityCodeDatas = CcbCommonUtils.findCityCodeValue() +////cityCodeDatas.each{city,cityId-> +//// +//// deleteThirdByCityId(cityId, 202207, sqlInstance236) +//// println city + "完成" +////} +// +// +//def lines = org.apache.commons.io.FileUtils.readLines(new File("D:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat")) +//for(line in lines) { +// def arrs = line.toString().split("\\|@\\|") +// if(arrs){ +// def sql = "delete from guided_price_gj where id='${arrs[0]}'" +// sqlInstance236.executeUpdate(sql) +// } //} - -//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927(1).xlsx" -//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1) -//def dealMap = [:] -//for(def d : datas){ -// dealMap.put(d['A'], d['B']) +// +//def deleteThirdByCityId(def cityId,def price_date, def sqlInstance236){ +// def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and price_date=${price_date}" +// sqlInstance236.executeUpdate(sql) //} -//ExcelUtils.multiThreadingModify(fileName,{d-> -// if(d['C']){ -// d['Q'] = dealMap.get(d['C']) -// } -//},2) - - - - -//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0910.xlsx" - -//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) -//def dealMap = [:] -//for(def d : datas){ +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区2022-8-23.xlsx", { d -> +//// if (d['C']) { +//// index ++T +//// def cityId = cityCodeDatas.get(d['A']) +//// def sources = findThirdSource(cityId, d['C'], 202207, sqlInstance236) +//// if(sources.size()>1){ +//// if(sources.contains("安居客") && sources.contains("58同城")){ +//// sources.remove("58同城") +//// }else if(sources.contains("链家") && sources.contains("贝壳")){ +//// sources.remove("贝壳") +//// } +//// } +//// if(sources.size()==1){ +//// println index +" 删除 "+ d['C']+" ${d['A']}:"+ cityId +//// deleteThirdParty(cityId, d['C'], 202207, sqlInstance236) +//// } +//// } +////},0) +// +// +//def findAllThird(price_date, def sqlInstance236){ +// def sql = "select commid, data_source from bigdata_third_party_price_gj where price_date=${price_date}" // def map = [:] -// map.put("land_area",d['D']) -// map.put("build_area",d['F']) -// map.put("greeningrate_area",d['H']) -// map.put("parkingcnt_area",d['J']) -// map.put("zts_area",d['L']) -// map.put("buildyear_area",d['N']) -// map.put("plotratio",d['P']) +// sqlInstance236.eachRow(sql,{r-> +// def sources = map.get(r.commid) +// if(!sources){ +// sources = [] +// map.put(r.commid, sources) +// } +// sources << r.data_source +// }); +// return map +//} // -// dealMap.put(d['A'], map) +// +//def deleteThirdParty(def cityId, def commid, def price_date, def sqlInstance236){ +// def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}" +// sqlInstance236.executeUpdate(sql) +//} +// +//def findThirdSource(def cityId, def commid, def price_date, def sqlInstance236){ +// def sql = "select data_source from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}" +// def sources = [] +// sqlInstance236.eachRow(sql,{r-> +// sources << r.data_source +// }); +// return sources +//} +// +// +//def findAvmData(sqlInstance, def region, def name, def area, def price, def dealdate){ +// def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=202106 and data_type='成交' and cityname='上海' and distname='${region}' and commname='${name}' and buildarea=${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd') and target_avg_price=${price}" +// def count = 0 +// sqlInstance.eachRow(sql,{r-> +// count = r.count +// }); +// return count +//} +// +//def findDealDetail(def sqlInstance, def city, def region, def name, def area, def price, def dealdate){ +// def sql = "select dealunitprice from bigdata_deal_gj where cityname='${city}' and distname='${region}' and commname='${name}' and buildarea= ${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd')" +// def count = 0 +// sqlInstance.eachRow(sql,{r-> +// if(r.dealunitprice == price){ +// count++ +// }else{ +// println r.dealunitprice +" price:"+price +// } +// +// }); +// return count +//} +// +// +//def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def price, def dealdate){ +// def date = DateUtils.parseFromTightByPattern(dealdate, "yyyy-MM-dd") +// def q = new BasicDBObject() +// q.put("contractTime", date); +//// q.put("area", new BasicDBObject('$gte', 20)) +// q.put("fjCity",city); +// q.put("fjRegion",region); +// q.put("fjName",name); +// q.put("area",area); +// q.put("price", price) +// def count = youda_esfhouse_coll.count(q) +// return count; +//} +// +// +////def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" +////def dealMap = findDealCount(sqlInstance236, "2022-02-01", "2022-03-01") +////def testMap = findCaluateCityPPE(sqlInstance236, 202203) +////ExcelUtils.modify(fileName,{d-> +//// if(d['C']){ +//// def oldcount = dealMap.get(d['C']) +//// d['X'] = oldcount +//// def casecount = testMap.get(d['C']) +//// d['Y'] = casecount +//// println d +//// } +////},0) +////def rows = findTradePpeData(sqlInstance236) +////// rows.unique() +////def map=findTradeDetail(sqlInstance236,null) +////def datas=[] +////map.each{key,row-> +//// if(!rows.contains(key)){ +//// datas< +//// def rows = findTradePpeData(sqlInstance236, 202203,code) +//// def alltrades=findTradeDetail(sqlInstance236,code) +//// alltrades.each{it-> +//// def commid=it[0] +//// if(!rows.contains(commid)){ +//// datas< +//// if(d['I'] && d['K']){ +//// def dist = CommonDatautils.findDistrict("上海",d['I'], d['K'], "district") +//// if(dist){ +//// d['S'] = dist._id.toString() +//// } +//// } +////},0) +// +////0-正常 1-非小区,2-地块, 3-无效小区,7-自然村 9-高德比对王一期不存在删除 +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\27城一期数据.xlsx",{d-> +//// if(d['A']){ +//// def obj = CommonDatautils.findDistrictById(d['A']) +//// if(obj && obj.name.contains("一期")){ +//// def newName = obj.name.toString().substring(0, obj.name.indexOf("一期")) +//// if(newName){ +//// def masterDist = CommonDatautils.findDistrict(obj.city, obj.region, newName, "district") +//// if(masterDist && masterDist.name_cat==0){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 9) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", obj._id) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// }else{ +//// d['D'] = "主名不存在" +//// } +//// } +//// } +//// } +////},0) +// +////def index = 0 +////ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d-> +//// def flag = d['M'] +//// if(flag==0.0){ +//// if(d['D']){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 11) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", new ObjectId(d['D'])) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// index++ +//// println index +//// } +//// } +//// +////},0) +// +////ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> +//// if(d['A']){ +////// def region = d['B'] +////// def newRegion = RegionData.formatRegion(d['A'], region) +////// if(newRegion) { +////// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 11) +//// udata.put("status", 1) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", dist._id) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// }else{ +//// println d +//// } +////// } +//// } +////},0) +// +// +////ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> +//// if(d['A']){ +////// def region = d['B'] +////// def newRegion = RegionData.formatRegion(d['A'], region) +////// if(newRegion) { +////// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 11) +//// udata.put("status", 1) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", dist._id) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// }else{ +//// println d +//// } +////// } +//// } +////},0) +// +// +// +// +////def citys = CcbCommonUtils.citys +////def rets = [] +////for(def city : citys){ +//// def count = CommonDatautils.findSeaweeds(city, "district", 1) +//// rets << [city, count] +////} +////ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\统计.xlsx", rets, "数据") +// +////def citys = ["广州","重庆","深圳","上海","北京","南京","泉州"] +////for(def city : citys){ +//// def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["name_cat"]) +//// def index = 0 +//// for(def dist : districts){ +//// if(dist.name_cat && dist.name_cat==1){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 0) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", dist._id) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// index++ +//// } +//// } +//// println city +" "+index +////} +// +// +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤的小区.xlsx",{d-> +//// if(d['C']){ +////// def udata = new BasicDBObject() +////// udata.put("name_cat", 0) +////// udata.put("m_date",new Date()) +////// +////// def q = new BasicDBObject() +////// q.put("_id", new ObjectId(d['C'])) +////// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// def count = findDistCount(d['C'], sqlInstance); +//// if(count==0){ +//// println d['C'] +//// } +//// } +////},0) +// +// +////def index = 0 +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d-> +//// if(d['A']){ +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// +//// def alias = [] +//// if(d['H'] && d['H']!=dist.name){ +//// alias << d['H'] +//// if(d['H'].contains("·")){ +//// alias << d['H'].replace("·", "") +//// } +//// if(dist.alias){ +//// for(def aliasName : dist.alias){ +//// if(!alias.contains(aliasName)){ +//// alias << aliasName +//// } +//// } +//// } +//// } +//// +//// def address = null +//// if(d['I']){ +//// address = AddressUtil.formatAddress(dist.city, d['I']) +//// } +//// +//// Long lng = null +//// Long lat = null +//// Long lat2 = null +//// Long lng2 = null +//// if(d['J']){ +//// lng = d['J'].toString().toDouble() * 10000000000l; +//// lat = d['K'].toString().toDouble() * 10000000000l; +//// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) +//// if(json != null && json.containsKey("x")) { +//// lat2 = json.get("y") * 10000000000l +//// lng2 = json.get("x") * 10000000000l +//// } +//// } +//// +//// def region = null +//// if(dist.region!=d['F']){ +//// region = dist.region +//// } +//// +//// def udata = new BasicDBObject() +//// if(region){ +//// udata.put("region", region) +//// } +//// +//// if(address){ +//// udata.put("address", address) +//// } +//// if(lat2){ +//// udata.put("lng", lng) +//// udata.put("lat", lat) +//// udata.put("lng2", lng2) +//// udata.put("lat2", lat2) +//// } +//// +//// if(alias){ +//// udata.put("alias", alias) +//// } +//// if(udata){ +//// index++ +//// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +//// println index+ " "+dist.city +" "+dist.name+" "+udata +//// } +//// +//// } +//// } +//// +////},0) +// +// +// +////def fileName = "C:\\Users\\gaojuan\\Desktop\\分期分区小区.xlsx" +////def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +////ExcelUtils.multiThreadingModify(fileName,{d-> +//// def obj = CommonDatautils.findDistrictFieldById(d['C'],["alias"]) +//// if(obj && obj.alias){ +//// d['H'] = obj.alias.join(" ") +//// } +////// def citycode = d['A'] +////// cityCodeDataMap.each {k,v-> +////// if(citycode.contains(v)){ +////// d['D'] = k +////// } +////// } +////},0) +// +// +// +// +////def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx" +////def datas = ExcelUtils.read(fileName, 0, 1) +////def dealMap = [:] +////for(def d : datas){ +//// dealMap.put(d['A'], d['B']) +////} +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d-> +//// if(d['A']){ +//// d['W'] = dealMap.get(d['A']) +//// } +////},0) +// +// +////def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" +////def updateIndex = 0 +////def deleteIndex = 0 +////ExcelUtils.modify(fileName,{d-> +//// if(d['R'] && d['R'].toString().toDouble().toInteger()==1){ +//// def dist = CommonDatautils.findDistrictById(d['D']) +//// def city = dist.city +//// def region = RegionData.formatRegion(city, d['Q']) +//// if(region && dist.region!=region){ +//// println city +" new:"+region +" old:"+d['C'] +//// def udata = new BasicDBObject() +//// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") +//// if(obj){ +//// deleteIndex++ +//// udata.put("status", 1) +//// udata.put("m_date", new Date()) +//// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) +//// println deleteIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" +//// }else{ +//// updateIndex++ +//// udata.put("region", region) +//// udata.put("m_date", new Date()) +//// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) +//// println updateIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" +//// } +//// }else{ +//// println d['D']+" 不存在" +//// } +//// } +////},0) +// +////def index = 0 +////ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\小区信息修正的数据(3897)_0118.xlsx", { d -> +//// if(d['A']){ +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// if(d['K']){ +//// def formatAddress = AddressUtil.formatAddress(d['B'], d['K']) +//// def address = null +//// if(formatAddress && formatAddress.endsWith("号")) { +//// address = formatAddress +//// } +//// +//// +//// def data = new BasicDBObject(); +//// data.put("m_date", new Date()) +//// if(d['M']){ +//// long glat = d['M'].toString().toDouble() * 10000000000L +//// long glng = d['L'].toString().toDouble() * 10000000000L +//// data.put("lat",glat) +//// data.put("lng",glng) +//// +//// def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3) +//// if(baidu){ +//// long blng = baidu.x * 10000000000L +//// long blat = baidu.y * 10000000000L +//// data.put("lat2",blat) +//// data.put("lng2",blng) +//// } +//// +//// } +//// +//// if(address){ +//// data.put("address", address) +//// } +//// index++ +//// println index +" "+d['B'] + " "+data +//// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', data)) +//// } +//// } +//// } +////}) +// +// +//def findDealCount(sqlInstance, def startDate, def endDate){ +// def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate +// map.put(r.cityname, r.count) +// }); +// return map +//} +// +// +//def findCaluateCityPPE(def sqlInstance, def yearMonth){ +// def sql = "select * from avm_evaluation_gj where year_month=${yearMonth}"; +// def map = [:] +// sqlInstance.eachRow(sql,{r-> +// map.put(r.CITYNAME, r.DEAL_TEST_CASE_NUM) +// }); +// return map +//} +// +// +//def findTradePpeData(def sqlInstance){ +// def sql = "SELECT commid,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate,target_avg_price FROM avm_evaluation_item_gj where year_month=202203 and data_type='成交'"; +// def rows = new HashSet() +// sqlInstance.eachRow(sql,{r-> +// rows< +// def key=r.commid+"##"+r.buildarea+"##"+r.dealunitprice+"##"+r.dealdate +// def row=[] +// row< +// count = r.count +// }); +// return count +//} +// +//def findCount(cityid, sqlInstance){ +// def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0"; +// def count = 0 +// +// sqlInstance.eachRow(sql,{r-> +// count = r.count +// }); +// return count +//} +// +// +// +//def findDistCount(commid, sqlInstance){ +// def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0"; +// def count = 0 +// +// sqlInstance.eachRow(sql,{r-> +// count = r.count +// }); +// return count +//} +// +//def findFiledCount(cityid, field, sqlInstance){ +// def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null"; +// def count = 0 +// sqlInstance.eachRow(sql,{r-> +// count = r.count +// }); +// return count +//} +// +// +////def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" +////def updateIndex = 0 +////def delIndex = 0 +////def cityMap = [:] +////ExcelUtils.multiThreadingModify(fileName,{d-> +//// if(!d['R']){ +//// +//// def obj = CommonDatautils.findDistrictById(d['D']) +//// if(obj){ +//// def ranges = cityMap.get(obj.city) +//// if(!ranges){ +//// def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city)) +//// ranges = [] +//// for(def r : dbcursor){ +//// ranges << r +//// } +//// cityMap.put(obj.city, ranges) +//// } +//// +//// if(ranges){ +//// def lng = obj.lng/10000000000.0 +//// def lat = obj.lat/10000000000.0 +//// +//// def region = null +//// for(def r : ranges){ +//// try{ +//// def flag = isInArea(lng, lat, r.polyline) +//// if(flag){ +//// def regionStr = r.region +//// region = RegionData.formatRegion(obj.city, regionStr) +//// break +//// } +//// }catch(Exception e){ +//// println e.getMessage() +//// } +//// +//// } +//// if(region && region!=obj.region){ +//// updateIndex++ +//// DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) +//// println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name +//// }else if(region && region==obj.region){ +//// println "[same] "+obj.city +" "+obj.name +//// }else{ +//// DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1))) +//// delIndex++ +//// println delIndex +" [delete] "+obj.city +" "+obj.name +//// } +//// }else{ +//// println obj.city +" 没有此城市边框信息" +//// } +//// +//// }else{ +//// println d['D']+" 没找到该小区" +//// } +//// +//// } +////},0); +// +// +// +////def city = "杭州" +////def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) +////def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",city)) +////def ranges = [] +////for(def d : dbcursor){ +//// ranges << d +////} +////def index =0 +////def updateIndex = 0 +////def delIndex = 0 +////for(def dist : dists){ +////// if(dist.region=="江干"){ +//// def lat = dist.lat/10000000000.0 +//// def lng = dist.lng/10000000000.0 +//// +//// def regionStr =null +//// def region = null +//// for(def r : ranges){ +//// def flag = isInArea(lng, lat, r.polyline) +//// if(flag){ +//// regionStr = r.region +//// region = RegionData.formatRegion(dist.city, regionStr) +//// break +//// } +//// } +//// // def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) +//// if(regionStr && region!=dist.region){ +//// +//// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") +//// if(obj){ +//// delIndex++ +//// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1))) +//// println delIndex+" [del]"+region +" "+dist.name +//// }else{ +//// updateIndex++ +//// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) +//// println updateIndex +" [update]old:${dist.region} new:"+region+" "+dist.name +//// } +//// }else{ +//// index++ +//// println index+" [没找到]"+dist.name +//// } +////// } +////} +// +// +// +// +// +// +//def isInArea(lng, lat, points){ +// List pointList = InsidePolygon.createPoint2(points); +// //Point point = new Point(lng, lat); +// //def flag = InsidePolygon.aboutZone(point, pointList) +// def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) +// return flag; +//} +// +//def getStreetInfo(def city, def glocation){ +// def township = null +// if(glocation){ +// def jsonObject = GaoDeApiUtils.geocode(glocation) +// if(jsonObject && jsonObject.regeocode) { +// def addressComponent = jsonObject.regeocode.addressComponent +// if(addressComponent && addressComponent.city.contains(city)){ +// township = addressComponent.township +// } +// } +// } +// return township; +//} +// +// +////def oracleDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\oracle城市统计.xlsx",0,1) +////def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +////def oracleMap = [:] +////for(def d : oracleDates){ +//// oracleMap.put(d['A'], d['B']) +////} +//// +////def chDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\clickhouse.xlsx",0,1) +////def chMap = [:] +////for(def h : chDates){ +//// chMap.put(h['A'].toString().replace(".0",""), h['B']) +////} +// +////def citys = CcbCommonUtils.citys; +////def map = [:] +////def rets = [] +////for(def c : citys){ +//// def citycode = cityCodeDataMap.get(c) +//// rets << [c, chMap.get(citycode), oracleMap.get(citycode)] +////} +////ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\新导房源比对.xlsx", rets, "统计") +// +////def weekend = DateUtils.getFirstDayOfWeek(new Date()) +////def districts = CommonDatautils.findAllDistrictByCity("上海", "district", ["_id","city", "region", "name", "lng", "lat"]); +////def statMap = getDistrictStatByCity(DIST_SELL_STAT,"上海", weekend) +////def rets = [] +//// +////for(def dist : districts){ +//// def row = [] +//// row << dist.get("_id").toString() +//// row << dist.city +//// row << dist.region +//// row << dist.name +//// if(dist.lng){ +//// row << dist.lng/10000000000.0 +//// row << dist.lat/10000000000.0 +//// }else{ +//// row << "" +//// row << "" +//// } +//// +//// row << statMap.get(dist.region +"##"+dist.name) +//// rets << row +////} +////ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据") +// +////def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1) +////def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1) +//// +////def keys = [] +////for(def d : datas2){ +//// keys << d['A']+"##"+d['C']+"##"+d['B'] +////} +//// +////def index = 0 +////for(def d : datas1){ +//// def k = d['A']+"##"+d['C']+"##"+d['B'] +//// if(!keys.contains(k)){ +//// index ++ +//// println index +" "+k +//// } +////} +// +// +// +// +//def getDistrictStatByCity(DIST_SELL_STAT, city, weekend){ +// def q = new BasicDBObject() +// q.put("city", city) +// q.put("weekend", weekend) +// def dbcursor = DIST_SELL_STAT.find(q) +// def map = new ConcurrentHashMap() +// for(def d : dbcursor){ +// map.put(d.region+"##"+d.district_name, d.avg_price) +// } +// return map //} -//def cityCodeDatas = CcbCommonUtils.findCityCodeValue() -////小区量 -//ExcelUtils.multiThreadingModify(fileName, { d -> -// def city = d['B'] -// if (city) { -// def cityid = cityCodeDatas.get(city) -// def total = findCount(cityid, sqlInstance) -// d['P'] = total + + +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\moban_ccb_all.xlsx" + +//ExcelUtils.multiThreadingModify(fileName, {d-> +// if(d['A']){ +//// def region = RegionData.formatRegion(d['B'], d['C']) +//// if(region){ +//// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") +//// if(dist){ +//// d['E'] = dist._id.toString() +//// } +//// } +//// if(!d['Q']){ +// def dist = CommonDatautils.findDistrictById(d['A']) +//// +// if(dist){ +// d['H'] = dist.region +//// d['G'] = dist.name +// d['I'] = dist.status +// } +//// } +// // } //},0) -////小区字段覆盖率 -//ExcelUtils.multiThreadingModify(fileName,{d-> -// def city = d['A'] -// if(city){ -// def cityid = cityCodeDatas.get(city) -// def total = findCount(cityid, sqlInstance) -// d['AA'] = total -// -// def a = findEstateType(cityid, sqlInstance) -// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) -// -// def b = findFiledCount(cityid, 'buildyear', sqlInstance) -// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) -// -// def c = findFiledCount(cityid, 'address', sqlInstance) -// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) -// -// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) -// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) -// -// def e = findFiledCount(cityid, 'landarea', sqlInstance) -// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) -// -// def f = findFiledCount(cityid, 'buildarea', sqlInstance) -// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) -// -// def g = findFiledCount(cityid, 'plotratio', sqlInstance) -// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) -// -// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) -// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) -// -// def i = findFiledCount(cityid, 'latitude', sqlInstance) -// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) -// -// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) -// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) -// println d -// } -//},1) - -//def fileName = "D:\\数据\\建行\\房产估值\\上海成交.xlsx" -//def datas =ExcelUtils.read(fileName,0,1) -//def index = 0 -//def rows = [] -//for(def d : datas) { -//// def count = findDealDetail(sqlInstance, d['A'], d['B'], d['C'], d['P'], d['I'].toInteger(), d['K']) -// def count = findAvmData(sqlInstance, d['B'], d['C'], d['P'], d['I'].toInteger(), d['K']) -// if (count == 0) { -// index++ -// rows << d -// println index +" "+d -// } -//// if(rows.size()>20){ -//// break -//// } -//// def dists = CommonDatautils.findDistrictsByName(d['A'],d['C'],"district") -//// if(dists.size()==1){ -//// def dist = dists[0] -//// def count = youdaDeal(youda_esfhouse_coll, d['A'], dist.region, d['C'], d['P'], d['I'].toInteger(), d['K']) -//// if (count > 0) { -//// index++ -//// println index +" "+d -//// } -//// }else{ -//// for(def r :dists){ -//// if(d['B'].contains(r.region)){ -//// def count = youdaDeal(youda_esfhouse_coll, d['A'], r.region, d['C'], d['P'], d['I'].toInteger(), d['K']) -//// if (count > 0) { -//// index++ -//// println index +" "+d -//// } -//// } -//// } -//// } -//} -//ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交本次未使用的记录.xlsx", rows, "数据") -//println "共查到:"+index -def index = 0 -//def commIdThirdMap = findAllThird(202207, sqlInstance236); -def cityCodeDatas = CcbCommonUtils.findCityCodeValue() -//cityCodeDatas.each{city,cityId-> -// -// deleteThirdByCityId(cityId, 202207, sqlInstance236) -// println city + "完成" -//} - - -def lines = org.apache.commons.io.FileUtils.readLines(new File("D:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat")) -for(line in lines) { - def arrs = line.toString().split("\\|@\\|") - if(arrs){ - def sql = "delete from guided_price_gj where id='${arrs[0]}'" +ExcelUtils.modify(fileName, {d-> + if(d['A']){ + def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'" + println sql sqlInstance236.executeUpdate(sql) } -} - -def deleteThirdByCityId(def cityId,def price_date, def sqlInstance236){ - def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and price_date=${price_date}" - sqlInstance236.executeUpdate(sql) -} -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区2022-8-23.xlsx", { d -> -// if (d['C']) { -// index ++T -// def cityId = cityCodeDatas.get(d['A']) -// def sources = findThirdSource(cityId, d['C'], 202207, sqlInstance236) -// if(sources.size()>1){ -// if(sources.contains("安居客") && sources.contains("58同城")){ -// sources.remove("58同城") -// }else if(sources.contains("链家") && sources.contains("贝壳")){ -// sources.remove("贝壳") -// } -// } -// if(sources.size()==1){ -// println index +" 删除 "+ d['C']+" ${d['A']}:"+ cityId -// deleteThirdParty(cityId, d['C'], 202207, sqlInstance236) -// } -// } -//},0) - - -def findAllThird(price_date, def sqlInstance236){ - def sql = "select commid, data_source from bigdata_third_party_price_gj where price_date=${price_date}" - def map = [:] - sqlInstance236.eachRow(sql,{r-> - def sources = map.get(r.commid) - if(!sources){ - sources = [] - map.put(r.commid, sources) - } - sources << r.data_source - }); - return map -} - - -def deleteThirdParty(def cityId, def commid, def price_date, def sqlInstance236){ - def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}" - sqlInstance236.executeUpdate(sql) -} - -def findThirdSource(def cityId, def commid, def price_date, def sqlInstance236){ - def sql = "select data_source from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}" - def sources = [] - sqlInstance236.eachRow(sql,{r-> - sources << r.data_source - }); - return sources -} - - -def findAvmData(sqlInstance, def region, def name, def area, def price, def dealdate){ - def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=202106 and data_type='成交' and cityname='上海' and distname='${region}' and commname='${name}' and buildarea=${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd') and target_avg_price=${price}" - def count = 0 - sqlInstance.eachRow(sql,{r-> - count = r.count - }); - return count -} - -def findDealDetail(def sqlInstance, def city, def region, def name, def area, def price, def dealdate){ - def sql = "select dealunitprice from bigdata_deal_gj where cityname='${city}' and distname='${region}' and commname='${name}' and buildarea= ${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd')" - def count = 0 - sqlInstance.eachRow(sql,{r-> - if(r.dealunitprice == price){ - count++ - }else{ - println r.dealunitprice +" price:"+price - } - - }); - return count -} - - -def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def price, def dealdate){ - def date = DateUtils.parseFromTightByPattern(dealdate, "yyyy-MM-dd") - def q = new BasicDBObject() - q.put("contractTime", date); -// q.put("area", new BasicDBObject('$gte', 20)) - q.put("fjCity",city); - q.put("fjRegion",region); - q.put("fjName",name); - q.put("area",area); - q.put("price", price) - def count = youda_esfhouse_coll.count(q) - return count; -} - - -//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" -//def dealMap = findDealCount(sqlInstance236, "2022-02-01", "2022-03-01") -//def testMap = findCaluateCityPPE(sqlInstance236, 202203) -//ExcelUtils.modify(fileName,{d-> -// if(d['C']){ -// def oldcount = dealMap.get(d['C']) -// d['X'] = oldcount -// def casecount = testMap.get(d['C']) -// d['Y'] = casecount -// println d -// } -//},0) -//def rows = findTradePpeData(sqlInstance236) -//// rows.unique() -//def map=findTradeDetail(sqlInstance236,null) -//def datas=[] -//map.each{key,row-> -// if(!rows.contains(key)){ -// datas< -// def rows = findTradePpeData(sqlInstance236, 202203,code) -// def alltrades=findTradeDetail(sqlInstance236,code) -// alltrades.each{it-> -// def commid=it[0] -// if(!rows.contains(commid)){ -// datas< -// if(d['I'] && d['K']){ -// def dist = CommonDatautils.findDistrict("上海",d['I'], d['K'], "district") -// if(dist){ -// d['S'] = dist._id.toString() -// } -// } -//},0) - -//0-正常 1-非小区,2-地块, 3-无效小区,7-自然村 9-高德比对王一期不存在删除 -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\27城一期数据.xlsx",{d-> -// if(d['A']){ -// def obj = CommonDatautils.findDistrictById(d['A']) -// if(obj && obj.name.contains("一期")){ -// def newName = obj.name.toString().substring(0, obj.name.indexOf("一期")) -// if(newName){ -// def masterDist = CommonDatautils.findDistrict(obj.city, obj.region, newName, "district") -// if(masterDist && masterDist.name_cat==0){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 9) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", obj._id) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// }else{ -// d['D'] = "主名不存在" -// } -// } -// } -// } -//},0) - -//def index = 0 -//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d-> -// def flag = d['M'] -// if(flag==0.0){ -// if(d['D']){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 11) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", new ObjectId(d['D'])) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// index++ -// println index -// } -// } -// -//},0) - -//ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> -// if(d['A']){ -//// def region = d['B'] -//// def newRegion = RegionData.formatRegion(d['A'], region) -//// if(newRegion) { -//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 11) -// udata.put("status", 1) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", dist._id) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// }else{ -// println d -// } -//// } -// } -//},0) - - -//ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> -// if(d['A']){ -//// def region = d['B'] -//// def newRegion = RegionData.formatRegion(d['A'], region) -//// if(newRegion) { -//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 11) -// udata.put("status", 1) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", dist._id) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// }else{ -// println d -// } -//// } -// } -//},0) - - - - -//def citys = CcbCommonUtils.citys -//def rets = [] -//for(def city : citys){ -// def count = CommonDatautils.findSeaweeds(city, "district", 1) -// rets << [city, count] -//} -//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\统计.xlsx", rets, "数据") - -//def citys = ["广州","重庆","深圳","上海","北京","南京","泉州"] -//for(def city : citys){ -// def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["name_cat"]) -// def index = 0 -// for(def dist : districts){ -// if(dist.name_cat && dist.name_cat==1){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 0) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", dist._id) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// index++ -// } -// } -// println city +" "+index -//} - - -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤的小区.xlsx",{d-> -// if(d['C']){ -//// def udata = new BasicDBObject() -//// udata.put("name_cat", 0) -//// udata.put("m_date",new Date()) -//// -//// def q = new BasicDBObject() -//// q.put("_id", new ObjectId(d['C'])) -//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// def count = findDistCount(d['C'], sqlInstance); -// if(count==0){ -// println d['C'] -// } -// } -//},0) - - -//def index = 0 -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d-> -// if(d['A']){ -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// -// def alias = [] -// if(d['H'] && d['H']!=dist.name){ -// alias << d['H'] -// if(d['H'].contains("·")){ -// alias << d['H'].replace("·", "") -// } -// if(dist.alias){ -// for(def aliasName : dist.alias){ -// if(!alias.contains(aliasName)){ -// alias << aliasName -// } -// } -// } -// } -// -// def address = null -// if(d['I']){ -// address = AddressUtil.formatAddress(dist.city, d['I']) -// } -// -// Long lng = null -// Long lat = null -// Long lat2 = null -// Long lng2 = null -// if(d['J']){ -// lng = d['J'].toString().toDouble() * 10000000000l; -// lat = d['K'].toString().toDouble() * 10000000000l; -// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) -// if(json != null && json.containsKey("x")) { -// lat2 = json.get("y") * 10000000000l -// lng2 = json.get("x") * 10000000000l -// } -// } -// -// def region = null -// if(dist.region!=d['F']){ -// region = dist.region -// } -// -// def udata = new BasicDBObject() -// if(region){ -// udata.put("region", region) -// } -// -// if(address){ -// udata.put("address", address) -// } -// if(lat2){ -// udata.put("lng", lng) -// udata.put("lat", lat) -// udata.put("lng2", lng2) -// udata.put("lat2", lat2) -// } -// -// if(alias){ -// udata.put("alias", alias) -// } -// if(udata){ -// index++ -// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) -// println index+ " "+dist.city +" "+dist.name+" "+udata -// } -// -// } -// } -// -//},0) - - - -//def fileName = "C:\\Users\\gaojuan\\Desktop\\分期分区小区.xlsx" -//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -//ExcelUtils.multiThreadingModify(fileName,{d-> -// def obj = CommonDatautils.findDistrictFieldById(d['C'],["alias"]) -// if(obj && obj.alias){ -// d['H'] = obj.alias.join(" ") -// } -//// def citycode = d['A'] -//// cityCodeDataMap.each {k,v-> -//// if(citycode.contains(v)){ -//// d['D'] = k -//// } -//// } -//},0) - - - - -//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx" -//def datas = ExcelUtils.read(fileName, 0, 1) -//def dealMap = [:] -//for(def d : datas){ -// dealMap.put(d['A'], d['B']) -//} -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d-> -// if(d['A']){ -// d['W'] = dealMap.get(d['A']) -// } -//},0) - - -//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" -//def updateIndex = 0 -//def deleteIndex = 0 -//ExcelUtils.modify(fileName,{d-> -// if(d['R'] && d['R'].toString().toDouble().toInteger()==1){ -// def dist = CommonDatautils.findDistrictById(d['D']) -// def city = dist.city -// def region = RegionData.formatRegion(city, d['Q']) -// if(region && dist.region!=region){ -// println city +" new:"+region +" old:"+d['C'] -// def udata = new BasicDBObject() -// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") -// if(obj){ -// deleteIndex++ -// udata.put("status", 1) -// udata.put("m_date", new Date()) -// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) -// println deleteIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" -// }else{ -// updateIndex++ -// udata.put("region", region) -// udata.put("m_date", new Date()) -// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) -// println updateIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" -// } -// }else{ -// println d['D']+" 不存在" -// } -// } -//},0) - -//def index = 0 -//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\小区信息修正的数据(3897)_0118.xlsx", { d -> -// if(d['A']){ -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// if(d['K']){ -// def formatAddress = AddressUtil.formatAddress(d['B'], d['K']) -// def address = null -// if(formatAddress && formatAddress.endsWith("号")) { -// address = formatAddress -// } -// -// -// def data = new BasicDBObject(); -// data.put("m_date", new Date()) -// if(d['M']){ -// long glat = d['M'].toString().toDouble() * 10000000000L -// long glng = d['L'].toString().toDouble() * 10000000000L -// data.put("lat",glat) -// data.put("lng",glng) -// -// def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3) -// if(baidu){ -// long blng = baidu.x * 10000000000L -// long blat = baidu.y * 10000000000L -// data.put("lat2",blat) -// data.put("lng2",blng) -// } -// -// } -// -// if(address){ -// data.put("address", address) -// } -// index++ -// println index +" "+d['B'] + " "+data -// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', data)) -// } -// } -// } -//}) - - -def findDealCount(sqlInstance, def startDate, def endDate){ - def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate - map.put(r.cityname, r.count) - }); - return map -} - - -def findCaluateCityPPE(def sqlInstance, def yearMonth){ - def sql = "select * from avm_evaluation_gj where year_month=${yearMonth}"; - def map = [:] - sqlInstance.eachRow(sql,{r-> - map.put(r.CITYNAME, r.DEAL_TEST_CASE_NUM) - }); - return map -} - - -def findTradePpeData(def sqlInstance){ - def sql = "SELECT commid,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate,target_avg_price FROM avm_evaluation_item_gj where year_month=202203 and data_type='成交'"; - def rows = new HashSet() - sqlInstance.eachRow(sql,{r-> - rows< - def key=r.commid+"##"+r.buildarea+"##"+r.dealunitprice+"##"+r.dealdate - def row=[] - row< - count = r.count - }); - return count -} - -def findCount(cityid, sqlInstance){ - def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0"; - def count = 0 - - sqlInstance.eachRow(sql,{r-> - count = r.count - }); - return count -} - - - -def findDistCount(commid, sqlInstance){ - def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0"; - def count = 0 - - sqlInstance.eachRow(sql,{r-> - count = r.count - }); - return count -} - -def findFiledCount(cityid, field, sqlInstance){ - def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null"; - def count = 0 - sqlInstance.eachRow(sql,{r-> - count = r.count - }); - return count -} - - -//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" -//def updateIndex = 0 -//def delIndex = 0 -//def cityMap = [:] -//ExcelUtils.multiThreadingModify(fileName,{d-> -// if(!d['R']){ -// -// def obj = CommonDatautils.findDistrictById(d['D']) -// if(obj){ -// def ranges = cityMap.get(obj.city) -// if(!ranges){ -// def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city)) -// ranges = [] -// for(def r : dbcursor){ -// ranges << r -// } -// cityMap.put(obj.city, ranges) -// } -// -// if(ranges){ -// def lng = obj.lng/10000000000.0 -// def lat = obj.lat/10000000000.0 -// -// def region = null -// for(def r : ranges){ -// try{ -// def flag = isInArea(lng, lat, r.polyline) -// if(flag){ -// def regionStr = r.region -// region = RegionData.formatRegion(obj.city, regionStr) -// break -// } -// }catch(Exception e){ -// println e.getMessage() -// } -// -// } -// if(region && region!=obj.region){ -// updateIndex++ -// DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) -// println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name -// }else if(region && region==obj.region){ -// println "[same] "+obj.city +" "+obj.name -// }else{ -// DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1))) -// delIndex++ -// println delIndex +" [delete] "+obj.city +" "+obj.name -// } -// }else{ -// println obj.city +" 没有此城市边框信息" -// } -// -// }else{ -// println d['D']+" 没找到该小区" -// } -// -// } -//},0); - - - -//def city = "杭州" -//def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) -//def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",city)) -//def ranges = [] -//for(def d : dbcursor){ -// ranges << d -//} -//def index =0 -//def updateIndex = 0 -//def delIndex = 0 -//for(def dist : dists){ -//// if(dist.region=="江干"){ -// def lat = dist.lat/10000000000.0 -// def lng = dist.lng/10000000000.0 -// -// def regionStr =null -// def region = null -// for(def r : ranges){ -// def flag = isInArea(lng, lat, r.polyline) -// if(flag){ -// regionStr = r.region -// region = RegionData.formatRegion(dist.city, regionStr) -// break -// } -// } -// // def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) -// if(regionStr && region!=dist.region){ -// -// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") -// if(obj){ -// delIndex++ -// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1))) -// println delIndex+" [del]"+region +" "+dist.name -// }else{ -// updateIndex++ -// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) -// println updateIndex +" [update]old:${dist.region} new:"+region+" "+dist.name -// } -// }else{ -// index++ -// println index+" [没找到]"+dist.name -// } -//// } -//} - - - - - - -def isInArea(lng, lat, points){ - List pointList = InsidePolygon.createPoint2(points); - //Point point = new Point(lng, lat); - //def flag = InsidePolygon.aboutZone(point, pointList) - def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) - return flag; -} - -def getStreetInfo(def city, def glocation){ - def township = null - if(glocation){ - def jsonObject = GaoDeApiUtils.geocode(glocation) - if(jsonObject && jsonObject.regeocode) { - def addressComponent = jsonObject.regeocode.addressComponent - if(addressComponent && addressComponent.city.contains(city)){ - township = addressComponent.township - } - } - } - return township; -} - - -//def oracleDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\oracle城市统计.xlsx",0,1) -//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -//def oracleMap = [:] -//for(def d : oracleDates){ -// oracleMap.put(d['A'], d['B']) -//} -// -//def chDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\clickhouse.xlsx",0,1) -//def chMap = [:] -//for(def h : chDates){ -// chMap.put(h['A'].toString().replace(".0",""), h['B']) -//} - -//def citys = CcbCommonUtils.citys; -//def map = [:] -//def rets = [] -//for(def c : citys){ -// def citycode = cityCodeDataMap.get(c) -// rets << [c, chMap.get(citycode), oracleMap.get(citycode)] -//} -//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\新导房源比对.xlsx", rets, "统计") - -//def weekend = DateUtils.getFirstDayOfWeek(new Date()) -//def districts = CommonDatautils.findAllDistrictByCity("上海", "district", ["_id","city", "region", "name", "lng", "lat"]); -//def statMap = getDistrictStatByCity(DIST_SELL_STAT,"上海", weekend) -//def rets = [] -// -//for(def dist : districts){ -// def row = [] -// row << dist.get("_id").toString() -// row << dist.city -// row << dist.region -// row << dist.name -// if(dist.lng){ -// row << dist.lng/10000000000.0 -// row << dist.lat/10000000000.0 -// }else{ -// row << "" -// row << "" -// } -// -// row << statMap.get(dist.region +"##"+dist.name) -// rets << row -//} -//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据") - -//def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1) -//def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1) -// -//def keys = [] -//for(def d : datas2){ -// keys << d['A']+"##"+d['C']+"##"+d['B'] -//} -// -//def index = 0 -//for(def d : datas1){ -// def k = d['A']+"##"+d['C']+"##"+d['B'] -// if(!keys.contains(k)){ -// index ++ -// println index +" "+k -// } -//} - - - - -def getDistrictStatByCity(DIST_SELL_STAT, city, weekend){ - def q = new BasicDBObject() - q.put("city", city) - q.put("weekend", weekend) - def dbcursor = DIST_SELL_STAT.find(q) - def map = new ConcurrentHashMap() - for(def d : dbcursor){ - map.put(d.region+"##"+d.district_name, d.avg_price) - } - return map -} +},0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 4a38d4ca3..b2bbe9713 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -75,13 +75,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","5月总量","6月总量","7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","6月总量","7月总量","8月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-05-01" - def date = "2022-06-01" - def endDate = "2022-07-01" - def newDate = "2022-08-01" + def startDate = "2022-06-01" + def date = "2022-07-01" + def endDate = "2022-08-01" + def newDate = "2022-09-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 33b350c5c..e5281ae53 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -754,7 +754,7 @@ class CommonDatautils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","森望芳草绿岸") +// q.put("name","湖畔佳苑") DBObject keys = new BasicDBObject(); if(fields){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index a13780761..616012302 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2022-07-01" + static def statDate = "2022-08-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser();//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2022年9月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 @@ -284,8 +284,8 @@ class YouDaDataParser { } - def esfParser(){ - def files = ["D:\\数据\\友达\\2022年8月\\查询1.DBF"] + def esfParser(path){ + def files = [path] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index 2560bb842..eac332041 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -86,7 +86,7 @@ class DistrictIndexBatch extends BaseBatch{ city_avg_total_price = 531.2615913 } if(city_avg_total_price){ - calcute(city, dists, code, city_avg_total_price) + // calcute(city, dists, code, city_avg_total_price) calcuteIndex(city, dists, code, city_avg_total_price) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index f975f90b6..c42cfa5ba 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -250,7 +250,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-01-01", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-07-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e, regionText) @@ -469,7 +469,7 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] - def citys= ["石家庄","贵阳"] + def citys= ["苏州"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index 2b8b3cf9b..9bdcef9c4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -36,13 +36,13 @@ class PaimaiDataSend { def task = new PaimaiDataSend() // task.jdRunByMongo() // task.jdRun() -// task.aliZcRun() - task.aliZcRun2() +//// task.aliZcRun() +// task.aliZcRun2() // task.aliFpRun() // task.rmfysszcRun() // task.checkUrl() -// task.paimaiSendToEs() + task.paimaiSendToEs() } @@ -66,7 +66,8 @@ class PaimaiDataSend { def rmfysszcRun(){ def index = 0 def q = new BasicDBObject() - q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"])) +// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"])) + q.put("province","广东省") // q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml") def dbcursor = rmfysszc_coll.find(q).batchSize(3000) for(def d : dbcursor){ @@ -149,10 +150,12 @@ class PaimaiDataSend { def jdRun(){ def index = 0 def q = new BasicDBObject() - q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"])) +// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"])) + q.put("province","广东") // q.put("parser_status", new BasicDBObject('$ne',3)) // q.put("url","https://paimai.jd.com/115669796") def dbcursor = jd_coll.find(q).batchSize(3000) + def rets = [] for(def d : dbcursor){ index++ println "jd:"+index @@ -175,16 +178,49 @@ class PaimaiDataSend { result.put("notice_detail_content", d.notice_detail_content.content) json.put("result", result) - byte[] packedMsg = MessageUtils.pack(json) - CommonUtils.sendMessage("result2parse2dev", packedMsg) + rets << json + if(rets.size()==10000){ + println "共:"+rets.size() + sendJD(rets) + rets.clear() + } + } - def udata = new BasicDBObject() - udata.put("parser_status", 3) - udata.put("m_date", new Date()) - jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata)) + if(rets.size()>0){ + println "共:"+rets.size() + sendJD(rets) } } + def sendJD(def rets){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i - println(data) - def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address) - if (lngLat != null && lngLat.size() == 2 ) { - JSONObject jsonObject = new JSONObject() - jsonObject.put("lon", Double.valueOf(lngLat['lng'])) - jsonObject.put("lat", Double.valueOf(lngLat['lat'])) - EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id)) - } + ElasticUtils.executeSqlQuery("select * from dws_poi where location is not null", { dataT -> + final def data = dataT + executorService.submit(new Runnable() { + @Override + void run() { + println(data) + def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address) + if (lngLat != null && lngLat.size() == 2 ) { + JSONObject jsonObject = new JSONObject() + jsonObject.put("lon", Double.valueOf(lngLat['lng'])) + jsonObject.put("lat", Double.valueOf(lngLat['lat'])) + EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id)) + } + } + }) + }) + executorService.shutdown() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 147d08336..ca74c0184 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -178,11 +178,11 @@ class SeaweedAroundPoi extends BaseBatch { // put("写字楼", "120201"); // put("学校", "141200|141201|141202|141203|141204"); // put("银行", "1601|1603"); -// put("大型超市", "0604"); + put("大型超市", "0604"); // put("商场", "0601"); - put("餐饮", "0501|0502|0503|0504"); +// put("餐饮", "0501|0502|0503|0504"); // put("旅游景点", "110000"); -// put("公园广场", "110100|110101|110102|110103|110104|110105") + put("公园广场", "110100|110101|110102|110103|110104|110105") // put("图书馆", "140500"); // put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107"); // put("丧葬设施", "071900|071901|071902|071903") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiFromExcel.groovy new file mode 100644 index 000000000..66abaacf9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiFromExcel.groovy @@ -0,0 +1,84 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBList +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取 + * + * @author shenjian + * @since 2021/5/24 + */ +class SeaweedAroundPoiFromExcel { + + static void main(String[] args) { + def types = listAllType() + dealPoi(new BasicDBObject(), types) + } + + static dealPoi(DBObject query, def types) { + com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区\\东莞银行房价数据测试20220916-v3.xlsx", { List item -> + def city = item['H'] + def lng = item['AQ'] + def lat = item['AR'] + if (lng) { + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi", jsonStr) + } + Thread.sleep(1000) + print(lng + " " + lat) + } + }); + }; + + + + public static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + }; + + public static final Map KEYWORDS_MAP = new HashMap() { + { +// put("餐饮", "05"); +// put("地铁", "150500"); +// put("公交", "1507"); +// put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); +// put("写字楼", "120201"); +// put("学校", "141200|141201|141202|141203|141204"); +// put("银行", "1601|1603"); + put("大型超市", "0604"); +// put("商场", "0601"); +// put("餐饮", "0501|0502|0503|0504"); +// put("旅游景点", "110000"); + put("公园广场", "110100|110101|110102|110103|110104|110105") +// put("图书馆", "140500"); +// put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107"); +// put("丧葬设施", "071900|071901|071902|071903") + + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 0aa5d9ce5..50ac505aa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -146,11 +146,13 @@ class GaoDeUtils { while (result.size() < count) { params.page = "${page}".toString() page++ - params.key = this.getGaodeKey("v3/place/around") + //params.key = this.getGaodeKey("v3/place/around") + params.key = "25b2d459fd133e22f8abcebeac972692" params.offset = "100" JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); if (json == null) { - params.key = this.getGaodeKey("v3/place/around") + //params.key = this.getGaodeKey("v3/place/around") + params.key = "25b2d459fd133e22f8abcebeac972692" params.offset = "100" json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); }