diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java index dca91ddd9..f4594ad70 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java @@ -177,8 +177,8 @@ public class BaseAvgImportServlet extends HttpServlet { String realName) { int totalNum = sheet.getLastRowNum(); System.out.println("上传文件共:" + totalNum + "个"); - if (totalNum > 1000) { - totalNum = 1000; + if (totalNum > 5000) { + totalNum = 5000; } Integer totalLine = 0;// 总行数 diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/GeoUtils.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/GeoUtils.java index 00f40baa6..bcb042e67 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/GeoUtils.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/GeoUtils.java @@ -257,4 +257,9 @@ public class GeoUtils { ret += (150.0 * Math.sin(x / 12.0 * pi) + 300.0 * Math.sin(x / 30.0 * pi)) * 2.0 / 3.0; return ret; } + + public static void main(String[] args){ + Map location = gps84ToGcj02(31.218024629574998,121.36562884455); + System.out.println(location.toString()); + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index 82bf9b1ce..078ca3b0f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -1235,7 +1235,7 @@ public class WeiXinEvaluationController { Point point = new Point(Double.valueOf(ll[0]), Double.valueOf(ll[1])); points.add(point); } - Point point = new Point(district.getLng()/10000000000.0, district.getLat()/10000000000.0); + Point point = new Point(district.getLng2()/10000000000.0, district.getLat2()/10000000000.0); //boolean flag = InsidePolygon.aboutZone(point, points); //boolean flag = InsidePolygon.isPtInPoly(point.getLon(), point.getLat(), points); boolean flag = InsidePolygon2.checkIsInPolygon(point, points); diff --git a/projects/fangjia-open/src/main/webapp/static/js/report.js b/projects/fangjia-open/src/main/webapp/static/js/report.js index b61a6e2d1..f3d833ccf 100644 --- a/projects/fangjia-open/src/main/webapp/static/js/report.js +++ b/projects/fangjia-open/src/main/webapp/static/js/report.js @@ -362,7 +362,7 @@ $('.formal_btn').on('click', function () { }); function clickFormalReport(){ - $.prompt('姝e紡鎶ュ憡灏嗗敖蹇偖瀵勭粰鎮紝璇锋敞鎰忔煡鏀讹紝璋㈣阿锛佸鏈夌枒闂彲涓庢垜浠仈绯伙細62392317', '璇峰~鍐欐寮忔姤鍛婇偖瀵勫湴鍧�', function (value) { + $.prompt('姝e紡鎶ュ憡灏嗗敖蹇偖瀵勭粰鎮紝璇锋敞鎰忔煡鏀讹紝璋㈣阿锛� ', '璇峰~鍐欐寮忔姤鍛婇偖瀵勫湴鍧�', function (value) { var address = $(".modal-text-input").val(); if(address==""){ $.alert('鍦板潃涓嶈兘涓虹┖'); diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl index f16637105..eaab26cb0 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl @@ -37,14 +37,13 @@

${fjtEvaluation.address!} <#if fjtEvaluation.houseNum?? && fjtEvaluation.houseNum?contains("鍙�")>${fjtEvaluation.houseNum!}<#else>${fjtEvaluation.houseNum!}鍙�${fjtEvaluation.roomNum!}瀹� 閲嶆柊璇勪及 >

*璇风‘璁や笌浜ц瘉鍦板潃涓�鑷�

- <#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.auditStatus!=0> -

璇勪及鎬讳环锛�<#if fjtEvaluation??> - ${(bohaiEvaluationAudit.inquirePrice/10000)?int} 涓囧厓

-

璇勪及鍗曚环锛�<#if bohaiEvaluationAudit.inquirePrice gt 0>${(bohaiEvaluationAudit.inquirePrice/fjtEvaluation.houseArea)?int} 鍏�/骞崇背

+ <#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.auditStatus!=0 && bohaiEvaluationAudit.inquirePrice gt 0> +

璇勪及鎬讳环锛�<#if fjtEvaluation??><#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice/10000)?int}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice/10000)?int} 涓囧厓

+

璇勪及鍗曚环锛�<#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice/fjtEvaluation.houseArea)?int}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice/fjtEvaluation.houseArea)?int} 鍏�/骞崇背

<#if role?? && role==17 && loanInfoMap??>

鎵�灞炲垎绫伙細${loanInfoMap.type!}绫�

鍙捶鎴愭暟锛�${loanInfoMap.loan_ratio!}鎴�

-

鍙捶閲戦锛�${(bohaiEvaluationAudit.inquirePrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}涓囧厓

+

鍙捶閲戦锛�<#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}涓囧厓

璐锋鍒╃巼锛�4.5%

<#else> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 97f0a5bee..e92c8b656 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -15,6 +15,7 @@ import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl import groovy.sql.Sql import java.util.concurrent.ConcurrentHashMap @@ -32,6 +33,9 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ + def startDate = 202206 + def endDate = 202208 + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] @@ -41,6 +45,7 @@ class BigdataDealBatch extends BaseBatch{ DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -102,6 +107,9 @@ class BigdataDealBatch extends BaseBatch{ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(rets.size()) +// def lianjiaPriceMap = findThirdPartyPrice("上海") + def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd') + def lianjiaPriceMap = getDistrictStatByCity("上海", startDate) for(def i=0;i 3000) { + price = db.lianjia_esf_price + } + if (!price && db.beike_esf_price && db.beike_esf_price > 3000) { + price = db.beike_esf_price + } + if(price){ + map.put(db.region+"##"+db.name, price) + } + + } + return map + } + + def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def lianjiaPriceMap){ def ret = [] def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district") if(!obj){ return ret; } - ret << cityCodeDatas.get(trade.city) - ret << trade._id.toString() - ret << trade.city - def data = distCodeMap.get(trade.fjRegion) - if(data){ - ret << data.district_code - ret << data.district - }else{ - ret << "" - ret << "" - } - def street = "" - def streetNumber = "" - if(trade.address){ - if(trade.address.toString().contains("弄")){ - String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) - if(number){ - street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) - streetNumber = number + def price = lianjiaPriceMap.get(obj.region+"##"+obj.name) + if(price){ + def mod = java.lang.Math.abs((trade.price-price)*100/price) + if(mod<=10){ + ret << cityCodeDatas.get(trade.city) + ret << trade._id.toString() + ret << trade.city + def data = distCodeMap.get(trade.fjRegion) + if(data){ + ret << data.district_code + ret << data.district + }else{ + ret << "" + ret << "" } - } - if(!street){ - def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name) - if(jsonObj && jsonObj.searchAddress){ - if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){ + def street = "" + def streetNumber = "" + if(trade.address){ + if(trade.address.toString().contains("弄")){ String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) if(number){ street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) streetNumber = number } - }else{ - street = jsonObj.searchAddress.street - streetNumber = jsonObj.searchAddress.streetNumber } + if(!street){ + def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name) + if(jsonObj && jsonObj.searchAddress){ + if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){ + String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) + if(number){ + street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) + streetNumber = number + } + }else{ + street = jsonObj.searchAddress.street + streetNumber = jsonObj.searchAddress.streetNumber + } + } + } } - } - } - if(!street){ - if(obj.address){ - if(obj.address.toString().contains("弄")){ - String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄")) - if(number){ - street = obj.address.toString().substring(0, obj.address.toString().indexOf(number)) - streetNumber = number + if(!street){ + if(obj.address){ + if(obj.address.toString().contains("弄")){ + String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄")) + if(number){ + street = obj.address.toString().substring(0, obj.address.toString().indexOf(number)) + streetNumber = number + } + }else{ + def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name) + if(jsonObj && jsonObj.searchAddress){ + street = jsonObj.searchAddress.street + streetNumber = jsonObj.searchAddress.streetNumber + } + } } + } + + ret << street + ret << streetNumber + + + if(obj){ + ret << obj._id.toString() + ret << trade.fjName + } + + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属 + ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life) + ret << ""//期数 + String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室")) + if(roomNum && roomNum.contains("0")){ + ret << roomNum.substring(0, roomNum.lastIndexOf("0")) }else{ - def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name) - if(jsonObj && jsonObj.searchAddress){ - street = jsonObj.searchAddress.street - streetNumber = jsonObj.searchAddress.streetNumber - } + ret << trade.floor } + + ret << "" + ret << trade.height + def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型 + def buildType = '' + if(arrs){ + buildType = arrs[0] + } + ret << buildType + ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure) + ret << trade.area + ret << "" + ret << getHouseRoom(buildType, trade.room, trade.hall) + ret << trade.room + ret << trade.hall + ret << "" + ret << trade.toilet + ret << CcbCodeUtils.parseHouseUse(obj.estate_type2) + ret << "" + ret << CcbCodeUtils.getDirectionCode(trade.direction) + ret << CcbCodeUtils.getFitmentCode(trade.fitment) + ret << "" + ret << trade.price + ret << trade.totalPrice * 10000 + ret << "" + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd") + ret << "02" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "99" + + ret << "" + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + ret << trade.totalPrice * 10000 + ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") } } - ret << street - ret << streetNumber - - - if(obj){ - ret << obj._id.toString() - ret << trade.fjName - } - - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属 - ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life) - ret << ""//期数 - String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室")) - if(roomNum && roomNum.contains("0")){ - ret << roomNum.substring(0, roomNum.lastIndexOf("0")) - }else{ - ret << trade.floor - } - - ret << "" - ret << trade.height - def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型 - def buildType = '' - if(arrs){ - buildType = arrs[0] - } - ret << buildType - ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure) - ret << trade.area - ret << "" - ret << getHouseRoom(buildType, trade.room, trade.hall) - ret << trade.room - ret << trade.hall - ret << "" - ret << trade.toilet - ret << CcbCodeUtils.parseHouseUse(obj.estate_type2) - ret << "" - ret << CcbCodeUtils.getDirectionCode(trade.direction) - ret << CcbCodeUtils.getFitmentCode(trade.fitment) - ret << "" - ret << trade.price - ret << trade.totalPrice * 10000 - ret << "" - ret << "" - ret << "" - ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd") - ret << "02" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "" - ret << "99" - - ret << "" - ret << "" - ret << "" - ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") - ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") - ret << "FJW" - ret << trade.totalPrice * 10000 - ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") return ret; } @@ -543,11 +581,14 @@ class BigdataDealBatch extends BaseBatch{ def getDistrictStatByCity(city, weekend){ def q = new BasicDBObject() q.put("city", city) - q.put("weekend", weekend) - def dbcursor = DIST_SELL_STAT.find(q) + q.put("weekend", new BasicDBObject('$gte', weekend)) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("weekend",-1)) def map = new ConcurrentHashMap() for(def d : dbcursor){ - map.put(d.region+"@"+d.district_name, d.avg_price) + def price = map.get(d.region+"##"+d.district_name) + if(!price && d.value_type==2){ + map.put(d.region+"##"+d.district_name, d.avg_price) + } } return map } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 407d37700..6e1b76aa5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220718, 20220819); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220818, 20220919); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index db3beb6ef..e4f47e2fd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -77,8 +77,16 @@ class CcbCommonUtils { q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate))); def dbcursor = GUIDING_PRICE_COLL.find(q) def rets = [] + def sets = new HashSet() for(def d : dbcursor){ - rets << d + def guiding_date = DateUtils.formatDateByPattern(d.guiding_date, 'yyyy-MM-dd') + def key = d.city +"##"+d.region+"##"+d.name+"##" + guiding_date + if(!sets.contains(key)){ + rets << d + sets << key + }else{ + println key + } } return rets } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 13d0f6c73..6dcffbacf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -370,24 +370,25 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ - def citys = CcbCommonUtils.citys + def citys = ["上海","广州","东莞"]//CcbCommonUtils.citys def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def publishTime = '2021-10-01' - def month = 202206 + def month = 202207 def rows = [] + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys) { - def rets = findPeGteThirdParty(city, 202207) + def rets = findPeGteThirdParty(city, 202208) def total = 0 println city +" "+rets.size() for(def m : rets){ - def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}" + def cityId = cityCodeDatas.get(city) + def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}" sqlInstance236.executeUpdate(sql) } } } def findPeGteThirdParty(cityName, yearmonth){ - def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30" + def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>25" def rets = [] sqlInstance236.eachRow(sql,{r-> def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy new file mode 100644 index 000000000..0715fc907 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -0,0 +1,209 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import groovy.sql.Sql +import org.apache.commons.lang.StringUtils + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class FjwPricePkCcbPrice { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new FjwPricePkCcbPrice() +// task.run() +// task.updatePrice() + task.addFjwPrice() + } + + def updatePrice(){ + def index = 0 + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\测试\\ccb.xlsx",{d-> + index++ + println index + def dbo = CommonDatautils.queryDistrictPrice(d['B'], d['C'], d['D'], DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd')) + if(dbo){ + d['G'] = dbo.future_value + } + }) + } + + def addFjwPrice(){ + ExcelUtils.modify("D:\\数据\\建行\\测试\\人工查价(7455)_0824_0913.xlsx",{d-> + if(!d['M']){ + def dist = CommonDatautils.findDistrictById(d['D']) + if(dist){ + def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd")) + if(dbo){ + d['M'] = dbo.avg_price + } + } + } + + }) + } + + def run(){ + def ccbPrices = getCommunityBasePriceGj(202208) + println "ccb总量:"+ccbPrices.size() + def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd')) + println "fjw总量:"+ccbPrices.size() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(ccbPrices.size()) + + + def rets = new CopyOnWriteArrayList() + for(def i=0;i + def map = [:] + map.cityname = r.CITYNAME + map.distname = r.DISTNAME + map.commid = r.COMMID + map.commname = r.COMMNAME + map.year_month = r.YEAR_MONTH + map.calc_avg_price = r.CALC_AVG_PRICE + map.avg_price = r.AVG_PRICE + // 计算类型,1:人工估值 2:样本计算 3:神经网络 + if(r.CALC_TYPE){ + if(r.CALC_TYPE == 1){ + map.calc_type = "人工估值" + }else if(r.CALC_TYPE == 2){ + map.calc_type = "样本计算" + }else if(r.CALC_TYPE == 3){ + map.calc_type = "神经网络" + } + } + allRows << map + }); + return allRows + } + + + def batchCreateFile(ret, fileName, headers, charset){ + OutputStream out = new FileOutputStream(fileName); + def sb = new StringBuffer(); +// sb.append(headers.join("@|@")) +// sb.append(System.getProperty("line.separator")); + + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + + def index = 0 + for(def row : ret){ + index++ + sb.append(StringUtils.join(row.toArray(), "@|@")) + sb.append(System.getProperty("line.separator")); + + println index + + if(index%10000==0){ + println index+"开始写入CSV "+new Date().toLocaleString() + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + println index+"结束写入CSV "+new Date().toLocaleString() + } + + } + + if(sb.length()>0){ + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + } + out.close(); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 6838e8e0a..666909b2e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-07-22", "2022-08-21") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-08-24", "2022-09-21") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index bef978957..44494cad2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -30,8 +30,8 @@ import java.util.regex.*; //def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", // "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") -//def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", -// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") //def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", // "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -45,862 +45,971 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) -//def url = "https://sh.anjuke.com" -//def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com") -//println s - -//def districts = CommonDatautils.findAllFiledDistrict("district", "南宁") -//for(def d : districts){ -// println d.name -//} - -//def dists = CommonDatautils.findAllDistrictByCity("南宁", "district", ["city","region","name"]) -//for(def d : dists){ -// def o = CommonDatautils.findDistrictById(d._id.toString()) -// println o._id.toString() + " "+o.name -//} - -//def citys = CcbCommonUtils.citys; -//def rows = [] +////def url = "https://sh.anjuke.com" +////def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com") +////println s // -//for(def city : citys) { -// def data = CcbCommonUtils.findCityCodeValue() -// def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'"; +////def districts = CommonDatautils.findAllFiledDistrict("district", "南宁") +////for(def d : districts){ +//// println d.name +////} // -// def count = 0 -// sqlInstance.execute(sql,{ -// count = it.count -// }); -// println city +" "+count -//} - -//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927(1).xlsx" -//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1) -//def dealMap = [:] -//for(def d : datas){ -// dealMap.put(d['A'], d['B']) -//} -//ExcelUtils.multiThreadingModify(fileName,{d-> -// if(d['C']){ -// d['Q'] = dealMap.get(d['C']) -// } -//},2) - - - - -//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0910.xlsx" - -//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) -//def dealMap = [:] -//for(def d : datas){ -// def map = [:] -// map.put("land_area",d['D']) -// map.put("build_area",d['F']) -// map.put("greeningrate_area",d['H']) -// map.put("parkingcnt_area",d['J']) -// map.put("zts_area",d['L']) -// map.put("buildyear_area",d['N']) -// map.put("plotratio",d['P']) +////def dists = CommonDatautils.findAllDistrictByCity("南宁", "district", ["city","region","name"]) +////for(def d : dists){ +//// def o = CommonDatautils.findDistrictById(d._id.toString()) +//// println o._id.toString() + " "+o.name +////} // -// dealMap.put(d['A'], map) -//} - -//def cityCodeDatas = CcbCommonUtils.findCityCodeValue() -////小区量 -//ExcelUtils.multiThreadingModify(fileName, { d -> -// def city = d['B'] -// if (city) { -// def cityid = cityCodeDatas.get(city) -// def total = findCount(cityid, sqlInstance) -// d['P'] = total -// } -//},0) - -////小区字段覆盖率 -//ExcelUtils.multiThreadingModify(fileName,{d-> -// def city = d['A'] -// if(city){ -// def cityid = cityCodeDatas.get(city) -// def total = findCount(cityid, sqlInstance) -// d['AA'] = total -// -// def a = findEstateType(cityid, sqlInstance) -// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) -// -// def b = findFiledCount(cityid, 'buildyear', sqlInstance) -// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) -// -// def c = findFiledCount(cityid, 'address', sqlInstance) -// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) -// -// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) -// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) -// -// def e = findFiledCount(cityid, 'landarea', sqlInstance) -// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) -// -// def f = findFiledCount(cityid, 'buildarea', sqlInstance) -// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) -// -// def g = findFiledCount(cityid, 'plotratio', sqlInstance) -// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) -// -// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) -// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) -// -// def i = findFiledCount(cityid, 'latitude', sqlInstance) -// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) -// -// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) -// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) -// println d -// } -//},1) - -//def fileName = "D:\\数据\\建行\\房产估值\\上海成交.xlsx" -//def datas =ExcelUtils.read(fileName,0,1) -//def index = 0 -//def rows = [] -//for(def d : datas) { -//// def count = findDealDetail(sqlInstance, d['A'], d['B'], d['C'], d['P'], d['I'].toInteger(), d['K']) -// def count = findAvmData(sqlInstance, d['B'], d['C'], d['P'], d['I'].toInteger(), d['K']) -// if (count == 0) { -// index++ -// rows << d -// println index +" "+d -// } -//// if(rows.size()>20){ -//// break -//// } -//// def dists = CommonDatautils.findDistrictsByName(d['A'],d['C'],"district") -//// if(dists.size()==1){ -//// def dist = dists[0] -//// def count = youdaDeal(youda_esfhouse_coll, d['A'], dist.region, d['C'], d['P'], d['I'].toInteger(), d['K']) -//// if (count > 0) { -//// index++ -//// println index +" "+d -//// } -//// }else{ -//// for(def r :dists){ -//// if(d['B'].contains(r.region)){ -//// def count = youdaDeal(youda_esfhouse_coll, d['A'], r.region, d['C'], d['P'], d['I'].toInteger(), d['K']) -//// if (count > 0) { -//// index++ -//// println index +" "+d -//// } -//// } -//// } -//// } -//} -//ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交本次未使用的记录.xlsx", rows, "数据") -//println "共查到:"+index - - -def findAvmData(sqlInstance, def region, def name, def area, def price, def dealdate){ - def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=202106 and data_type='成交' and cityname='上海' and distname='${region}' and commname='${name}' and buildarea=${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd') and target_avg_price=${price}" - def count = 0 - sqlInstance.eachRow(sql,{r-> - count = r.count - }); - return count -} - -def findDealDetail(def sqlInstance, def city, def region, def name, def area, def price, def dealdate){ - def sql = "select dealunitprice from bigdata_deal_gj where cityname='${city}' and distname='${region}' and commname='${name}' and buildarea= ${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd')" - def count = 0 - sqlInstance.eachRow(sql,{r-> - if(r.dealunitprice == price){ - count++ - }else{ - println r.dealunitprice +" price:"+price - } - - }); - return count -} - - -def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def price, def dealdate){ - def date = DateUtils.parseFromTightByPattern(dealdate, "yyyy-MM-dd") - def q = new BasicDBObject() - q.put("contractTime", date); -// q.put("area", new BasicDBObject('$gte', 20)) - q.put("fjCity",city); - q.put("fjRegion",region); - q.put("fjName",name); - q.put("area",area); - q.put("price", price) - def count = youda_esfhouse_coll.count(q) - return count; -} - - -//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" -//def dealMap = findDealCount(sqlInstance236, "2022-02-01", "2022-03-01") -//def testMap = findCaluateCityPPE(sqlInstance236, 202203) -//ExcelUtils.modify(fileName,{d-> -// if(d['C']){ -// def oldcount = dealMap.get(d['C']) -// d['X'] = oldcount -// def casecount = testMap.get(d['C']) -// d['Y'] = casecount -// println d -// } -//},0) -//def rows = findTradePpeData(sqlInstance236) -//// rows.unique() -//def map=findTradeDetail(sqlInstance236,null) -//def datas=[] -//map.each{key,row-> -// if(!rows.contains(key)){ -// datas< -// def rows = findTradePpeData(sqlInstance236, 202203,code) -// def alltrades=findTradeDetail(sqlInstance236,code) -// alltrades.each{it-> -// def commid=it[0] -// if(!rows.contains(commid)){ -// datas< -// if(d['I'] && d['K']){ -// def dist = CommonDatautils.findDistrict("上海",d['I'], d['K'], "district") -// if(dist){ -// d['S'] = dist._id.toString() -// } -// } -//},0) - -//0-正常 1-非小区,2-地块, 3-无效小区,7-自然村 9-高德比对王一期不存在删除 -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\27城一期数据.xlsx",{d-> -// if(d['A']){ -// def obj = CommonDatautils.findDistrictById(d['A']) -// if(obj && obj.name.contains("一期")){ -// def newName = obj.name.toString().substring(0, obj.name.indexOf("一期")) -// if(newName){ -// def masterDist = CommonDatautils.findDistrict(obj.city, obj.region, newName, "district") -// if(masterDist && masterDist.name_cat==0){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 9) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", obj._id) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// }else{ -// d['D'] = "主名不存在" -// } -// } -// } -// } -//},0) - -//def index = 0 -//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d-> -// def flag = d['M'] -// if(flag==0.0){ -// if(d['D']){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 11) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", new ObjectId(d['D'])) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// index++ -// println index -// } -// } -// -//},0) - -ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> - if(d['A']){ -// def region = d['B'] -// def newRegion = RegionData.formatRegion(d['A'], region) -// if(newRegion) { -// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") - def dist = CommonDatautils.findDistrictById(d['A']) - if(dist){ - def udata = new BasicDBObject() - udata.put("name_cat", 11) - udata.put("status", 1) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", dist._id) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - }else{ - println d - } -// } - } -},0) - - -ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> - if(d['A']){ -// def region = d['B'] -// def newRegion = RegionData.formatRegion(d['A'], region) -// if(newRegion) { -// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") - def dist = CommonDatautils.findDistrictById(d['A']) - if(dist){ - def udata = new BasicDBObject() - udata.put("name_cat", 11) - udata.put("status", 1) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", dist._id) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - }else{ - println d - } -// } - } -},0) - - - - -//def citys = CcbCommonUtils.citys -//def rets = [] -//for(def city : citys){ -// def count = CommonDatautils.findSeaweeds(city, "district", 1) -// rets << [city, count] -//} -//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\统计.xlsx", rets, "数据") - -//def citys = ["广州","重庆","深圳","上海","北京","南京","泉州"] -//for(def city : citys){ -// def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["name_cat"]) -// def index = 0 -// for(def dist : districts){ -// if(dist.name_cat && dist.name_cat==1){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 0) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", dist._id) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// index++ -// } -// } -// println city +" "+index -//} - - -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤的小区.xlsx",{d-> -// if(d['C']){ -//// def udata = new BasicDBObject() -//// udata.put("name_cat", 0) -//// udata.put("m_date",new Date()) +////def citys = CcbCommonUtils.citys; +////def rows = [] //// -//// def q = new BasicDBObject() -//// q.put("_id", new ObjectId(d['C'])) -//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// def count = findDistCount(d['C'], sqlInstance); -// if(count==0){ -// println d['C'] -// } -// } -//},0) - - +////for(def city : citys) { +//// def data = CcbCommonUtils.findCityCodeValue() +//// def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'"; +//// +//// def count = 0 +//// sqlInstance.execute(sql,{ +//// count = it.count +//// }); +//// println city +" "+count +////} +// +////def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927(1).xlsx" +////def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1) +////def dealMap = [:] +////for(def d : datas){ +//// dealMap.put(d['A'], d['B']) +////} +////ExcelUtils.multiThreadingModify(fileName,{d-> +//// if(d['C']){ +//// d['Q'] = dealMap.get(d['C']) +//// } +////},2) +// +// +// +// +////def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0910.xlsx" +// +////def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) +////def dealMap = [:] +////for(def d : datas){ +//// def map = [:] +//// map.put("land_area",d['D']) +//// map.put("build_area",d['F']) +//// map.put("greeningrate_area",d['H']) +//// map.put("parkingcnt_area",d['J']) +//// map.put("zts_area",d['L']) +//// map.put("buildyear_area",d['N']) +//// map.put("plotratio",d['P']) +//// +//// dealMap.put(d['A'], map) +////} +// +////def cityCodeDatas = CcbCommonUtils.findCityCodeValue() +//////小区量 +////ExcelUtils.multiThreadingModify(fileName, { d -> +//// def city = d['B'] +//// if (city) { +//// def cityid = cityCodeDatas.get(city) +//// def total = findCount(cityid, sqlInstance) +//// d['P'] = total +//// } +////},0) +// +//////小区字段覆盖率 +////ExcelUtils.multiThreadingModify(fileName,{d-> +//// def city = d['A'] +//// if(city){ +//// def cityid = cityCodeDatas.get(city) +//// def total = findCount(cityid, sqlInstance) +//// d['AA'] = total +//// +//// def a = findEstateType(cityid, sqlInstance) +//// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) +//// +//// def b = findFiledCount(cityid, 'buildyear', sqlInstance) +//// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) +//// +//// def c = findFiledCount(cityid, 'address', sqlInstance) +//// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) +//// +//// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) +//// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) +//// +//// def e = findFiledCount(cityid, 'landarea', sqlInstance) +//// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) +//// +//// def f = findFiledCount(cityid, 'buildarea', sqlInstance) +//// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) +//// +//// def g = findFiledCount(cityid, 'plotratio', sqlInstance) +//// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) +//// +//// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) +//// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) +//// +//// def i = findFiledCount(cityid, 'latitude', sqlInstance) +//// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) +//// +//// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) +//// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) +//// println d +//// } +////},1) +// +////def fileName = "D:\\数据\\建行\\房产估值\\上海成交.xlsx" +////def datas =ExcelUtils.read(fileName,0,1) +////def index = 0 +////def rows = [] +////for(def d : datas) { +////// def count = findDealDetail(sqlInstance, d['A'], d['B'], d['C'], d['P'], d['I'].toInteger(), d['K']) +//// def count = findAvmData(sqlInstance, d['B'], d['C'], d['P'], d['I'].toInteger(), d['K']) +//// if (count == 0) { +//// index++ +//// rows << d +//// println index +" "+d +//// } +////// if(rows.size()>20){ +////// break +////// } +////// def dists = CommonDatautils.findDistrictsByName(d['A'],d['C'],"district") +////// if(dists.size()==1){ +////// def dist = dists[0] +////// def count = youdaDeal(youda_esfhouse_coll, d['A'], dist.region, d['C'], d['P'], d['I'].toInteger(), d['K']) +////// if (count > 0) { +////// index++ +////// println index +" "+d +////// } +////// }else{ +////// for(def r :dists){ +////// if(d['B'].contains(r.region)){ +////// def count = youdaDeal(youda_esfhouse_coll, d['A'], r.region, d['C'], d['P'], d['I'].toInteger(), d['K']) +////// if (count > 0) { +////// index++ +////// println index +" "+d +////// } +////// } +////// } +////// } +////} +////ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交本次未使用的记录.xlsx", rows, "数据") +////println "共查到:"+index +// +// //def index = 0 -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d-> -// if(d['A']){ -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ +////def commIdThirdMap = findAllThird(202207, sqlInstance236); +////def cityCodeDatas = CcbCommonUtils.findCityCodeValue() +////cityCodeDatas.each{city,cityId-> +//// +//// deleteThirdByCityId(cityId, 202207, sqlInstance236) +//// println city + "完成" +////} // -// def alias = [] -// if(d['H'] && d['H']!=dist.name){ -// alias << d['H'] -// if(d['H'].contains("·")){ -// alias << d['H'].replace("·", "") -// } -// if(dist.alias){ -// for(def aliasName : dist.alias){ -// if(!alias.contains(aliasName)){ -// alias << aliasName -// } -// } -// } -// } // -// def address = null -// if(d['I']){ -// address = AddressUtil.formatAddress(dist.city, d['I']) -// } -// -// Long lng = null -// Long lat = null -// Long lat2 = null -// Long lng2 = null -// if(d['J']){ -// lng = d['J'].toString().toDouble() * 10000000000l; -// lat = d['K'].toString().toDouble() * 10000000000l; -// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) -// if(json != null && json.containsKey("x")) { -// lat2 = json.get("y") * 10000000000l -// lng2 = json.get("x") * 10000000000l -// } -// } -// -// def region = null -// if(dist.region!=d['F']){ -// region = dist.region -// } -// -// def udata = new BasicDBObject() -// if(region){ -// udata.put("region", region) -// } -// -// if(address){ -// udata.put("address", address) -// } -// if(lat2){ -// udata.put("lng", lng) -// udata.put("lat", lat) -// udata.put("lng2", lng2) -// udata.put("lat2", lat2) -// } -// -// if(alias){ -// udata.put("alias", alias) -// } -// if(udata){ -// index++ -// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) -// println index+ " "+dist.city +" "+dist.name+" "+udata -// } -// -// } +//def lines = org.apache.commons.io.FileUtils.readLines(new File("D:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat")) +//for(line in lines) { +// def arrs = line.toString().split("\\|@\\|") +// if(arrs){ +// def sql = "delete from guided_price_gj where id='${arrs[0]}'" +// sqlInstance236.executeUpdate(sql) // } +//} // -//},0) - - - -//def fileName = "C:\\Users\\gaojuan\\Desktop\\分期分区小区.xlsx" -//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -//ExcelUtils.multiThreadingModify(fileName,{d-> -// def obj = CommonDatautils.findDistrictFieldById(d['C'],["alias"]) -// if(obj && obj.alias){ -// d['H'] = obj.alias.join(" ") -// } -//// def citycode = d['A'] -//// cityCodeDataMap.each {k,v-> -//// if(citycode.contains(v)){ -//// d['D'] = k +//def deleteThirdByCityId(def cityId,def price_date, def sqlInstance236){ +// def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and price_date=${price_date}" +// sqlInstance236.executeUpdate(sql) +//} +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区2022-8-23.xlsx", { d -> +//// if (d['C']) { +//// index ++T +//// def cityId = cityCodeDatas.get(d['A']) +//// def sources = findThirdSource(cityId, d['C'], 202207, sqlInstance236) +//// if(sources.size()>1){ +//// if(sources.contains("安居客") && sources.contains("58同城")){ +//// sources.remove("58同城") +//// }else if(sources.contains("链家") && sources.contains("贝壳")){ +//// sources.remove("贝壳") +//// } +//// } +//// if(sources.size()==1){ +//// println index +" 删除 "+ d['C']+" ${d['A']}:"+ cityId +//// deleteThirdParty(cityId, d['C'], 202207, sqlInstance236) //// } //// } -//},0) - - - - -//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx" -//def datas = ExcelUtils.read(fileName, 0, 1) -//def dealMap = [:] -//for(def d : datas){ -// dealMap.put(d['A'], d['B']) +////},0) +// +// +//def findAllThird(price_date, def sqlInstance236){ +// def sql = "select commid, data_source from bigdata_third_party_price_gj where price_date=${price_date}" +// def map = [:] +// sqlInstance236.eachRow(sql,{r-> +// def sources = map.get(r.commid) +// if(!sources){ +// sources = [] +// map.put(r.commid, sources) +// } +// sources << r.data_source +// }); +// return map //} -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d-> -// if(d['A']){ -// d['W'] = dealMap.get(d['A']) -// } -//},0) - - -//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" -//def updateIndex = 0 -//def deleteIndex = 0 -//ExcelUtils.modify(fileName,{d-> -// if(d['R'] && d['R'].toString().toDouble().toInteger()==1){ -// def dist = CommonDatautils.findDistrictById(d['D']) -// def city = dist.city -// def region = RegionData.formatRegion(city, d['Q']) -// if(region && dist.region!=region){ -// println city +" new:"+region +" old:"+d['C'] -// def udata = new BasicDBObject() -// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") -// if(obj){ -// deleteIndex++ -// udata.put("status", 1) -// udata.put("m_date", new Date()) -// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) -// println deleteIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" -// }else{ -// updateIndex++ -// udata.put("region", region) -// udata.put("m_date", new Date()) -// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) -// println updateIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" -// } -// }else{ -// println d['D']+" 不存在" -// } -// } -//},0) - -//def index = 0 -//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\小区信息修正的数据(3897)_0118.xlsx", { d -> -// if(d['A']){ -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// if(d['K']){ -// def formatAddress = AddressUtil.formatAddress(d['B'], d['K']) -// def address = null -// if(formatAddress && formatAddress.endsWith("号")) { -// address = formatAddress -// } // // -// def data = new BasicDBObject(); -// data.put("m_date", new Date()) -// if(d['M']){ -// long glat = d['M'].toString().toDouble() * 10000000000L -// long glng = d['L'].toString().toDouble() * 10000000000L -// data.put("lat",glat) -// data.put("lng",glng) -// -// def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3) -// if(baidu){ -// long blng = baidu.x * 10000000000L -// long blat = baidu.y * 10000000000L -// data.put("lat2",blat) -// data.put("lng2",blng) -// } -// -// } -// -// if(address){ -// data.put("address", address) -// } -// index++ -// println index +" "+d['B'] + " "+data -// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', data)) -// } -// } -// } -//}) - - -def findDealCount(sqlInstance, def startDate, def endDate){ - def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate - map.put(r.cityname, r.count) - }); - return map -} - - -def findCaluateCityPPE(def sqlInstance, def yearMonth){ - def sql = "select * from avm_evaluation_gj where year_month=${yearMonth}"; - def map = [:] - sqlInstance.eachRow(sql,{r-> - map.put(r.CITYNAME, r.DEAL_TEST_CASE_NUM) - }); - return map -} - - -def findTradePpeData(def sqlInstance){ - def sql = "SELECT commid,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate,target_avg_price FROM avm_evaluation_item_gj where year_month=202203 and data_type='成交'"; - def rows = new HashSet() - sqlInstance.eachRow(sql,{r-> - rows< - def key=r.commid+"##"+r.buildarea+"##"+r.dealunitprice+"##"+r.dealdate - def row=[] - row< - count = r.count - }); - return count -} - -def findCount(cityid, sqlInstance){ - def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0"; - def count = 0 - - sqlInstance.eachRow(sql,{r-> - count = r.count - }); - return count -} - - - -def findDistCount(commid, sqlInstance){ - def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0"; - def count = 0 - - sqlInstance.eachRow(sql,{r-> - count = r.count - }); - return count -} - -def findFiledCount(cityid, field, sqlInstance){ - def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null"; - def count = 0 - sqlInstance.eachRow(sql,{r-> - count = r.count - }); - return count -} - - -//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" -//def updateIndex = 0 -//def delIndex = 0 -//def cityMap = [:] -//ExcelUtils.multiThreadingModify(fileName,{d-> -// if(!d['R']){ -// -// def obj = CommonDatautils.findDistrictById(d['D']) -// if(obj){ -// def ranges = cityMap.get(obj.city) -// if(!ranges){ -// def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city)) -// ranges = [] -// for(def r : dbcursor){ -// ranges << r -// } -// cityMap.put(obj.city, ranges) -// } -// -// if(ranges){ -// def lng = obj.lng/10000000000.0 -// def lat = obj.lat/10000000000.0 -// -// def region = null -// for(def r : ranges){ -// try{ -// def flag = isInArea(lng, lat, r.polyline) -// if(flag){ -// def regionStr = r.region -// region = RegionData.formatRegion(obj.city, regionStr) -// break -// } -// }catch(Exception e){ -// println e.getMessage() -// } -// -// } -// if(region && region!=obj.region){ -// updateIndex++ -// DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) -// println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name -// }else if(region && region==obj.region){ -// println "[same] "+obj.city +" "+obj.name -// }else{ -// DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1))) -// delIndex++ -// println delIndex +" [delete] "+obj.city +" "+obj.name -// } -// }else{ -// println obj.city +" 没有此城市边框信息" -// } -// -// }else{ -// println d['D']+" 没找到该小区" -// } -// -// } -//},0); - - - -//def city = "杭州" -//def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) -//def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",city)) -//def ranges = [] -//for(def d : dbcursor){ -// ranges << d +//def deleteThirdParty(def cityId, def commid, def price_date, def sqlInstance236){ +// def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}" +// sqlInstance236.executeUpdate(sql) //} -//def index =0 -//def updateIndex = 0 -//def delIndex = 0 -//for(def dist : dists){ -//// if(dist.region=="江干"){ -// def lat = dist.lat/10000000000.0 -// def lng = dist.lng/10000000000.0 // -// def regionStr =null -// def region = null -// for(def r : ranges){ -// def flag = isInArea(lng, lat, r.polyline) -// if(flag){ -// regionStr = r.region -// region = RegionData.formatRegion(dist.city, regionStr) -// break -// } -// } -// // def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) -// if(regionStr && region!=dist.region){ +//def findThirdSource(def cityId, def commid, def price_date, def sqlInstance236){ +// def sql = "select data_source from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}" +// def sources = [] +// sqlInstance236.eachRow(sql,{r-> +// sources << r.data_source +// }); +// return sources +//} // -// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") -// if(obj){ -// delIndex++ -// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1))) -// println delIndex+" [del]"+region +" "+dist.name -// }else{ -// updateIndex++ -// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) -// println updateIndex +" [update]old:${dist.region} new:"+region+" "+dist.name -// } +// +//def findAvmData(sqlInstance, def region, def name, def area, def price, def dealdate){ +// def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=202106 and data_type='成交' and cityname='上海' and distname='${region}' and commname='${name}' and buildarea=${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd') and target_avg_price=${price}" +// def count = 0 +// sqlInstance.eachRow(sql,{r-> +// count = r.count +// }); +// return count +//} +// +//def findDealDetail(def sqlInstance, def city, def region, def name, def area, def price, def dealdate){ +// def sql = "select dealunitprice from bigdata_deal_gj where cityname='${city}' and distname='${region}' and commname='${name}' and buildarea= ${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd')" +// def count = 0 +// sqlInstance.eachRow(sql,{r-> +// if(r.dealunitprice == price){ +// count++ // }else{ -// index++ -// println index+" [没找到]"+dist.name +// println r.dealunitprice +" price:"+price // } +// +// }); +// return count +//} +// +// +//def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def price, def dealdate){ +// def date = DateUtils.parseFromTightByPattern(dealdate, "yyyy-MM-dd") +// def q = new BasicDBObject() +// q.put("contractTime", date); +//// q.put("area", new BasicDBObject('$gte', 20)) +// q.put("fjCity",city); +// q.put("fjRegion",region); +// q.put("fjName",name); +// q.put("area",area); +// q.put("price", price) +// def count = youda_esfhouse_coll.count(q) +// return count; +//} +// +// +////def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" +////def dealMap = findDealCount(sqlInstance236, "2022-02-01", "2022-03-01") +////def testMap = findCaluateCityPPE(sqlInstance236, 202203) +////ExcelUtils.modify(fileName,{d-> +//// if(d['C']){ +//// def oldcount = dealMap.get(d['C']) +//// d['X'] = oldcount +//// def casecount = testMap.get(d['C']) +//// d['Y'] = casecount +//// println d //// } -//} - - - - - - -def isInArea(lng, lat, points){ - List pointList = InsidePolygon.createPoint2(points); - //Point point = new Point(lng, lat); - //def flag = InsidePolygon.aboutZone(point, pointList) - def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) - return flag; -} - -def getStreetInfo(def city, def glocation){ - def township = null - if(glocation){ - def jsonObject = GaoDeApiUtils.geocode(glocation) - if(jsonObject && jsonObject.regeocode) { - def addressComponent = jsonObject.regeocode.addressComponent - if(addressComponent && addressComponent.city.contains(city)){ - township = addressComponent.township - } - } - } - return township; -} - - -//def oracleDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\oracle城市统计.xlsx",0,1) -//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -//def oracleMap = [:] -//for(def d : oracleDates){ -// oracleMap.put(d['A'], d['B']) +////},0) +////def rows = findTradePpeData(sqlInstance236) +////// rows.unique() +////def map=findTradeDetail(sqlInstance236,null) +////def datas=[] +////map.each{key,row-> +//// if(!rows.contains(key)){ +//// datas< +//// def rows = findTradePpeData(sqlInstance236, 202203,code) +//// def alltrades=findTradeDetail(sqlInstance236,code) +//// alltrades.each{it-> +//// def commid=it[0] +//// if(!rows.contains(commid)){ +//// datas< +//// if(d['I'] && d['K']){ +//// def dist = CommonDatautils.findDistrict("上海",d['I'], d['K'], "district") +//// if(dist){ +//// d['S'] = dist._id.toString() +//// } +//// } +////},0) +// +////0-正常 1-非小区,2-地块, 3-无效小区,7-自然村 9-高德比对王一期不存在删除 +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\27城一期数据.xlsx",{d-> +//// if(d['A']){ +//// def obj = CommonDatautils.findDistrictById(d['A']) +//// if(obj && obj.name.contains("一期")){ +//// def newName = obj.name.toString().substring(0, obj.name.indexOf("一期")) +//// if(newName){ +//// def masterDist = CommonDatautils.findDistrict(obj.city, obj.region, newName, "district") +//// if(masterDist && masterDist.name_cat==0){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 9) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", obj._id) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// }else{ +//// d['D'] = "主名不存在" +//// } +//// } +//// } +//// } +////},0) +// +////def index = 0 +////ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d-> +//// def flag = d['M'] +//// if(flag==0.0){ +//// if(d['D']){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 11) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", new ObjectId(d['D'])) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// index++ +//// println index +//// } +//// } +//// +////},0) +// +////ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> +//// if(d['A']){ +////// def region = d['B'] +////// def newRegion = RegionData.formatRegion(d['A'], region) +////// if(newRegion) { +////// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 11) +//// udata.put("status", 1) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", dist._id) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// }else{ +//// println d +//// } +////// } +//// } +////},0) +// +// +////ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> +//// if(d['A']){ +////// def region = d['B'] +////// def newRegion = RegionData.formatRegion(d['A'], region) +////// if(newRegion) { +////// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 11) +//// udata.put("status", 1) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", dist._id) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// }else{ +//// println d +//// } +////// } +//// } +////},0) +// +// +// +// +////def citys = CcbCommonUtils.citys +////def rets = [] +////for(def city : citys){ +//// def count = CommonDatautils.findSeaweeds(city, "district", 1) +//// rets << [city, count] +////} +////ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\统计.xlsx", rets, "数据") +// +////def citys = ["广州","重庆","深圳","上海","北京","南京","泉州"] +////for(def city : citys){ +//// def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["name_cat"]) +//// def index = 0 +//// for(def dist : districts){ +//// if(dist.name_cat && dist.name_cat==1){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 0) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", dist._id) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// index++ +//// } +//// } +//// println city +" "+index +////} +// +// +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤的小区.xlsx",{d-> +//// if(d['C']){ +////// def udata = new BasicDBObject() +////// udata.put("name_cat", 0) +////// udata.put("m_date",new Date()) +////// +////// def q = new BasicDBObject() +////// q.put("_id", new ObjectId(d['C'])) +////// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +//// def count = findDistCount(d['C'], sqlInstance); +//// if(count==0){ +//// println d['C'] +//// } +//// } +////},0) +// +// +////def index = 0 +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d-> +//// if(d['A']){ +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// +//// def alias = [] +//// if(d['H'] && d['H']!=dist.name){ +//// alias << d['H'] +//// if(d['H'].contains("·")){ +//// alias << d['H'].replace("·", "") +//// } +//// if(dist.alias){ +//// for(def aliasName : dist.alias){ +//// if(!alias.contains(aliasName)){ +//// alias << aliasName +//// } +//// } +//// } +//// } +//// +//// def address = null +//// if(d['I']){ +//// address = AddressUtil.formatAddress(dist.city, d['I']) +//// } +//// +//// Long lng = null +//// Long lat = null +//// Long lat2 = null +//// Long lng2 = null +//// if(d['J']){ +//// lng = d['J'].toString().toDouble() * 10000000000l; +//// lat = d['K'].toString().toDouble() * 10000000000l; +//// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) +//// if(json != null && json.containsKey("x")) { +//// lat2 = json.get("y") * 10000000000l +//// lng2 = json.get("x") * 10000000000l +//// } +//// } +//// +//// def region = null +//// if(dist.region!=d['F']){ +//// region = dist.region +//// } +//// +//// def udata = new BasicDBObject() +//// if(region){ +//// udata.put("region", region) +//// } +//// +//// if(address){ +//// udata.put("address", address) +//// } +//// if(lat2){ +//// udata.put("lng", lng) +//// udata.put("lat", lat) +//// udata.put("lng2", lng2) +//// udata.put("lat2", lat2) +//// } +//// +//// if(alias){ +//// udata.put("alias", alias) +//// } +//// if(udata){ +//// index++ +//// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +//// println index+ " "+dist.city +" "+dist.name+" "+udata +//// } +//// +//// } +//// } +//// +////},0) +// +// +// +////def fileName = "C:\\Users\\gaojuan\\Desktop\\分期分区小区.xlsx" +////def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +////ExcelUtils.multiThreadingModify(fileName,{d-> +//// def obj = CommonDatautils.findDistrictFieldById(d['C'],["alias"]) +//// if(obj && obj.alias){ +//// d['H'] = obj.alias.join(" ") +//// } +////// def citycode = d['A'] +////// cityCodeDataMap.each {k,v-> +////// if(citycode.contains(v)){ +////// d['D'] = k +////// } +////// } +////},0) +// +// +// +// +////def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx" +////def datas = ExcelUtils.read(fileName, 0, 1) +////def dealMap = [:] +////for(def d : datas){ +//// dealMap.put(d['A'], d['B']) +////} +////ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d-> +//// if(d['A']){ +//// d['W'] = dealMap.get(d['A']) +//// } +////},0) +// +// +////def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" +////def updateIndex = 0 +////def deleteIndex = 0 +////ExcelUtils.modify(fileName,{d-> +//// if(d['R'] && d['R'].toString().toDouble().toInteger()==1){ +//// def dist = CommonDatautils.findDistrictById(d['D']) +//// def city = dist.city +//// def region = RegionData.formatRegion(city, d['Q']) +//// if(region && dist.region!=region){ +//// println city +" new:"+region +" old:"+d['C'] +//// def udata = new BasicDBObject() +//// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") +//// if(obj){ +//// deleteIndex++ +//// udata.put("status", 1) +//// udata.put("m_date", new Date()) +//// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) +//// println deleteIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" +//// }else{ +//// updateIndex++ +//// udata.put("region", region) +//// udata.put("m_date", new Date()) +//// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) +//// println updateIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" +//// } +//// }else{ +//// println d['D']+" 不存在" +//// } +//// } +////},0) +// +////def index = 0 +////ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\小区信息修正的数据(3897)_0118.xlsx", { d -> +//// if(d['A']){ +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// if(d['K']){ +//// def formatAddress = AddressUtil.formatAddress(d['B'], d['K']) +//// def address = null +//// if(formatAddress && formatAddress.endsWith("号")) { +//// address = formatAddress +//// } +//// +//// +//// def data = new BasicDBObject(); +//// data.put("m_date", new Date()) +//// if(d['M']){ +//// long glat = d['M'].toString().toDouble() * 10000000000L +//// long glng = d['L'].toString().toDouble() * 10000000000L +//// data.put("lat",glat) +//// data.put("lng",glng) +//// +//// def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3) +//// if(baidu){ +//// long blng = baidu.x * 10000000000L +//// long blat = baidu.y * 10000000000L +//// data.put("lat2",blat) +//// data.put("lng2",blng) +//// } +//// +//// } +//// +//// if(address){ +//// data.put("address", address) +//// } +//// index++ +//// println index +" "+d['B'] + " "+data +//// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', data)) +//// } +//// } +//// } +////}) +// +// +//def findDealCount(sqlInstance, def startDate, def endDate){ +// def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate +// map.put(r.cityname, r.count) +// }); +// return map //} // -//def chDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\clickhouse.xlsx",0,1) -//def chMap = [:] -//for(def h : chDates){ -// chMap.put(h['A'].toString().replace(".0",""), h['B']) -//} - -//def citys = CcbCommonUtils.citys; -//def map = [:] -//def rets = [] -//for(def c : citys){ -// def citycode = cityCodeDataMap.get(c) -// rets << [c, chMap.get(citycode), oracleMap.get(citycode)] -//} -//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\新导房源比对.xlsx", rets, "统计") - -//def weekend = DateUtils.getFirstDayOfWeek(new Date()) -//def districts = CommonDatautils.findAllDistrictByCity("上海", "district", ["_id","city", "region", "name", "lng", "lat"]); -//def statMap = getDistrictStatByCity(DIST_SELL_STAT,"上海", weekend) -//def rets = [] // -//for(def dist : districts){ -// def row = [] -// row << dist.get("_id").toString() -// row << dist.city -// row << dist.region -// row << dist.name -// if(dist.lng){ -// row << dist.lng/10000000000.0 -// row << dist.lat/10000000000.0 -// }else{ -// row << "" -// row << "" +//def findCaluateCityPPE(def sqlInstance, def yearMonth){ +// def sql = "select * from avm_evaluation_gj where year_month=${yearMonth}"; +// def map = [:] +// sqlInstance.eachRow(sql,{r-> +// map.put(r.CITYNAME, r.DEAL_TEST_CASE_NUM) +// }); +// return map +//} +// +// +//def findTradePpeData(def sqlInstance){ +// def sql = "SELECT commid,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate,target_avg_price FROM avm_evaluation_item_gj where year_month=202203 and data_type='成交'"; +// def rows = new HashSet() +// sqlInstance.eachRow(sql,{r-> +// rows< +// def key=r.commid+"##"+r.buildarea+"##"+r.dealunitprice+"##"+r.dealdate +// def row=[] +// row< +// count = r.count +// }); +// return count +//} +// +//def findCount(cityid, sqlInstance){ +// def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0"; +// def count = 0 +// +// sqlInstance.eachRow(sql,{r-> +// count = r.count +// }); +// return count +//} +// +// +// +//def findDistCount(commid, sqlInstance){ +// def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0"; +// def count = 0 +// +// sqlInstance.eachRow(sql,{r-> +// count = r.count +// }); +// return count +//} +// +//def findFiledCount(cityid, field, sqlInstance){ +// def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null"; +// def count = 0 +// sqlInstance.eachRow(sql,{r-> +// count = r.count +// }); +// return count +//} +// +// +////def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" +////def updateIndex = 0 +////def delIndex = 0 +////def cityMap = [:] +////ExcelUtils.multiThreadingModify(fileName,{d-> +//// if(!d['R']){ +//// +//// def obj = CommonDatautils.findDistrictById(d['D']) +//// if(obj){ +//// def ranges = cityMap.get(obj.city) +//// if(!ranges){ +//// def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city)) +//// ranges = [] +//// for(def r : dbcursor){ +//// ranges << r +//// } +//// cityMap.put(obj.city, ranges) +//// } +//// +//// if(ranges){ +//// def lng = obj.lng/10000000000.0 +//// def lat = obj.lat/10000000000.0 +//// +//// def region = null +//// for(def r : ranges){ +//// try{ +//// def flag = isInArea(lng, lat, r.polyline) +//// if(flag){ +//// def regionStr = r.region +//// region = RegionData.formatRegion(obj.city, regionStr) +//// break +//// } +//// }catch(Exception e){ +//// println e.getMessage() +//// } +//// +//// } +//// if(region && region!=obj.region){ +//// updateIndex++ +//// DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) +//// println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name +//// }else if(region && region==obj.region){ +//// println "[same] "+obj.city +" "+obj.name +//// }else{ +//// DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1))) +//// delIndex++ +//// println delIndex +" [delete] "+obj.city +" "+obj.name +//// } +//// }else{ +//// println obj.city +" 没有此城市边框信息" +//// } +//// +//// }else{ +//// println d['D']+" 没找到该小区" +//// } +//// +//// } +////},0); +// +// +// +////def city = "杭州" +////def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) +////def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",city)) +////def ranges = [] +////for(def d : dbcursor){ +//// ranges << d +////} +////def index =0 +////def updateIndex = 0 +////def delIndex = 0 +////for(def dist : dists){ +////// if(dist.region=="江干"){ +//// def lat = dist.lat/10000000000.0 +//// def lng = dist.lng/10000000000.0 +//// +//// def regionStr =null +//// def region = null +//// for(def r : ranges){ +//// def flag = isInArea(lng, lat, r.polyline) +//// if(flag){ +//// regionStr = r.region +//// region = RegionData.formatRegion(dist.city, regionStr) +//// break +//// } +//// } +//// // def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) +//// if(regionStr && region!=dist.region){ +//// +//// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") +//// if(obj){ +//// delIndex++ +//// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1))) +//// println delIndex+" [del]"+region +" "+dist.name +//// }else{ +//// updateIndex++ +//// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) +//// println updateIndex +" [update]old:${dist.region} new:"+region+" "+dist.name +//// } +//// }else{ +//// index++ +//// println index+" [没找到]"+dist.name +//// } +////// } +////} +// +// +// +// +// +// +//def isInArea(lng, lat, points){ +// List pointList = InsidePolygon.createPoint2(points); +// //Point point = new Point(lng, lat); +// //def flag = InsidePolygon.aboutZone(point, pointList) +// def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) +// return flag; +//} +// +//def getStreetInfo(def city, def glocation){ +// def township = null +// if(glocation){ +// def jsonObject = GaoDeApiUtils.geocode(glocation) +// if(jsonObject && jsonObject.regeocode) { +// def addressComponent = jsonObject.regeocode.addressComponent +// if(addressComponent && addressComponent.city.contains(city)){ +// township = addressComponent.township +// } +// } // } -// -// row << statMap.get(dist.region +"##"+dist.name) -// rets << row -//} -//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据") - -//def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1) -//def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1) -// -//def keys = [] -//for(def d : datas2){ -// keys << d['A']+"##"+d['C']+"##"+d['B'] +// return township; //} // -//def index = 0 -//for(def d : datas1){ -// def k = d['A']+"##"+d['C']+"##"+d['B'] -// if(!keys.contains(k)){ -// index ++ -// println index +" "+k +// +////def oracleDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\oracle城市统计.xlsx",0,1) +////def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +////def oracleMap = [:] +////for(def d : oracleDates){ +//// oracleMap.put(d['A'], d['B']) +////} +//// +////def chDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\clickhouse.xlsx",0,1) +////def chMap = [:] +////for(def h : chDates){ +//// chMap.put(h['A'].toString().replace(".0",""), h['B']) +////} +// +////def citys = CcbCommonUtils.citys; +////def map = [:] +////def rets = [] +////for(def c : citys){ +//// def citycode = cityCodeDataMap.get(c) +//// rets << [c, chMap.get(citycode), oracleMap.get(citycode)] +////} +////ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\新导房源比对.xlsx", rets, "统计") +// +////def weekend = DateUtils.getFirstDayOfWeek(new Date()) +////def districts = CommonDatautils.findAllDistrictByCity("上海", "district", ["_id","city", "region", "name", "lng", "lat"]); +////def statMap = getDistrictStatByCity(DIST_SELL_STAT,"上海", weekend) +////def rets = [] +//// +////for(def dist : districts){ +//// def row = [] +//// row << dist.get("_id").toString() +//// row << dist.city +//// row << dist.region +//// row << dist.name +//// if(dist.lng){ +//// row << dist.lng/10000000000.0 +//// row << dist.lat/10000000000.0 +//// }else{ +//// row << "" +//// row << "" +//// } +//// +//// row << statMap.get(dist.region +"##"+dist.name) +//// rets << row +////} +////ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据") +// +////def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1) +////def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1) +//// +////def keys = [] +////for(def d : datas2){ +//// keys << d['A']+"##"+d['C']+"##"+d['B'] +////} +//// +////def index = 0 +////for(def d : datas1){ +//// def k = d['A']+"##"+d['C']+"##"+d['B'] +//// if(!keys.contains(k)){ +//// index ++ +//// println index +" "+k +//// } +////} +// +// +// +// +//def getDistrictStatByCity(DIST_SELL_STAT, city, weekend){ +// def q = new BasicDBObject() +// q.put("city", city) +// q.put("weekend", weekend) +// def dbcursor = DIST_SELL_STAT.find(q) +// def map = new ConcurrentHashMap() +// for(def d : dbcursor){ +// map.put(d.region+"##"+d.district_name, d.avg_price) // } +// return map //} +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\moban_ccb_all.xlsx" -def getDistrictStatByCity(DIST_SELL_STAT, city, weekend){ - def q = new BasicDBObject() - q.put("city", city) - q.put("weekend", weekend) - def dbcursor = DIST_SELL_STAT.find(q) - def map = new ConcurrentHashMap() - for(def d : dbcursor){ - map.put(d.region+"##"+d.district_name, d.avg_price) +//ExcelUtils.multiThreadingModify(fileName, {d-> +// if(d['A']){ +//// def region = RegionData.formatRegion(d['B'], d['C']) +//// if(region){ +//// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") +//// if(dist){ +//// d['E'] = dist._id.toString() +//// } +//// } +//// if(!d['Q']){ +// def dist = CommonDatautils.findDistrictById(d['A']) +//// +// if(dist){ +// d['H'] = dist.region +//// d['G'] = dist.name +// d['I'] = dist.status +// } +//// } +// +// } +//},0) + + + +ExcelUtils.modify(fileName, {d-> + if(d['A']){ + def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'" + println sql + sqlInstance236.executeUpdate(sql) } - return map -} +},0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 261a7f702..c4a27815e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202207//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202205 - def endDate = 202207 + def sDate= 202208//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202206 + def endDate = 202208 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 4a38d4ca3..b2bbe9713 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -75,13 +75,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","5月总量","6月总量","7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","6月总量","7月总量","8月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-05-01" - def date = "2022-06-01" - def endDate = "2022-07-01" - def newDate = "2022-08-01" + def startDate = "2022-06-01" + def date = "2022-07-01" + def endDate = "2022-08-01" + def newDate = "2022-09-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 94c151d1a..e5281ae53 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -754,7 +754,7 @@ class CommonDatautils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","森望芳草绿岸") +// q.put("name","湖畔佳苑") DBObject keys = new BasicDBObject(); if(fields){ @@ -2049,6 +2049,20 @@ class CommonDatautils { } + public static def queryAllDistrictPrice(startDate){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put('weekend', startDate); + DBCursor cur = DIST_STAT_COLL.find(queryCondition) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def rMap = [:]; + while(cur.hasNext()) { + DBObject dbo = cur.next() + rMap.put(dbo.get("city") + "##" + dbo.get("region")+"##"+ dbo.get("district_name"), dbo) + } + return rMap; + } + + public static def queryDistrictPrices(city, startDate){ DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); @@ -3152,8 +3166,7 @@ class CommonDatautils { DBObject dbo; def prices = [] def index = 0 - while(cur.hasNext()) { - def obj = cur.next() + for(def obj : cur) { if(index == 0){ dbo = obj } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/crawl/SchoolLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/crawl/SchoolLocation.groovy new file mode 100644 index 000000000..534a844e7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/crawl/SchoolLocation.groovy @@ -0,0 +1,159 @@ +package com.fangjia.script.crawl + +import com.alibaba.fastjson.JSONArray +import com.fangjia.core.util.GeocodeUtils +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.lang.StringUtils + +class SchoolLocation { + + DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018) + + static main(args) { + def task = new SchoolLocation() + task.run() + } + + def run(){ + def query = new BasicDBObject() + query.put("province","江苏") + //query.put("lat",null) + query.put("lat",new BasicDBObject('$exists',false)) + def cursor = school_parse_coll.find(query) + def index = 0 + for(def d : cursor){ + index++ + def datas = getSchoolLatAndLng(d.city, d.name, d.address, false) + if(datas){ + for(def ret : datas){ + def addr = d.address + if(d.address.contains(d.city+"市")){ + def cityStr = d.city+"市" + addr = d.address.substring(d.address.indexOf(cityStr)+cityStr.size()) + } + if(ret.name == d.name || (d.address.contains("号") && (ret.address.contains(addr) || addr.contains(ret.address)))){ + def udata = new BasicDBObject() + udata.put("lat", ret.lat.toString().toDouble()) + udata.put("lng", ret.lng.toString().toDouble()) + udata.put("status", 0) + school_parse_coll.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata)) + println index +" "+udata + break; + } + } + } + } + } + + def getSchoolLatAndLng(city, name, address, isFullMatch){ + if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){ + return null; + } + address = address.trim().replace(" ","").replace("?","").replace(" ","") + String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=学校&city=" + city + "&output=json&offset=5&page=1&key=4e9592208336433ee1b7aa4d1d434418&extensions=all" + + List poiList = null; + com.alibaba.fastjson.JSONObject result = null; + try { + HttpRequest request = new HttpRequest(); + Map params = new HashMap(); + String response = request.get(url, params, "UTF-8"); + com.alibaba.fastjson.JSONObject jsonResponse = com.alibaba.fastjson.JSONObject.parseObject(response) + if (jsonResponse != null) { + JSONArray ja = jsonResponse.getJSONArray("pois"); + if(ja != null && ja.size() > 0) { + poiList = new ArrayList<>(); + for(int i=0;i + udate.put(k,v) + } + udate.removeField("_id") + FZ_DIST_COLL.update(new BasicDBObject('_id',d.get('_id')), new BasicDBObject('$set', udate)); + }else{ + FZ_DIST_COLL.save(d) + } + + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }); + } + latch.await() + + } + + def getFzDistricts(def city){ + def q = new BasicDBObject() + q.put("city", city) + def rets = [] + FZ_DIST_COLL.find(q).each{it-> + rets << it + } + return rets + } + def processAlias(alias, def addr_alias){ def newAlias = [] if(alias){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 276c86cec..14a8a6fac 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -29,7 +29,9 @@ class DistrictPriceExportBatch extends BaseBatch{ def charset = "GBK" public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com") + def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580] static main(args) { runBatch(new DistrictPriceExportBatch(), args) } @@ -86,10 +88,21 @@ class DistrictPriceExportBatch extends BaseBatch{ return sb.toString() } + def getFzDistricts(def city){ + def q = new BasicDBObject() + q.put("city", city) + def rets = [] + FZ_DIST_COLL.find(q).each{it-> + rets << it + } + return rets + } + def createDatas(def city){ - def districts = CommonDatautils.findAllDistricts(city, "district", null); - def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); - districts.addAll(removeDists) +// def districts = CommonDatautils.findAllDistricts(city, "district", null); +// def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); +// districts.addAll(removeDists) + def districts = getFzDistricts(city) def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10" def dataStr = DateUtils.add(endDateStr, -30) @@ -124,9 +137,15 @@ class DistrictPriceExportBatch extends BaseBatch{ curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) } } + }else{ + curPrice = specialDistsPriceMap.get(rMap.name) + if(curPrice){ + def randMarkup = nextDouble(0, 1) + curPrice = (int)(curPrice*randMarkup/100 + curPrice) + } } def m = [] - if(curPrice){ + if(curPrice && dbo){ m << dbo.get("_id").toString()//均价ID }else{ m << new ObjectId().toString() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index cad3879ac..a624061d9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -136,7 +136,7 @@ class FzDataResultStat { def regionPriceTrendRun(){ def city = "福州" - def add = -6; + def add = -9; def months = [] while(add<0){ months << DateUtils.addMonth(new Date(), add) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy index b5aa0e611..32c4d1f34 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy @@ -325,8 +325,8 @@ class HouseExportBatch extends BaseBatch{ // int endDate = 20191201 def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyyMMdd") - // def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd"); - def endDate = "20220710" + def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd"); + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); // QueryBuilder room_range = QueryBuilders.rangeQuery("room").from(1).to(5).includeLower(true).includeUpper(false); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index c9a2c19be..7cdcc589a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202207.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202208.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index a13780761..616012302 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2022-07-01" + static def statDate = "2022-08-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser();//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2022年9月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 @@ -284,8 +284,8 @@ class YouDaDataParser { } - def esfParser(){ - def files = ["D:\\数据\\友达\\2022年8月\\查询1.DBF"] + def esfParser(path){ + def files = [path] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/CityStat.groovy similarity index 94% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/CityStat.groovy index 9b523bef2..960b75f38 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/CityStat.groovy @@ -1,4 +1,4 @@ -package com.fangjia.script.gj.test +package com.fangjia.script.gj.commStat import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils @@ -28,11 +28,30 @@ class CityStat { def task = new CityStat() // task.run() // task.run1(); - task.run3(); +// task.run3(); // task.run4() // task.run6() // task.run7() // task.run8() + task.dealPriceStat() + } + + def dealPriceStat(){ + //def citys = ["上海","北京","广州","合肥","南京","深圳","武汉","郑州","重庆","福州","厦门","青岛","太原","长春","苏州","南宁","沈阳","天津","西安","长沙","大连","南昌","东莞","佛山","济南","南通","镇江","宁波","烟台","哈尔滨"] + def citys = ["石家庄","贵阳"] + def rets = [] + rets << ["城市","2022-4月总量"] + def startDate = DateUtils.parseFromTightByPattern('2022-01-01','yyyy-MM-dd') + def endDate = DateUtils.parseFromTightByPattern('2022-02-01','yyyy-MM-dd') + for(def c : citys){ + def ret = [] + ret << c + ret << findDealPriceCount(c, startDate, endDate) + rets << ret + println ret + } + + ExcelUtils.write("D:\\数据\\2021成交数据.xlsx", rets, "月统计") } // def run8(){ @@ -464,6 +483,14 @@ class CityStat { return count; } + def findDealPriceCount(def city, def startDate, def endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt', endDate)) + def count = DEAL_PRICE_COLL.count(q) + return count; + } + def calTradeDistCount(def city, def tradeDate, def districts){ def q = new BasicDBObject() q.put("city", city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DataStatTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/DataStatTest.groovy similarity index 92% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DataStatTest.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/DataStatTest.groovy index 4610f71ee..d2d11ec50 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DataStatTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/DataStatTest.groovy @@ -1,4 +1,4 @@ -package com.fangjia.script.gj.test +package com.fangjia.script.gj.commStat import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils @@ -29,24 +29,24 @@ DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "ave DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43") -def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"] +//def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"] //for(def city : citys){ // //} -ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\新疆数据统计.xlsx",{d-> -// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL) -// if(keys){ -// d['G'] = keys.size() +//ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\新疆数据统计.xlsx",{d-> +//// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL) +//// if(keys){ +//// d['G'] = keys.size() +//// } +// +//// def rets = findHousesByCity(d['A'], "sell", 20210701) +// if(d['A']){ +// def count = findHouseCount(d['A']) +// d['H'] = count // } - -// def rets = findHousesByCity(d['A'], "sell", 20210701) - if(d['A']){ - def count = findHouseCount(d['A']) - d['H'] = count - } - -}) +// +//}) def findHouseCount(city){ @@ -306,3 +306,23 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){ // rows << [c, districts.size(), alias_count, addr_alias_count] //} //ExcelUtils.write("D:\\数据\\江苏\\新疆数据统计.xlsx", rows, "结果") + +//区域级统计 +def city = "合肥" +def regions = CommonDatautils.getCityRegions(city) +def rows = [] +for(def region : regions){ + def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"]) + def alias_count = 0 + def addr_alias_count = 0 + for(def r : districts){ + if(r.alias && r.alias.size()>0){ + alias_count ++ + } + if(r.addr_alias && r.addr_alias.size()>0){ + addr_alias_count++ + } + } + rows << [region, districts.size(), alias_count, addr_alias_count] +} +ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果") \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 6ff2d9586..9f3670a44 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -110,6 +110,7 @@ class FangjiaClientUtils { public static def saveToEsPaimai(def data){ String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data) HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/paiMai", jsonStr) +// HttpUtils.postForJson("http://localhost:8084/FangJiaDataService/paiMai", jsonStr) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy new file mode 100644 index 000000000..2e9a536ef --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy @@ -0,0 +1,135 @@ +package com.fangjia.script.gj.dongguanbank + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import org.apache.lucene.queryparser.classic.QueryParser + +class DongGuanBankTest { + static main(args) { + def task = new DongGuanBankTest() + def path = "D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx" +// task.run(path) + task.addPrice(path) + } + + def run(path){ + ExcelUtils.multiThreadingModify(path,{d-> + def city = d['A'].replace("市","") + if(city){ + def region = RegionData.formatRegion(city, d['B']) + def results = OpenApiClient.addressMatch(city, d['F'], "property", false, region, d['C']) + if(results && results.precise>=0){ + d['H'] = results.city + d['I'] = results.district + d['J'] = results.name + d['Q'] = results.propertytype + } + } + + }) + } + + + def addPrice(path){ + ExcelUtils.multiThreadingModify("D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx", { d -> + if(d['H']){ + def dist = CommonDatautils.findDistrict(d['H'], d['I'], d['J'], "district") + def priceObj = CommonDatautils.queryDistrictPrice(d['H'], d['I'], d['J'], DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd")) + if(priceObj){ + d['M'] = priceObj.avg_price + d['N'] = (priceObj.avg_price * d['G']).toInteger() + d['O'] = ((int)(priceObj.avg_price/100))*100 + if(dist){ + d['R'] = dist.estate_type2 + } + if(priceObj.value_type){ + if(priceObj.value_type == 1){ + d['S'] = "指导价" + }else if(priceObj.value_type == 2){ + d['S'] = "自身样本" + }else if(priceObj.value_type == 3){ + d['S'] = "第三方" + }else if(priceObj.value_type == 4){ + d['S'] = "交易数据" + }else if(priceObj.value_type == 5){ + d['S'] = "周边楼盘" + }else if(priceObj.value_type == 6){ + d['S'] = "推导" + }else if(priceObj.value_type == 9){ + d['S'] = "神经网络" + }else if(priceObj.value_type == 10){ + d['S'] = "无方式" + } + } + } + + if(!d['P']){ + def alias = [] + if(!dist.alias){ + alias = [] + }else{ + alias = dist.alias + } + alias << dist.name + def map = findMaxAvgPrice(dist.city, dist.region, alias) + if(map){ + d['P'] = map.max + d['Q'] = map.min + } + } + + + } + }) + } + + + def findMaxAvgPrice(def city, def region, def alias){ + def set = null + try{ + def aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= 20210101 and region='${region}' and district_name in ('${aliasStr}')" + + set = new HashSet() + ElasticUtils.executeSqlQuery(sql,{data-> + set << data.avg_price + }) + }catch(Exception e){ + e.printStackTrace() + } + + if(set && set.size()>=3){ + def map = [:] + set = set.sort() + map.min = set[0] + map.max = set[set.size()-1] + return map + } + return null + } + + + + /** + * 转义字符串中的特殊字符 + * 仅过滤Lucene regexps, 其他的勿用 + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + + def findMinPrice(city){ + def avg_price = null + ElasticUtils.executeSqlQuery("select min(avg_price) as avg_price from seawater where p_date >= 20210901 and city='${city}'", { + avg_price = it.avg_price + }) + return avg_price + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index 2560bb842..eac332041 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -86,7 +86,7 @@ class DistrictIndexBatch extends BaseBatch{ city_avg_total_price = 531.2615913 } if(city_avg_total_price){ - calcute(city, dists, code, city_avg_total_price) + // calcute(city, dists, code, city_avg_total_price) calcuteIndex(city, dists, code, city_avg_total_price) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 6ce8eb214..42489aafe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -104,9 +104,23 @@ public class BeiKeCrawlByChrome { def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) for(def m : citys){ - //def detailUrls = getPageUrls(driver, cityUrl) - def detailUrls = getPageDeail(driver, m.city, m.url) - println m.city +" 详情链接:"+detailUrls.size() + driver.get(m.url+"/chengjiao/") + Thread.sleep(10000) + def doc = Jsoup.parse(driver.getPageSource()); + def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) + if(regions){ + for(def r : regions){ + def regionText = r.text() + def href = m.url+ r.attr("href") + + println "开始抓取【${m.city}】【${regionText}】【${href}】" + def detailUrls = getPageDeail(driver, m.city, href, regionText) + println m.url +" 详情链接:"+detailUrls.size() + } + } + +// def detailUrls = getPageDeail(driver, m.city, m.url) +// println m.city +" 详情链接:"+detailUrls.size() // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) } } @@ -142,12 +156,12 @@ public class BeiKeCrawlByChrome { } - def getPageDeail(driver, city, cityUrl){ + def getPageDeail(driver, city, cityUrl, region){ def stop = false def page = 1 def detailUrls = [] while(!stop){ - def url = cityUrl + "/chengjiao/pg${page}/" + def url = cityUrl + "pg${page}/" try{ driver.get(url) }catch(Exception e){ @@ -164,7 +178,7 @@ public class BeiKeCrawlByChrome { def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl - parse(city, e) + parse(city, e, region) }else{ stop = true; break; @@ -182,7 +196,7 @@ public class BeiKeCrawlByChrome { return detailUrls } - def parse(city,e){ + def parse(city,e, region){ try { def title = e.select(".title a").text() if (title) { @@ -190,7 +204,7 @@ public class BeiKeCrawlByChrome { def arrs = title.split(" ") detail.city = city detail.district_name = arrs[0] - detail.region = findRegionByThirdPary(detail.city, detail.district_name) + detail.region = region//findRegionByThirdPary(detail.city, detail.district_name) if(detail.region){ detail.region = RegionData.formatRegion(city, detail.region) } @@ -381,7 +395,7 @@ public class BeiKeCrawlByChrome { cityMap.city = e.text() cityMap.url = "https:"+href list << cityMap - //cityMap.put(e.text(), "https:"+href) +// cityMap.put(e.text(), "https:"+href) // } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy index 2bba76ea4..447a05bb2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy @@ -7,6 +7,8 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import org.apache.commons.lang.StringUtils +import org.apache.commons.lang.math.NumberUtils +import java.util.Map.Entry; import java.util.regex.Pattern @@ -175,22 +177,23 @@ class ExportPaimai { def run(){ - def city = "苏州" - def region = "姑苏" + def city = "合肥" + def region = null def datas = findPaimaiData(city,region) println "总量:"+datas.size() def rows = [] def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层", "拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价", - "变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"] + "变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数", "折价率","映射类型","拍卖类型", + "映射资产类型","映射拍卖状态","URL"] rows << header -// def set = new HashSet() + def set = new HashSet() def index = 0 for(def d : datas){ -// if(!set.contains(d.title)){ + if(!set.contains(d.title)){ index++ println index -// set << d.title + set << d.title def row = [] row << d.title.replace(",",",") row << d.province @@ -226,14 +229,19 @@ class ExportPaimai { row << d.lookCount row << d.callCount row << d.delayCount - row << d.url + row << d.rateDiscount//折价率 + row << d.mappingEstateType;//映射类型 + row << d.zcType;//拍卖类型:资产拍卖 司法拍卖 + row << d.mappingAgencyName//映射资产类型 + row << d.mappingStatus//映射拍卖状态 + row << d.url rows << row -// }else{ -// println d.title -// } + }else{ + println d.title +" "+d.url + } } - ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\京东拍卖(${city}-姑苏-撤回中止).xlsx",rows,"数据") + ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\合肥法拍.xlsx",rows,"数据") // batchCreateFile(rows, "D:\\数据\\重庆农商行\\法拍\\淘宝拍卖(新).csv",null,"GBK") println "生成完成........" @@ -452,12 +460,12 @@ class ExportPaimai { if(region){ pmquery.put("region",region) } - pmquery.put("status", new BasicDBObject('$in', ["已撤回","已暂缓"])) +// pmquery.put("status", new BasicDBObject('$in', ["已撤回","已暂缓"])) // pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01"))) // Pattern pattern = Pattern.compile("rmfysszc.gov.cn", Pattern.MULTILINE); - Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE); +// Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE); // Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE); // pmquery.put("url", new BasicDBObject('$regex', pattern)) def dbcursor = paimai43_COLL.find(pmquery).batchSize(2000) @@ -465,6 +473,28 @@ class ExportPaimai { for(def db : dbcursor){ rets << db } + +// Collections.sort(rets, new Comparator() { +// @Override +// int compare(BasicDBObject o1, BasicDBObject o2) { +// if(o1.finishDate && o2.finishDate){ +// return -o1.finishDate.compareTo(o2.finishDate) +// }else{ +// return 0 +// } +// +// } +// }) + rets.sort {a,b-> + if(a.finishDate && b.finishDate){ + def v1 = a.finishDate + def v2 = b.finishDate + return v1.compareTo(v2) + }else{ + return 0 + } + } + rets = rets.reverse() return rets } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 50470f713..c42cfa5ba 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -34,9 +34,9 @@ public class LianjiaCrawlByChrome { static main(args) { def task = new LianjiaCrawlByChrome(); -// task.run() + task.run() // task.run1() - task.addDealRegion(); +// task.addDealRegion(); task.standardRegion() // task.updadteDealPriceField() } @@ -129,7 +129,7 @@ public class LianjiaCrawlByChrome { def standardRegion(){ def q = new BasicDBObject() - q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd"))) + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd"))) q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) ) def dbcursor = DEAL_PRICE_COLL.find(q) for(def d : dbcursor){ @@ -152,8 +152,9 @@ public class LianjiaCrawlByChrome { def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") lastMonth = lastMonth +"-01" def q = new BasicDBObject() +// q.put("city", "福州") q.put("region", null) - q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-05-01","yyyy-MM-dd"))) + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd"))) // q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) def dbcursor = DEAL_PRICE_COLL.find(q) def keys = [] @@ -187,8 +188,19 @@ public class LianjiaCrawlByChrome { DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) cityMap.each{city, cityUrl-> //def detailUrls = getPageUrls(driver, cityUrl) - def detailUrls = getPageDeail(driver, city, cityUrl) - println cityUrl +" 详情链接:"+detailUrls.size() + driver.get(cityUrl+"chengjiao/") + def doc = Jsoup.parse(driver.getPageSource()); + def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) + if(regions){ + for(def r : regions){ + def regionText = r.text() + def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href") + + println "开始抓取【${city}】【${regionText}】【${href}】" + def detailUrls = getPageDeail(driver, city, href, regionText) + println cityUrl +" 详情链接:"+detailUrls.size() + } + } // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) } } @@ -224,12 +236,12 @@ public class LianjiaCrawlByChrome { } - def getPageDeail(driver, city, cityUrl){ + def getPageDeail(driver, city, cityUrl, regionText){ def stop = false def page = 1 def detailUrls = [] while(!stop){ - def url = cityUrl + "chengjiao/pg${page}/" + def url = cityUrl + "pg${page}/" driver.get(url) def doc = Jsoup.parse(driver.getPageSource()); def elements = JQueryUtils.select(".listContent li", doc) @@ -238,10 +250,10 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-06-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-07-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl - parse(city, e) + parse(city, e, regionText) }else{ stop = true; break; @@ -259,7 +271,7 @@ public class LianjiaCrawlByChrome { return detailUrls } - def parse(city,e){ + def parse(city,e, regionText){ try { def title = e.select(".title a").text() if (title) { @@ -267,7 +279,11 @@ public class LianjiaCrawlByChrome { def arrs = title.split(" ") detail.city = city detail.district_name = arrs[0] - detail.region = findRegionByThirdPary(detail.city, detail.district_name) + if(regionText){ + detail.region = regionText + }else{ + detail.region = findRegionByThirdPary(detail.city, detail.district_name) + } if(detail.region){ detail.region = RegionData.formatRegion(city, detail.region) } @@ -275,50 +291,61 @@ public class LianjiaCrawlByChrome { def roomStr = arrs[1] if (roomStr) { if (roomStr.indexOf("室") > 0) { - detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() + def room = roomStr.substring(0, roomStr.indexOf("室")) + if(room && room.isNumber()){ + detail.room = room.toInteger() + } + } if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { - detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() + def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + if(hall && hall.isNumber()){ + detail.hall = hall.toInteger() + } } } - def areaStr = arrs[2].replace("平米", "") - if (areaStr) { - detail.area = areaStr.toDouble() - } - - def price = e.select(".totalPrice .number").text() - if (price && price.isNumber()) { - detail.total_price = price.toInteger() * 10000 - } - - if (detail.total_price && detail.area) { - detail.avg_price = (int) (detail.total_price / detail.area) - } - def tradeStr = e.select(".dealDate").text() - if (tradeStr) { - detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") - } - detail.source = "链家在线" - detail.url = e.select(".title a").attr("href") - def dbo = findTrade(detail) - if (!dbo) { - def houseInfo = e.select(".houseInfo").text() - def houseInfos = houseInfo.split("\\|") - detail.direction = houseInfos[0] - detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() - if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { - floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() + if("车位"!=roomStr){ + if(arrs && arrs.size()>=3){ + def areaStr = arrs[2].replace("平米", "") + if (areaStr) { + detail.area = areaStr.toDouble() } } - detail.create_date = new Date() - detail.m_date = new Date() - println detail - DEAL_PRICE_COLL.save(detail) + def price = e.select(".totalPrice .number").text() + if (price && price.isNumber()) { + detail.total_price = price.toInteger() * 10000 + } + + if (detail.total_price && detail.area) { + detail.avg_price = (int) (detail.total_price / detail.area) + } + def tradeStr = e.select(".dealDate").text() + if (tradeStr) { + detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") + } + detail.source = "链家在线" + detail.url = e.select(".title a").attr("href") + def dbo = findTrade(detail) + if (!dbo) { + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + def floorStr = e.select(".positionInfo").text() + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } } } }catch(Exception ex){ @@ -442,13 +469,13 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] - def citys= ["福州"] + def citys= ["苏州"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") -// if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia") && citys.contains(e.text())){ cityMap.put(e.text(), href) -// } + } } } return cityMap diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index 4f5343c14..9bdcef9c4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.AMQP +import com.rabbitmq.client.impl.AMQImpl import java.text.SimpleDateFormat import java.util.concurrent.CountDownLatch @@ -29,18 +30,19 @@ class PaimaiDataSend { def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) static main(args) { def task = new PaimaiDataSend() +// task.jdRunByMongo() // task.jdRun() -// task.aliZcRun() - task.aliZcRun2() +//// task.aliZcRun() +// task.aliZcRun2() // task.aliFpRun() // task.rmfysszcRun() // task.checkUrl() -// task.paimaiSendToEs() + task.paimaiSendToEs() } @@ -64,7 +66,9 @@ class PaimaiDataSend { def rmfysszcRun(){ def index = 0 def q = new BasicDBObject() -// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/76602.shtml") +// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"])) + q.put("province","广东省") +// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml") def dbcursor = rmfysszc_coll.find(q).batchSize(3000) for(def d : dbcursor){ index++ @@ -95,11 +99,22 @@ class PaimaiDataSend { } } - def jdRun(){ + def jdRunByMongo(){ + def pm_q = new BasicDBObject() + pm_q.put("city","宿迁") + pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01"))) + pm_q.put("mappingStage", null) + def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000) + println dbcursor.count() + for(def d : dbcursor){ + sendJd(d.url) + } + } + + def sendJd(url){ def index = 0 def q = new BasicDBObject() - q.put("city","宿迁市") -// q.put("url","https://paimai.jd.com/108245329") + q.put("url", url) def dbcursor = jd_coll.find(q).batchSize(3000) for(def d : dbcursor){ index++ @@ -110,6 +125,7 @@ class PaimaiDataSend { JSON result = new JSONObject() result.put("content", d.desc_content) + result.put("region", d.region) def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) result.put("bid_content", bid_content.toString()) @@ -131,20 +147,96 @@ class PaimaiDataSend { jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata)) } } + def jdRun(){ + def index = 0 + def q = new BasicDBObject() +// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"])) + q.put("province","广东") +// q.put("parser_status", new BasicDBObject('$ne',3)) +// q.put("url","https://paimai.jd.com/115669796") + def dbcursor = jd_coll.find(q).batchSize(3000) + def rets = [] + for(def d : dbcursor){ + index++ + println "jd:"+index + JSON json = new JSONObject() + json.put("project", "paimai_jd") + json.put("url", d.url) + + JSON result = new JSONObject() + result.put("content", d.desc_content) + result.put("region", d.region) + + def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) + result.put("bid_content", bid_content.toString()) + + result.put("sub_count", d.sub_count) + def data_json = net.sf.json.JSONObject.fromObject(d.json) + result.put("data_json", data_json) + result.put("content_json", d.content) + result.put("item_notice_content", d.item_notice_content) + result.put("notice_detail_content", d.notice_detail_content.content) + json.put("result", result) + + rets << json + if(rets.size()==10000){ + println "共:"+rets.size() + sendJD(rets) + rets.clear() + } + } + + if(rets.size()>0){ + println "共:"+rets.size() + sendJD(rets) + } + } + + def sendJD(def rets){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i0){ + println "共:"+rets.size() + send(rets) + } + } + def send(def rets){ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(rets.size()) for(def i=0;i0){ + println "共:"+rets.size() + send(rets) + } + +// println "共:"+rets.size() +// +// def totalCount = new AtomicLong() +// final CountDownLatch latch = new CountDownLatch(rets.size()) +// for(def i=0;i +// def city = RegionData.formatCity(d['E']) +// if(city && !d['J']){ +// def region = RegionData.formatRegion(city, d['G']) +// d['I'] = city +// d['J'] = region +// } +//}) + + +//ExcelUtils.modify("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx",{d-> +// def city = RegionData.formatCity(d['D']) +// if(!city){ +// city = RegionData.formatCity(d['F']) +// } +// if(city && !d['H']){ +// def region = RegionData.formatRegion(city, d['B']) +// d['G'] = city +// d['H'] = region +// } +//}) + +//def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] +//def datas = ExcelUtils.read("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx") + +//def fjwCitys = [] +//for(def d : datas){ +// fjwCitys << d['G'] +//} +//def rows = [] +//for(def d : datas){ +// if(citys.contains(d['G'])){ +// rows << d +// } +//} +//ExcelUtils.write("D:\\数据\\建行\\二期\\统计局行政区划与房价网区域比对V2.xlsx", rows, "区域") + +//for(def c : citys){ +// if(!fjwCitys.contains(c)){ +// println c +// } +//} + +// +//ExcelUtils.write("D:\\数据\\建行\\二期\\(特殊城市)统计局行政区划与房价网区域比对.xlsx", rows, "区域") +// + +//def map = [:] +//def datas1 = ExcelUtils.read("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx") +//for(def row : datas1){ +// if(row['J']){ +// def codeMap = [:] +// codeMap.prov = row['B'] +// codeMap.city = row['F'] +// codeMap.region = row['H'] +// map.put(row['I']+"#"+row['J'], codeMap) +// } +//} +//def datas2 = ExcelUtils.read("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx") +//for(def row : datas2){ +// if(row['H']){ +// def codeMap = [:] +// codeMap.prov = row['E'] +// codeMap.city = row['C'] +// codeMap.region = row['A'] +// map.put(row['G']+"#"+row['H'], codeMap) +// } +//} + +//ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d-> +// def key = d['B'] +"#"+d['C'] +// def codeMap = map.get(key) +// if(codeMap && !d['D']){ +// d['D'] = codeMap.prov +// d['E'] = codeMap.city +// d['F'] = codeMap.region +// } +//}) + +ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d-> + if(!d['D']){ + def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN) + if(infos && infos.size()==1){ + def info = infos[0] + d['D'] = info.province_code + d['E'] = info.area_code + d['F'] = info.street_code + } + } +}) + + +def findTownshipInfo(city, region, coll){ + def q = new BasicDBObject() + q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE)) + q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE)) + + def dbcursor = coll.find(q) + def rows = [] + for(def d : dbcursor){ +// if(d.street_name.contains(region)){ + rows << d +// } + } + return rows +} + + + +//def cityMap = CommonDatautils.findAllCityProvince() +//def rows = [] +//rows << ["省", "市", "区域"] +//cityMap.each{c,p-> +// def regions = CommonDatautils.getCityRegions(c) +// for(def r : regions){ +// def row = [] +// row << p +// row << c +// row << r +// rows << row +// } +//} +//ExcelUtils.write("D:\\数据\\亲邻\\房价网省市区.xlsx", rows, "区域") + + + + +//def q = new BasicDBObject() +//q.put("root", "anjuke.com") +//def dbcursor = coll.find(q) +//def map = [:] +//for(def d : dbcursor){ +// map.put(d.city, d.city_short) +//} +//println map.size() +//println map + + +//def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164") +//DBUtils.seaweed.save(dist) //DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId("62ce233a3705890001dfa164")), new BasicDBObject('$set', dist)) //SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); //Date date = new Date(1617211269868); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/TestSendSchoolData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/TestSendSchoolData.groovy new file mode 100644 index 000000000..e81f4cb67 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/TestSendSchoolData.groovy @@ -0,0 +1,142 @@ +package com.fangjia.script.gj.test + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +DBCollection school_coll = MongoUtils.getCollectionByName("crawl_tmp", "school_ruyile_web", "192.168.0.235", 27018) + +DBCollection sxue_school_coll = MongoUtils.getCollectionByName("crawl_tmp", "school_5sxue_web", "192.168.0.235", 27018) + +DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018) + +ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) +//Thread.sleep(60*60*1000) +//while (true){ + def q = new BasicDBObject() +// q.put("province",new BasicDBObject('$in', ["江苏","江苏省"])) + q.put("status",new BasicDBObject('$ne', 1)) + def dbcursor = school_coll.find(q).batchSize(5000) + def rets = [] + for(def d : dbcursor){ + JSON json = new JSONObject() + json.put("project", d.project) + json.put("url", d.url) + + JSON result = new JSONObject() + result.put("content", d.content) + def jsonStr = d.map_info + + jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1) + if(jsonStr.contains("\n")){ + jsonStr = jsonStr.replaceAll('\\n','') + } + result.put("data_json", net.sf.json.JSONObject.fromObject(jsonStr)) + + + result.put("type", d.type) + json.put("result", result) + + + rets << json + if(rets.size()==5000){ + println "共:"+rets.size() + send(rets, fixedThreadPool, school_coll) + rets.clear() + } + } + + if(rets.size()>0){ + println "共:"+rets.size() + send(rets, fixedThreadPool, school_coll) + rets.clear() + } +//} + + + +def send(def rets, def fixedThreadPool, def school_coll){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i0){ +// println "共:"+rets.size() +// send(rets, fixedThreadPool, school_coll) +// rets.clear() +//} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy new file mode 100644 index 000000000..a0c3ec24f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy @@ -0,0 +1,39 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.EsUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ElasticUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 补充学校经纬度 + * @author shenjian + * @since 2022/9/8 + */ +class FillLngLat { + + static ExecutorService executorService = Executors.newFixedThreadPool(4) + static void main(String[] args) { + ElasticUtils.executeSqlQuery("select * from dws_poi where location is not null", { dataT -> + final def data = dataT + executorService.submit(new Runnable() { + @Override + void run() { + println(data) + def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address) + if (lngLat != null && lngLat.size() == 2 ) { + JSONObject jsonObject = new JSONObject() + jsonObject.put("lon", Double.valueOf(lngLat['lng'])) + jsonObject.put("lat", Double.valueOf(lngLat['lat'])) + EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id)) + } + } + }) + + }) + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index f3c36ebd8..ca74c0184 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -10,6 +10,7 @@ import com.fangjia.script.temp.shenjian.util.HttpUtils import com.fangjia.script.temp.shenjian.util.SeaweedUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBList import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject @@ -27,15 +28,15 @@ import java.util.concurrent.Executors @BatchOwner(Watcher.ShenJian) class SeaweedAroundPoi extends BaseBatch { - static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.235", 27018) static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi" - static ExecutorService executorService = Executors.newFixedThreadPool(30) + //static ExecutorService executorService = Executors.newFixedThreadPool(2) static void main(String[] args) { def types = listAllType() //dealSpecialSeaweedAround(types) dealPoi(new BasicDBObject(), types) - executorService.shutdown() + //executorService.shutdown() } /** @@ -64,33 +65,40 @@ class SeaweedAroundPoi extends BaseBatch { } static dealPoi(DBObject query, def types) { - query.put("status", 1) - conn136.find(query).batchSize(5000).each {objTemp -> + query.put("canyin_status", 0) + conn136.find(query).batchSize(5000).limit(20000).noCursorTimeout(true).each {objTemp -> final def obj = objTemp - executorService.submit(new Runnable() { - @Override - void run() { +// executorService.submit(new Runnable() { +// @Override +// void run() { def city = obj.city - def lng = obj.lng - def lat = obj.lat - List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") - if (CollectionUtils.isNotEmpty(pois)) { - String jsonStr = JSONObject.toJSONString(pois) - HttpUtils.postForJson(URL, jsonStr) - conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) - } else { - conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + if (obj.lng) { + //def lng = SeaweedUtils.formatLatLng(obj.lng) + //def lat = SeaweedUtils.formatLatLng(obj.lat) + // {'lon': 121.810877, 'lat': 31.737642} + def lng = 121.810877 + def lat = 31.737642 + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("canyin_status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("canyin_status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) } - println("已处理小区:" + obj) } - }) - } +// }) +// } } static getNewSeaweed() { DBObject query = new BasicDBObject() query.put("cat", "district") query.put("visible", 0) + //query.put("city", new BasicDBObject('$in', ["连云港", "南通", "苏州", "无锡", "盐城", "扬州"])) + //query.put("city", "连云港") //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) DBObject filter = new BasicDBObject() @@ -163,17 +171,18 @@ class SeaweedAroundPoi extends BaseBatch { public static final Map KEYWORDS_MAP = new HashMap() { { - put("餐饮", "05"); +// put("餐饮", "05"); // put("地铁", "150500"); // put("公交", "1507"); // put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); // put("写字楼", "120201"); // put("学校", "141200|141201|141202|141203|141204"); // put("银行", "1601|1603"); -// put("大型超市", "0604"); + put("大型超市", "0604"); // put("商场", "0601"); // put("餐饮", "0501|0502|0503|0504"); // put("旅游景点", "110000"); + put("公园广场", "110100|110101|110102|110103|110104|110105") // put("图书馆", "140500"); // put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107"); // put("丧葬设施", "071900|071901|071902|071903") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiFromExcel.groovy new file mode 100644 index 000000000..66abaacf9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiFromExcel.groovy @@ -0,0 +1,84 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBList +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取 + * + * @author shenjian + * @since 2021/5/24 + */ +class SeaweedAroundPoiFromExcel { + + static void main(String[] args) { + def types = listAllType() + dealPoi(new BasicDBObject(), types) + } + + static dealPoi(DBObject query, def types) { + com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区\\东莞银行房价数据测试20220916-v3.xlsx", { List item -> + def city = item['H'] + def lng = item['AQ'] + def lat = item['AR'] + if (lng) { + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi", jsonStr) + } + Thread.sleep(1000) + print(lng + " " + lat) + } + }); + }; + + + + public static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + }; + + public static final Map KEYWORDS_MAP = new HashMap() { + { +// put("餐饮", "05"); +// put("地铁", "150500"); +// put("公交", "1507"); +// put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); +// put("写字楼", "120201"); +// put("学校", "141200|141201|141202|141203|141204"); +// put("银行", "1601|1603"); + put("大型超市", "0604"); +// put("商场", "0601"); +// put("餐饮", "0501|0502|0503|0504"); +// put("旅游景点", "110000"); + put("公园广场", "110100|110101|110102|110103|110104|110105") +// put("图书馆", "140500"); +// put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107"); +// put("丧葬设施", "071900|071901|071902|071903") + + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index f08b96c89..50ac505aa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -146,11 +146,15 @@ class GaoDeUtils { while (result.size() < count) { params.page = "${page}".toString() page++ - params.key = this.getGaodeKey("v5/place/around") - JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params); + //params.key = this.getGaodeKey("v3/place/around") + params.key = "25b2d459fd133e22f8abcebeac972692" + params.offset = "100" + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); if (json == null) { - params.key = this.getGaodeKey("v5/place/around") - json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params); + //params.key = this.getGaodeKey("v3/place/around") + params.key = "25b2d459fd133e22f8abcebeac972692" + params.offset = "100" + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); } if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() @@ -258,7 +262,7 @@ class GaoDeUtils { } if (json && json.info && json.info == "OK") { try { - List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoiDetail.class) return ret } catch(Exception ex) { ex.printStackTrace() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 0b7d904a5..1ae32a425 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "365淘房"// - def citys = findCrawlCity(source, 202206); + def source = "我爱我家"// + def citys = findCrawlCity(source, 202207); for(def city : citys){ - def count1 = findCount(source, city, 202206); - def count2 = findCount(source, city, 202207); + def count1 = findCount(source, city, 202207); + def count2 = findCount(source, city, 202208); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index a682afead..886acc0b1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -16,15 +16,12 @@ class SynchDistrictPrice { def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); - //"fang.com":"搜房", // def sourceMap = ["58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] - //"fang.com":"搜房", def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] - //,"城市房产":"城市房产" - //"365淘房":"365淘房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房", - def esSources = ["C21":"C21","365淘房":"365淘房"]//"吉屋":"吉屋","365淘房":"365淘房" + //"365淘房":"365淘房","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","吉屋":"吉屋" + def esSources = ["搜房":"搜房","我爱我家":"我爱我家","365淘房":"365淘房"] static main(args) { def task = new SynchDistrictPrice() // task.run() @@ -32,13 +29,13 @@ class SynchDistrictPrice { } def run1(){ + + def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") def citys = CommonUtils.findAllCity() -// citys.remove("上海") -// citys.remove("广州") esSources.each{esSource, mgSource-> for(def city : citys){ - def rets = findAllEsData(city, esSource, mgSource) + def rets = findAllEsData(city, esSource, mgSource, mdate) println city +" "+rets.size() synchData(city, rets, mgSource) } @@ -46,10 +43,10 @@ class SynchDistrictPrice { } - def findAllEsData(city,esSource, mgSource){ + def findAllEsData(city,esSource, mgSource, mdate){ def names = new HashSet() def rets = [] - ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='2022-07-01' order by m_date", {d-> + ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d-> def s = d.city +"#"+d.region+"#"+d.name if(!names.contains(s)){ names << s diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index a5baaf550..149149fd0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202206); - def count2 = findTotalCount(source, 202207); + def count1 = findTotalCount(source, 202207); + def count2 = findTotalCount(source, 202208); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 89e0e9483..3efe71b67 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -44,6 +44,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ void process(Object args) { println SHORT_DATE def citys = CommonDatautils.findAllCity(); + def weekend = DateUtils.getFirstDayOfWeek(new Date()); //def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15")); def cityIndex = 0; diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java index 8192b359c..14922257d 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java @@ -50,6 +50,21 @@ public class AmapPoiDetail { private BizExtDTO bizExt; @com.alibaba.fastjson.annotation.JSONField(name = "photos") private List photos; + @com.alibaba.fastjson.annotation.JSONField(name = "business") + private BusinessDTO business; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BusinessDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "business_area") + private String businessArea; + @com.alibaba.fastjson.annotation.JSONField(name = "keytag") + private String keytag; + @com.alibaba.fastjson.annotation.JSONField(name = "opentime_today") + private String opentimeToday; + @com.alibaba.fastjson.annotation.JSONField(name = "opentime_week") + private String opentimeWeek; + } @lombok.NoArgsConstructor @lombok.Data @@ -64,7 +79,7 @@ public class AmapPoiDetail { @lombok.Data public static class PhotosDTO { @com.alibaba.fastjson.annotation.JSONField(name = "title") - private List title; + private String title; @com.alibaba.fastjson.annotation.JSONField(name = "url") private String url; }