From 7579bbfd74f6842fde7cf2d566d0dbe544939c60 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 2 Aug 2022 09:53:13 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E6=97=A5=E5=BF=97=EF=BC=8C=E6=B5=8B=E8=AF=95=E4=BD=BF=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/weixin/WeiXinEvaluationController.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index 587b3888d..cf16573b2 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -670,6 +670,7 @@ public class WeiXinEvaluationController { model.setUserTrades(userTrade); if(fjtUser!=null && fjtUser.getRole()==17 && district.getCity().equals("上海")){ + System.out.println("渤海用户:"+fjtUser.getEmail()); Map retMap = getLoanInfo(district, evaluation.getHouseArea()); if(retMap!=null && retMap.size()>0){ model.setLoanInfoMap(retMap); @@ -1245,6 +1246,9 @@ public class WeiXinEvaluationController { } } Integer yearAge = getBuildYear(district.getFinishBuildingDate()); + if(pbObj!=null){ + System.out.println("渤海银行:"+district.getName() +" 年限:"+yearAge+" Level:"+pbObj.getLevel()); + } Map retsMap = new HashMap(); if(pbObj!=null && pbObj.getLevel()!=null && pbObj.getLevel()==1 && yearAge!=null && yearAge<=25 && area>=60){ retsMap.put("type", "A"); @@ -1262,6 +1266,7 @@ public class WeiXinEvaluationController { retsMap.put("type", "D"); retsMap.put("loan_ratio", 5); } + return retsMap; } From c2418c0a5fb90cbdd6ecf6d796246a6d97ac91ab Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 2 Aug 2022 16:36:37 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E6=9D=BF=E5=9D=97=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/weixin/WeiXinEvaluationController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index cf16573b2..82bf9b1ce 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -1247,7 +1247,7 @@ public class WeiXinEvaluationController { } Integer yearAge = getBuildYear(district.getFinishBuildingDate()); if(pbObj!=null){ - System.out.println("渤海银行:"+district.getName() +" 年限:"+yearAge+" Level:"+pbObj.getLevel()); + System.out.println("渤海银行:"+district.getName() +" 年限:"+yearAge+" Level:"+pbObj.getLevel()+" 所属板块:"+pbObj.getName()); } Map retsMap = new HashMap(); if(pbObj!=null && pbObj.getLevel()!=null && pbObj.getLevel()==1 && yearAge!=null && yearAge<=25 && area>=60){ From b0315e9a36db09ee7a07c1eaa10495cf417022cf Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 5 Aug 2022 17:18:22 +0800 Subject: [PATCH 3/3] update --- .../ccbevaluation/BigdataDealBatch.groovy | 76 ++-- .../DealAndThirdPriceDataProcess.groovy | 4 +- .../script/ccbevaluation/LoadGuide.groovy | 10 +- .../fangjia/script/ccbevaluation/Test.groovy | 77 ++-- .../groovy/com/fangjia/script/gj/Test.groovy | 70 ++-- .../script/gj/crawl/AnjukeDistSearch.groovy | 106 +++++- .../script/gj/crawl/AnjukeHouseCrawl.groovy | 8 +- .../script/gj/paimai/ExportPaimai.groovy | 329 ++++++++++++++++-- .../script/gj/paimai/PaimaiDataSend.groovy | 320 +++++++++++++++-- .../script/gj/test/DataStatTest.groovy | 164 +++++---- .../script/jiangshu/JsBlankTest.groovy | 31 ++ .../TestCrawlTotalCount.groovy | 4 +- 12 files changed, 972 insertions(+), 227 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangshu/JsBlankTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 7538e4786..62c7e0981 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -52,47 +52,47 @@ class BigdataDealBatch extends BaseBatch{ @Override public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) - def citys = CcbCommonUtils.citys; + def citys = ['南京', '无锡', '徐州', '常州', '苏州', '南通', '连云港', '淮安', '盐城', '扬州', '镇江', '泰州', '宿迁']//CcbCommonUtils.citys; // def ccbCitys = CcbCommonUtils.citys; -// for(def city : citys){ -//// if(!ccbCitys.contains(city)){ -// addDealPriceMatch(city); -//// } -// } - def rows = [] - def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def ppeDists = [:] - - def statMap = [:]//getDistrictStatByCity(city, weekend) - if(city=="上海"){ - def youDataTrade = getYouDaData(statMap) - def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") - rows.addAll(youdaDatas) - println city +" "+youdaDatas.size() - } - - - def trades = getCityData(city, statMap, ppeDists) - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") - rows.addAll(rets) - println city +" 成交共:"+rets.size() - - def cityDatas = [] - cityDatas << headers - cityDatas.addAll(rows) +// if(!ccbCitys.contains(city)){ + addDealPriceMatch(city); +// } } - - def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" - } - - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); - println "txt 生成" +// def rows = [] +// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// for(def city : citys){ +// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) +// def ppeDists = [:] +// +// def statMap = [:]//getDistrictStatByCity(city, weekend) +// if(city=="上海"){ +// def youDataTrade = getYouDaData(statMap) +// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") +// rows.addAll(youdaDatas) +// println city +" "+youdaDatas.size() +// } +// +// +// def trades = getCityData(city, statMap, ppeDists) +// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") +// rows.addAll(rets) +// println city +" 成交共:"+rets.size() +// +// def cityDatas = [] +// cityDatas << headers +// cityDatas.addAll(rows) +// } +// +// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" +// } +// +// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); +// println "txt 生成" } @@ -571,7 +571,7 @@ class BigdataDealBatch extends BaseBatch{ def addDealPriceMatch(city){ // def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) - def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20220101", "yyyyMMdd")) + def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd")) def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20220601", "yyyyMMdd")) def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 406d043a8..13d0f6c73 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -373,10 +373,10 @@ class DealPriceDataProcess { def citys = CcbCommonUtils.citys def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def publishTime = '2021-10-01' - def month = 202205 + def month = 202206 def rows = [] for(def city : citys) { - def rets = findPeGteThirdParty(city, 202206) + def rets = findPeGteThirdParty(city, 202207) def total = 0 println city +" "+rets.size() for(def m : rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy index 20e45bd1b..04bbec335 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy @@ -24,7 +24,7 @@ class LoadGuide { def run(){ // addName("D:\\数据\\建行\\房产估值\\人工指导价更新.xlsx") - addPersonGuided("D:\\数据\\建行\\房产估值\\指导价\\7月人工指导价.xlsx") + addPersonGuided("D:\\数据\\建行\\房产估值\\指导价\\7月23日建行库第一批录入.xlsx") // match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx") // saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx") @@ -61,6 +61,10 @@ class LoadGuide { def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) createSql( r, cityCodeDatas, distCodeMap) +// def dist = CommonDatautils.findDistrictById(r['A']) +// if(dist){ +// r['I'] = dist.region +// } } }) @@ -79,6 +83,7 @@ class LoadGuide { //def dist = CommonDatautils.findDistrict(cityName, sampleRegion, commName, "district") def dist = CommonDatautils.findDistrictById(commid) + def index = 0 if(dist){ def sampleRegion = dist.region @@ -89,7 +94,8 @@ class LoadGuide { def sql = "insert into guided_price_gj(ID,COMMID,COMMNAME,CITYID,CITYNAME,DISTID,DISTNAME,DEALDATE,AVG_PRICE,GUIDING_PERSON,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES " + "('${id.toString()}','${dist._id.toString()}', '${commName}','${cityId}','${cityName}','${distRegion.district_code}','${distName}',to_date('${dealDate}','yyyy-mm-dd'),${price},'房价网','',to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),'FJW')" - println sql + index ++ + println index+" "+sql sqlInstance.executeUpdate(sql) }else{ println "区域未标准化" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 41a0fe686..bef978957 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -45,9 +45,9 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) -def url = "https://sh.anjuke.com" -def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com") -println s +//def url = "https://sh.anjuke.com" +//def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com") +//println s //def districts = CommonDatautils.findAllFiledDistrict("district", "南宁") //for(def d : districts){ @@ -326,28 +326,55 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def // //},0) -//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除重复小区.xlsx",{d-> -// if(d['A']){ -//// def region = d['B'] -//// def newRegion = RegionData.formatRegion(d['A'], region) -//// if(newRegion) { -//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 11) -// udata.put("status", 1) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", dist._id) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// }else{ -// println d -// } -//// } -// } -//},0) +ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> + if(d['A']){ +// def region = d['B'] +// def newRegion = RegionData.formatRegion(d['A'], region) +// if(newRegion) { +// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + def udata = new BasicDBObject() + udata.put("name_cat", 11) + udata.put("status", 1) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", dist._id) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + }else{ + println d + } +// } + } +},0) + + +ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> + if(d['A']){ +// def region = d['B'] +// def newRegion = RegionData.formatRegion(d['A'], region) +// if(newRegion) { +// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + def udata = new BasicDBObject() + udata.put("name_cat", 11) + udata.put("status", 1) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", dist._id) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + }else{ + println d + } +// } + } +},0) + + + //def citys = CcbCommonUtils.citys //def rets = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/Test.groovy index 2996353c6..c5ee6bb09 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/Test.groovy @@ -18,7 +18,7 @@ import com.mongodb.DBObject -println new HttpClient().get("https://longyuanguojiguangchangbl.fang.com/","gbk"); +//println new HttpClient().get("https://longyuanguojiguangchangbl.fang.com/","gbk"); /** * Created by gaojuan on 2017/9/4. */ @@ -45,27 +45,53 @@ println new HttpClient().get("https://longyuanguojiguangchangbl.fang.com/","gbk" //} //ExcelUtils.write("D:\\数据\\结果\\区域信息.xlsx", ret, "数据") -ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆农商行测试地址.xlsx",{d-> - if(d['B']){ - def json = GaoDeApiUtils.geo("重庆", d['B']) - if(json && json.info=="OK"){ - def datas = json.geocodes - if(datas){ - def row = datas[0] - println row - d['C'] = row.country - d['D'] = row.province - d['E'] = row.city.toString().replace("[]","") - d['F'] = row.citycode.toString().replace("[]","") - d['G'] = row.district.toString().replace("[]","") - d['H'] = row.street.toString().replace("[]","") - d['I'] = row.number.toString().replace("[]","") - d['J'] = row.adcode.toString().replace("[]","") - d['K'] = row.location - d['L'] = row.level - } - } +//ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆农商行测试地址.xlsx",{d-> +// if(d['B']){ +// def json = GaoDeApiUtils.geo("重庆", d['B']) +// if(json && json.info=="OK"){ +// def datas = json.geocodes +// if(datas){ +// def row = datas[0] +// println row +// d['C'] = row.country +// d['D'] = row.province +// d['E'] = row.city.toString().replace("[]","") +// d['F'] = row.citycode.toString().replace("[]","") +// d['G'] = row.district.toString().replace("[]","") +// d['H'] = row.street.toString().replace("[]","") +// d['I'] = row.number.toString().replace("[]","") +// d['J'] = row.adcode.toString().replace("[]","") +// d['K'] = row.location +// d['L'] = row.level +// } +// } +// } +// +// +//}) +println "开始统计" +def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\成交\\202202.xlsx",0,1) +def map = [:] +for(def d : datas){ + def prices = map.get(d['B']) + if(!prices){ + prices = [] + map.put(d['B'], prices) } + prices << d['G'] +} +def regions = ["闵行","浦东","嘉定","普陀","青浦","静安","金山","松江","虹口","杨浦","奉贤","长宁","宝山","徐汇","崇明","黄浦"] +def rows = [] +rows << ["区域","数量", "均价"] -}) \ No newline at end of file +regions.each{e-> + def prices = map.get(e) + def price = (prices.sum()/prices.size()).toInteger() + rows << [e, prices.size(), price] +} +//map.each{region,prices-> +// def price = (prices.sum()/prices.size()).toInteger() +// rows << [region, prices.size(), price] +//} +ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\成交\\trend-2202.xlsx", rows, "结果") \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistSearch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistSearch.groovy index afa1280a4..93d7a9a30 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistSearch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistSearch.groovy @@ -1,38 +1,124 @@ package com.fangjia.script.gj.crawl +import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.util.ABuYunUtil import com.fangjia.script.util.ExcelUtils import net.sf.json.JSONObject +import org.jsoup.Jsoup +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver class AnjukeDistSearch { static main(args) { def task = new AnjukeDistSearch() - task.run() +// task.run() + task.run1() } - def run(){ + def run1(){ + def file = "D:\\数据\\朔州匹配房源1.xlsx" + def i = 0 + +// System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); +// System.setProperty("http.nonProxyHosts", "localhost"); +// WebDriver driver = new ChromeDriver() + ExcelUtils.modify(file,{d-> - def data = getDataInfo(282, d['C']) - if(data){ - def match = data.match - if(match){ - println match + i++ + def count = d['D'].toString().toDouble().toInteger() + if(count <=3 && !d['E']){ + def url = " https://m.anjuke.com/shuo/community/?q="+URLEncoder.encode(d['C'], "utf-8") +// def doc = getDocument(url, driver) + def doc = ABuYunUtil.doGet(url, null, null, "utf-8") + if(doc){ + def list = JQueryUtils.select(".c-community-list a", doc) + for(def r : list){ + + } + } + } + }) + } + + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + def s = driver.getPageSource() + doc = Jsoup.parse(s); + Thread.sleep(5000) + if(!doc || (s && s.contains("验证"))){ + error++ + driver.manage().deleteAllCookies() + println "[${error}] error: "+url + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } + + + + def run(){ + def file = "D:\\数据\\朔州匹配房源1.xlsx" + def i = 0 + ExcelUtils.modify(file,{d-> + i++ + def count = d['D'].toString().toDouble().toInteger() + if(count <=3 && !d['E']){ + def data = getDataInfo(282, d['C']) + if(data){ + def match = data.match + def ids = [] + if(match){ + println i+" "+ match + for(def r : match){ + ids << r.comm_id + } + } + d['E'] = ids.join(",") } } }) - getDataInfo() } def getDataInfo(cityId, keyword){ def url = "https://m.anjuke.com/ajax/autocomplete/?city_id=282&kw="+URLEncoder.encode(keyword, "utf-8") - def ret = ABuYunUtil.doGet(url, null, null, "utf-8") - if (ret) { + + def header = [:] + header.put("accept","application/json, text/plain, */*") + header.put("accept-encoding","gzip, deflate, br") + header.put("accept-language","zh-CN,zh;q=0.9") +// header.put("cookie","sessid=39D02194-D146-4BF9-A51E-79C24F4DE0FC; ajk-appVersion=; id58=CpQB3mLL+XJIv5mECb1VAg==; aQQ_ajkguid=02672143-A93A-42AF-B37F-C86984A83BEC; 58tj_uuid=0714884f-f17b-4145-b201-605539f483be; _ga=GA1.2.1000624676.1657534859; als=0; seo_source_type=0; twe=2; ctid=282; obtain_by=2; xxzl_cid=ef7523dbc335430f841f42cc67185e63; xzuid=56c772a8-8b4d-45c5-a356-e896e14efe43; fzq_h=68fb4a86e47dd3a3d8648011e8d6f05e_1658390102879_497b24518dd941868c10f254e7132af2_1917884102; new_session=1; init_refer=; new_uv=16") + header.put("cookie","ew_uv=16") + + header.put("referer","https://m.anjuke.com/shuo/") + header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1") + + def ret = ABuYunUtil.doGet(url, null, header, "utf-8") + if(ret && ret.contains("验证")){ + def num = new Random().nextInt() + header.put("cookie","ew_uv="+num) + ret = ABuYunUtil.doGet(url, null, header, "utf-8") + } + if (ret && !ret.contains("验证")) { JSONObject jsonObj = JSONObject.fromObject(ret); if (jsonObj && jsonObj.msg == "ok") { def data = jsonObj.data return data } + }else{ + println keyword +" 验证页面" } return null } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy index 065010b5c..abd0f5197 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy @@ -25,11 +25,11 @@ class AnjukeHouseCrawl { System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() - def href = "https://zj.lianjia.com/ershoufang/103120144204.html" + def href = "https://yz.esf.fang.com/loupan/1816376982/housedetail.htm" def doc = getDocument(href, driver) // def doc = ClaABuYun.getResponse(href) // def newUrl = href.substring(0, hrefs.indexOf("?")) - sendMq(href, doc, "sell_lianjia") + sendMq(href, doc, "community_fang") // for(def i=25;i<50;i++){ // def url = "https://fz.anjuke.com/sale/jinana/p${i}/" @@ -216,6 +216,10 @@ class AnjukeHouseCrawl { if(!els){ els = JQueryUtils.select(".main-wrap", doc) } + + if(!els){ + els = JQueryUtils.select(".con_item", doc) + } if(els && els.size()>0){ stop = true; }else{ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy index bf3e50a92..cb64409e6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy @@ -6,6 +6,9 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import org.apache.commons.lang.StringUtils + +import java.util.regex.Pattern class ExportPaimai { @@ -15,11 +18,27 @@ class ExportPaimai { static main(args) { def task = new ExportPaimai(); -// task.run() + task.run() +// task.runCsv() +// task.runCsv2() // task.addressStandard() // task.addPrice() - task.stat() +// task.stat() // task.standard() +// task.updateCount() + } + + def updateCount(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\京东拍卖.xlsx",{d-> + if(d['AC']){ + def pmquery = new BasicDBObject() + pmquery.put("url",d['AC']) + def obj = paimai43_COLL.findOne(pmquery) + if(obj){ + d['R'] = obj.addPrice + } + } + }) } def stat(){ @@ -108,34 +127,77 @@ class ExportPaimai { }) } - def run(){ + def runCsv(){ def datas = findPaimaiData() println "总量:"+datas.size() + exportCsv("D:\\数据\\重庆农商行\\法拍\\淘宝法拍.csv", datas) + } + + def runCsv2(){ + def datas = findPaimaiData() + println "总量:"+datas.size() + exportCsv("D:\\数据\\重庆农商行\\法拍\\淘宝法拍.csv", datas) + } + + + def batchCreateFile(ret, fileName, headers, charset){ + OutputStream out = new FileOutputStream(fileName); + def sb = new StringBuffer(); +// sb.append(headers.join("@|@")) +// sb.append(System.getProperty("line.separator")); + + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + + def index = 0 + for(def row : ret){ + index++ + sb.append(StringUtils.join(row.toArray(), "@|@")) + sb.append(System.getProperty("line.separator")); + + println index + + if(index%10000==0){ + println index+"开始写入CSV "+new Date().toLocaleString() + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + println index+"结束写入CSV "+new Date().toLocaleString() + } + + } + + if(sb.length()>0){ + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + } + out.close(); + } + + + def run(){ + def city = "宿迁" + def region = null + def datas = findPaimaiData(city,region) + println "总量:"+datas.size() def rows = [] def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层", "拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价", "变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"] rows << header - def set = new HashSet() +// def set = new HashSet() def index = 0 for(def d : datas){ - if(!set.contains(d.title)){ +// if(!set.contains(d.title)){ index++ println index - set << d.title +// set << d.title def row = [] - row << d.title + row << d.title.replace(",",",") row << d.province row << d.city row << d.region row << d.address row << "" -// def formatAddress = AddressUtil.formatAddress(d.city, d.title) -// if(formatAddress){ -// row << formatAddress -// }else{ -// row << "" -// } row << d.houseNum row << d.unitNum row << d.roomNum @@ -167,26 +229,239 @@ class ExportPaimai { row << d.url rows << row - }else{ - println d.title - } +// }else{ +// println d.title +// } } - ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\苏州商业法拍原数据(已流拍).xlsx",rows,"数据") + ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\京东拍卖(${city}).xlsx",rows,"数据") + +// batchCreateFile(rows, "D:\\数据\\重庆农商行\\法拍\\淘宝拍卖(新).csv",null,"GBK") + println "生成完成........" } - def findPaimaiData(){ + + def exportCsv(path, list){ + File file = new File(path); + if (file.exists()){ + file.delete(); + } + file.append("标地物地址,") + file.append("省,") + file.append("城市,") + file.append("区域,") + file.append("地址,") + file.append("标题提取地址,") + file.append("楼栋号,") + file.append("单元,") + file.append("室号,") + file.append("楼盘名称,") + file.append("建筑面积,") + file.append("竣工日期,") + file.append("房屋用途,") + file.append("总楼层,") + file.append("拍卖状态,") + file.append("是否成交,") + file.append("拍卖阶段,") + file.append("拍卖方式,") + file.append("成交价,") + file.append("拍卖开始时间,") + file.append("拍卖结束时间,") + file.append("起拍价,") + file.append("保证金,") + file.append("加价幅度,") + file.append("评估价,") + file.append("市场价,") + file.append("变卖预缴款,") + file.append("竞价周期,") + file.append("延时周期,") + file.append("优先购买权人,") + file.append("报名人数,") + file.append("围观人数,") + file.append("设置提醒人数,") + file.append("延迟次数,") + file.append("URL,") + file.append(System.getProperty("line.separator")); + list.each{ + def d = it; + file.append(d.title + ",","gbk") + file.append(d.province + ",","gbk") + file.append(d.city + ",","gbk") + if(d.region){ + file.append(d.region+",","gbk") + }else{ + file.append(",") + + } + + if(d.address){ + file.append(d.address +",","gbk") + }else{ + file.append(",") + + } + + file.append(",") + + if(d.houseNum){ + file.append(d.houseNum +",","gbk") + }else{ + file.append(",") + + } + if(d.unitNum){ + file.append(d.unitNum+",","gbk") + }else{ + file.append(",") + } + + if(d.roomNum){ + file.append(d.roomNum+",","gbk") + }else{ + file.append(",") + } + + if(d.matchName){ + file.append(d.matchName+",","gbk") + }else{ + file.append(",") + } + if(d.area){ + file.append(d.area+",","gbk") + }else{ + file.append(",") + } + if(d.buildYear){ + file.append(d.buildYear+",","gbk") + }else{ + file.append(",") + } + if(d.estateType){ + file.append(d.estateType+",","gbk") + }else{ + file.append(",") + } + file.append(",") + file.append(",") + + if(d.status){ + file.append(d.status+",","gbk") + }else{ + file.append(",") + } + + if(d.stage){ + file.append(d.stage+",","gbk") + }else{ + file.append(",") + } + file.append(",") + + if(d.tradePrice){ + file.append(d.tradePrice+",") + }else{ + file.append(",") + } + if(d.startDate){ + file.append(DateUtils.formatDateByPattern(d.startDate, "yyyy-MM-dd") +",") + }else{ + file.append(",") + } + if(d.finishDate){ + file.append(DateUtils.formatDateByPattern(d.finishDate, "yyyy-MM-dd") +",") + }else{ + file.append(",") + } + if(d.startPrice){ + file.append(d.startPrice +",") + }else{ + file.append(",") + } + if(d.depositPrice){ + file.append(d.depositPrice +",") + }else{ + file.append(",") + } + + if(d.addPrice){ + file.append(d.addPrice +",") + }else{ + file.append(",") + } + if(d.evalatePrice){ + file.append(d.evalatePrice+",") + }else{ + file.append(",") + } + if(d.marketPrice){ + file.append(d.marketPrice +",") + }else{ + file.append(",") + } + if(d.sellOffAdvancePay){ + file.append(d.sellOffAdvancePay +",") + }else{ + file.append(",") + } + if(d.pricePkWeekly){ + file.append(d.pricePkWeekly +",") + }else{ + file.append(",") + } + if(d.delayWeekly){ + file.append(d.delayWeekly +",") + }else{ + file.append(",") + } + if(d.preemptivePerson){ + file.append(d.preemptivePerson +",") + }else{ + file.append(",") + } + if(d.enrollCount){ + file.append(d.enrollCount +",") + }else{ + file.append(",") + } + if(d.lookCount){ + file.append(d.lookCount +",") + }else{ + file.append(",") + } + if(d.callCount){ + file.append(d.callCount +",") + }else{ + file.append(",") + } + if(d.delayCount){ + file.append(d.delayCount +",") + }else{ + file.append(",") + } + if(d.url){ + file.append(d.url +",") + }else{ + file.append(",") + } + file.append(System.getProperty("line.separator")); + } + } + + def findPaimaiData(city, region){ def pmquery = new BasicDBObject() - pmquery.put("city","苏州") - pmquery.put("region","姑苏") - pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) - pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01"))) - pmquery.put("status","已流拍") - def dbcursor = paimai43_COLL.find(pmquery).sort(new BasicDBObject("finishDate",-1)) + pmquery.put("city",city) + if(region){ + pmquery.put("region",region) + } + // pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) + pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01"))) + +// Pattern pattern = Pattern.compile("rmfysszc.gov.cn", Pattern.MULTILINE); + Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE); +// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE); + pmquery.put("url", new BasicDBObject('$regex', pattern)) + def dbcursor = paimai43_COLL.find(pmquery).batchSize(2000) def rets = [] for(def db : dbcursor){ -// if(db.estateType && db.estateType=="车库"){ -// continue -// } rets << db } return rets diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index c31879151..4a2ad0792 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -11,11 +11,295 @@ import com.mongodb.DBCollection import com.rabbitmq.client.AMQP import java.text.SimpleDateFormat +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern -DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ali_fapai_detail", "192.168.0.235", 27018) +class PaimaiDataSend { + + DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ali_fapai_detail", "192.168.0.235", 27018) + DBCollection ali_zc_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_ali_zc_detail", "192.168.0.235", 27018) + DBCollection ali_fp_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_ali_detail", "192.168.0.235", 27018) + DBCollection jd_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_jd_detail", "192.168.0.235", 27018) + DBCollection rmfysszc_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_rmfysszc_detail", "192.168.0.235", 27018) + + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + def task = new PaimaiDataSend() + task.jdRun() +// task.aliZcRun() +// task.aliZcRun2() +// task.aliFpRun() +// task.rmfysszcRun() +// task.checkUrl() + + } + + + def checkUrl(){ + def q = new BasicDBObject() + def keys = new BasicDBObject() + keys.put("url", 1) + def dbcursor = ali_fp_coll.find(q).batchSize(3000) + def index = 0 + for(def d : dbcursor){ + index++ + def pm = new BasicDBObject(); + pm.put("url", d.url) + def obj = paimai43_COLL.findOne(pm) + if(!obj){ + println d.url + } + } + } + + def rmfysszcRun(){ + def index = 0 + def q = new BasicDBObject() +// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/76602.shtml") + def dbcursor = rmfysszc_coll.find(q).batchSize(3000) + for(def d : dbcursor){ + index++ + println "rm: "+index + JSON json = new JSONObject() + json.put("project", "paimai_rmfysszc") + json.put("url", d.url) + + JSON result = new JSONObject() + result.put("content", d.content) + +// def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) + result.put("sub_count", d.sub_count) + result.put("data_json", d.json) + result.put("province", d.province) + result.put("city", d.city) + result.put("bid_res_content", d.bid_res_content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + + + def udata = new BasicDBObject() + udata.put("parser_status", 2) + udata.put("m_date", new Date()) + rmfysszc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata)) + } + } + + def jdRun(){ + def index = 0 + def q = new BasicDBObject() + q.put("city","宿迁市") +// q.put("url","https://paimai.jd.com/108245329") + def dbcursor = jd_coll.find(q).batchSize(3000) + for(def d : dbcursor){ + index++ + println "jd:"+index + JSON json = new JSONObject() + json.put("project", "paimai_jd") + json.put("url", d.url) + + JSON result = new JSONObject() + result.put("content", d.desc_content) + + def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) + result.put("bid_content", bid_content.toString()) + + result.put("sub_count", d.sub_count) + def data_json = net.sf.json.JSONObject.fromObject(d.json) + result.put("data_json", data_json) + result.put("content_json", d.content) + result.put("item_notice_content", d.item_notice_content) + result.put("notice_detail_content", d.notice_detail_content.content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + + def udata = new BasicDBObject() + udata.put("parser_status", 3) + udata.put("m_date", new Date()) + jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata)) + } + } + + def aliZcRun(){ + def pm_q = new BasicDBObject() + pm_q.put("city","苏州") + pm_q.put("region","姑苏") +// pm_q.put("url","https://sf-item.taobao.com/sf_item/596573117068.htm") + pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01"))) +// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE); +// pm_q.put("url", new BasicDBObject('$regex', pattern)) +// pm_q.put("status", null) + def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000) + println dbcursor.count() + for(def d : dbcursor){ + def flag = sendAliZc(d.url) + if(!flag){ + println "不存在:"+d.url + } + } + } + + def sendAliZc(def url){ + def index = 0 + def q = new BasicDBObject() + q.put("url", url) + def dbcursor = ali_zc_coll.find(q).batchSize(3000).sort(new BasicDBObject("_id",-1)) + def flag = false + for(def d : dbcursor){ + index++ + flag = true + println "ali-zc: "+index + JSON json = new JSONObject() + json.put("project", "paimai_taobao_zc") + json.put("url", d.url) + + JSON result = new JSONObject() + result.put("content", d.content) + def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) + result.put("trades", bid_content) +// result.put("condition", d.condition) + result.put("sub_count", d.sub_count) + def data_json = net.sf.json.JSONObject.fromObject(d.json) + result.put("data_json", data_json) + result.put("desc_content", d.desc_content) + result.put("item_notice_content", d.item_notice_content) + result.put("notice_detail_content", d.notice_detail_content) + + result.put("province", d.province) + result.put("city", d.city) + result.put("region", d.region) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + + + def udata = new BasicDBObject() + udata.put("parser_status", 1) + udata.put("m_date", new Date()) + ali_zc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata)) + } + return flag + } + + def aliFpRun(){ + def index = 0 + def q = new BasicDBObject() + q.put("city","宿迁") + q.put("url","https://sf-item.taobao.com/sf_item/631616791778.htm") + //https://sf-item.taobao.com/sf_item/657131607486.htm +// q.put("parser_status", new BasicDBObject('$exists',false)) + def dbcursor = ali_fp_coll.find(q).batchSize(2000) + + def rets = [] + + for(def d : dbcursor){ + index++ + println "ali-fp: "+index + JSON json = new JSONObject() + json.put("project", "paimai_taobao") + json.put("url", d.url) + + JSON result = new JSONObject() + result.put("content", d.content) + def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) + result.put("trades", bid_content) + result.put("province", d.province) + result.put("city", d.city) + result.put("region", d.region) + result.put("sub_count", d.sub_count) + def data_json = net.sf.json.JSONObject.fromObject(d.json) + result.put("data_json", data_json) + result.put("desc_content", d.desc_content) + result.put("item_notice_content", d.item_notice_content) + result.put("notice_detail_content", d.notice_detail_content) + json.put("result", result) + + rets << json + } + + println "共:"+rets.size() + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i +// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL) +// if(keys){ +// d['E'] = keys.size() // } -// def key = d.district_name+"##"+d.area+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date, "yyyyMMdd") -// if(!list.contains(key)){ -// list << key -// } -//} + +// def rets = findHousesByCity(d['A'], "sell", 20210701) + if(d['A']){ + def count = findHouseCount(d['A']) + d['F'] = count + } + +}) +def findHouseCount(city){ + def count = 0 + ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date >= 20210701 and city='${city}'", { + count = it.count + println("$city\t${it.count}") + }) + return count +} + + +def getDealPrices(def city, coll){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd"))) + def dbcursor = coll.find(q) + def keys = [] + for(def d : dbcursor){ +// def list = map.get(d.region) +// if(!list){ +// list = [] +// map.put(d.region, list) +// } + def key = d.district_name+"##"+d.area+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date, "yyyyMMdd") + if(!keys.contains(key)){ + keys << key + } + } + return keys; +} //def q = new BasicDBObject() //q.put("status","已成交") //q.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-07-01"))) @@ -71,42 +103,42 @@ DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "regio // //}) -def citys = ["锡林郭勒盟","保亭","塔城","陇南","海东","日喀则","迪庆州","忻州","昌吉","海西州","和田","克拉玛依","梅州","潮州","延安","楚雄","阿克苏","平凉","商洛","普洱","大兴安岭","白山","西双版纳州","临夏","榆林","阿勒泰","文昌","陵水","吐鲁番","庆阳","临沧","保山","黑河","喀什","白银","朔州","龙岩","韶关","揭阳","儋州","吕梁","来宾","渭南","肇庆","运城","丽江","德宏州","长治","黔南","张掖","金华","河源","邢台","晋中","宁德","曲靖","昭通","拉萨","玉溪","林芝","百色","仙桃","湘潭","滨州","随州","日照","呼伦贝尔","崇左","汕尾"] - -def districts = CommonDatautils.findAllDistrictByCity("朔州","district",["city","region","name","alias"]) - -def totalCount = new AtomicLong() -final CountDownLatch latch = new CountDownLatch(districts.size()) -println "小区量:"+districts.size() - -def retMap = new ConcurrentHashMap() -for(def i=0;i - def count = countMap.get(d['A']) - d['R'] = count -}) - -println "完成" +//def countMap = ["深圳":5,"中山":409,"贵阳":372,"南宁":1130,"台州":1325,"杭州":1177,"南昌":1369,"厦门":1370,"绍兴":1363,"徐州":1468,"长沙":1620,"东莞":1673,"郑州":1769,"遵义":1004,"温州":2012,"烟台":2080,"哈尔滨":5459,"湛江":392,"株洲":1068,"武汉":2486,"长春":2857,"岳阳":356,"肇庆":607,"镇江":1043,"芜湖":402,"银川":505,"合肥":2818,"宜昌":786,"淄博":2441,"佛山":3067,"马鞍山":227,"清远":537,"汕头":707,"洛阳":552,"济南":3456,"九江":138,"秦皇岛":1495,"宁波":3772,"乌鲁木齐":2273,"沧州":394,"泉州":4692,"廊坊":1120,"唐山":2626,"石家庄":4914,"漳州":4071,"张家口":357,"沈阳":5000,"青岛":5052,"柳州":2270,"安庆":756,"兰州":5399,"拉萨":784,"昆明":5436,"吉林":2274,"惠州":6206,"嘉兴":6142,"常德":963,"梅州":317,"绵阳":494,"海口":7883,"江门":663,"临沂":1473,"日照":2070,"襄阳":209,"湖州":3667,"平顶山":1501,"泰安":356,"大同":2814,"衡阳":241,"桂林":372,"临汾":372,"锦州":2875,"北海":2507,"赣州":1659,"商丘":644,"德州":639,"渭南":430,"咸阳":301,"聊城":932,"保定":3672,"许昌":266,"邢台":277,"潍坊":3743,"西宁":4002,"宝鸡":204,"邯郸":2361,"安阳":557,"济宁":2194,"龙岩":367,"太原":7524,"南充":568,"盐城":6512,"阜阳":198,"运城":462,"菏泽":320,"晋中":539,"新乡":768,"滁州":1025,"衡水":969,"信阳":1048,"长治":806,"韶关":696,"荆州":361,"汉中":118,"威海":3030,"孝感":984,"东营":874,"宜春":251,"开封":111,"驻马店":549,"湘潭":410,"黄冈":250,"三亚":7481,"上饶":164,"眉山":284,"绥化":851,"宁德":1469,"苏州":8876,"焦作":767,"牡丹江":1009,"齐齐哈尔":566,"枣庄":2636,"河源":167,"宿迁":3428,"遂宁":81,"滨州":3454,"漯河":308,"周口":1670,"茂名":347,"大庆":932,"平凉":210,"南阳":4087,"黄石":99,"鄂尔多斯":452,"营口":1470,"包头":4789,"吉安":442,"金华":13539,"通辽":238,"泸州":183,"乐山":2776,"佳木斯":506,"三门峡":574,"承德":2923,"荆门":64,"曲靖":1521,"忻州":770,"潮州":359,"永州":209,"揭阳":934,"呼伦贝尔":353,"扬州":12477,"宣城":293,"赤峰":2770,"吕梁":2618,"宜宾":322,"濮阳":2389,"宿州":264,"阳江":270,"延安":388,"丹东":885,"怀化":258,"黄山":312,"十堰":2687,"抚州":179,"郴州":4355,"衢州":1714,"陇南":4,"榆林":4694,"延边":1640,"抚顺":258,"丽水":817,"庆阳":2069,"玉溪":254,"玉林":544,"咸宁":1983,"舟山":1970,"达州":990,"亳州":289,"呼和浩特":10185,"张掖":64,"广元":148,"益阳":156,"汕尾":2392,"昌吉":225,"蚌埠":5496,"天水":173,"随州":211,"珠海":13972,"巴彦淖尔":0,"楚雄":95,"乌兰察布":172,"新余":86,"南平":7781,"广安":2129,"景德镇":474,"铁岭":2676,"松原":747,"淮安":13859,"三明":6425,"朝阳":3393,"酒泉":479,"四平":261,"梧州":248,"自贡":2983,"黔东南":205,"大理":1914,"黔南":866,"内江":177,"仙桃":70,"白银":2398,"朔州":1861,"娄底":2601,"六盘水":277,"百色":73,"阳泉":2135,"德阳":9330,"邵阳":2425,"盘锦":4269,"双鸭山":0,"白山":60,"通化":2379,"红河":4013,"鄂州":149,"吴忠":244,"毕节":88,"云浮":10,"伊犁":579,"丽江":331,"贵港":705,"黑河":10,"安康":381,"葫芦岛":6562,"资阳":1320,"鹤壁":1904,"萍乡":173,"锡林郭勒盟":34,"防城港":33,"钦州":2141,"阿克苏":540,"鹤岗":233,"铜川":12,"喀什":68,"巴中":2540,"凉山州":0,"鹰潭":46,"石嘴山":468,"保山":152,"伊春":54,"六安":7649,"定西":75,"鸡西":2384,"临夏":1598,"海西州":0,"商洛":58,"中卫":112,"普洱":302,"湘西":4,"来宾":8,"张家界":21,"西双版纳州":0,"恩施":2215,"攀枝花":87,"晋城":6613,"乌海":361,"七台河":172,"雅安":1152,"黔西南":391,"嘉峪关":37,"淮南":6767,"潜江":0,"辽源":163,"临沧":89,"塔城":1,"克拉玛依":121,"金昌":10,"儋州":33,"海东":106,"大兴安岭":5,"兴安盟":972,"和田":0,"吐鲁番":107,"迪庆州":14,"安顺":1939,"日喀则":32,"文昌":2,"林芝":9,"保亭":0,"陵水":0,"阿勒泰":177,"天门":370,"崇左":560,"莆田":12067,"白城":2422,"铜仁":2209,"河池":1926,"辽阳":4615,"昭通":1995,"哈密":2592,"本溪":4452,"武威":1885,"德宏州":1654,"大连":18125,"成都":17618,"博尔塔拉":2094,"贺州":2168,"池州":6831,"固原":4445,"阜新":6345,"鞍山":15277,"巴音郭楞":7481,"铜陵":9225,"泰州":26770,"南通":26075,"连云港":30835,"西安":28834,"淮北":16917,"北京":35728,"南京":32406,"无锡":48220,"广州":50209,"天津":63311,"上海":160627,"常州":155553,"重庆":221058] +//ExcelUtils.multiThreadingModify("D:\\数据\\房源\\估值数据统计结果20220715.xlsx",{d-> +// def count = countMap.get(d['A']) +// d['R'] = count +//}) +// +//println "完成" def findThirdParty(city, region, name, coll){ @@ -255,21 +287,25 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){ // list << d //} // + +//def citys = ["淮安","扬州","南京","苏州","无锡","常州","徐州","镇江","南通","泰州","宿迁","连云港","盐城"] //def rows = [] -//rows << ["区域","小区量", "别名量", "地址别名量"] -//map.each{region, list-> +//rows << ["城市","小区量", "别名量", "地址别名量"] +//for(def c : citys){ +// def districts = CommonDatautils.findAllDistrictByCity(c,"district",["city","region","name","alias", "addr_alias"]) // def alias_count = 0 // def addr_alias_count = 0 -// for(def r : list){ -// if(r.mapping_alias && r.mapping_alias.size()>0){ +// for(def r : districts){ +// if(r.alias && r.alias.size()>0){ // alias_count ++ // } // if(r.addr_alias && r.addr_alias.size()>0){ // addr_alias_count++ // } // } -// -// rows << [region, list.size(), alias_count, addr_alias_count] +// rows << [c, districts.size(), alias_count, addr_alias_count] //} // -//ExcelUtils.write("D:\\数据\\重庆农商行\\重庆数据统计.xlsx", rows, "楼盘统计") +// +// +//ExcelUtils.write("D:\\数据\\江苏\\江苏数据统计.xlsx", rows, "结果") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangshu/JsBlankTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangshu/JsBlankTest.groovy new file mode 100644 index 000000000..d58a2e38b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangshu/JsBlankTest.groovy @@ -0,0 +1,31 @@ +package com.fangjia.script.jiangshu + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +class JsBlankTest { + + static main(args) { + def task = new JsBlankTest(); + task.run(); + } + + def run(){ + def path = "D:\\数据\\江苏\\" + } + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['Q'] && row['D']){ + def city = row['B'] + def address = row['D'] + def region = row['C'] + println "[match] city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['Q'] = jsonObject.get("name") + row['R'] = jsonObject.get("id") + } + } + }) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 383911899..a5baaf550 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202205); - def count2 = findTotalCount(source, 202206); + def count1 = findTotalCount(source, 202206); + def count2 = findTotalCount(source, 202207); println source +" "+count1 +" "+count2 } }