From 808cd9590960ff0093b8439dbcd5ef7e482fe58c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 13 Sep 2022 10:25:14 +0800 Subject: [PATCH] update --- .../java/com/fangjia/core/util/GeoUtils.java | 5 + .../ccbevaluation/CcbCommonUtils.groovy | 10 +- .../DealAndThirdPriceDataProcess.groovy | 13 +- .../ccbevaluation/FjwPricePkCcbPrice.groovy | 193 +++++++++++++++ .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 165 +++++++++---- .../script/commData/CommonDatautils.groovy | 14 ++ .../script/crawl/SchoolLocation.groovy | 159 ++++++++++++ .../script/fuzhou/FzDataResultStat.groovy | 4 +- .../script/fuzhou/HouseExportBatch.groovy | 4 +- .../gj/district/FangjiaClientUtils.groovy | 1 + .../gj/paimai/BeiKeCrawlByChrome.groovy | 6 +- .../script/gj/paimai/ExportPaimai.groovy | 56 ++++- .../gj/paimai/LianjiaCrawlByChrome.groovy | 121 +++++---- .../script/gj/paimai/PaimaiDataSend.groovy | 229 ++++++++++++++---- .../fangjia/script/gj/test/CityStat.groovy | 28 ++- .../com/fangjia/script/gj/test/Test.groovy | 16 +- .../script/gj/test/TestSendSchoolData.groovy | 136 +++++++++++ .../script/thirdPriceCrawl/DataStat.groovy | 8 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 17 +- .../TestCrawlTotalCount.groovy | 4 +- 21 files changed, 1004 insertions(+), 187 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/crawl/SchoolLocation.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/TestSendSchoolData.groovy diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/GeoUtils.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/GeoUtils.java index 00f40baa6..bcb042e67 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/GeoUtils.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/GeoUtils.java @@ -257,4 +257,9 @@ public class GeoUtils { ret += (150.0 * Math.sin(x / 12.0 * pi) + 300.0 * Math.sin(x / 30.0 * pi)) * 2.0 / 3.0; return ret; } + + public static void main(String[] args){ + Map location = gps84ToGcj02(31.218024629574998,121.36562884455); + System.out.println(location.toString()); + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index db3beb6ef..e4f47e2fd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -77,8 +77,16 @@ class CcbCommonUtils { q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate))); def dbcursor = GUIDING_PRICE_COLL.find(q) def rets = [] + def sets = new HashSet() for(def d : dbcursor){ - rets << d + def guiding_date = DateUtils.formatDateByPattern(d.guiding_date, 'yyyy-MM-dd') + def key = d.city +"##"+d.region+"##"+d.name+"##" + guiding_date + if(!sets.contains(key)){ + rets << d + sets << key + }else{ + println key + } } return rets } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 13d0f6c73..6dcffbacf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -370,24 +370,25 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ - def citys = CcbCommonUtils.citys + def citys = ["上海","广州","东莞"]//CcbCommonUtils.citys def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def publishTime = '2021-10-01' - def month = 202206 + def month = 202207 def rows = [] + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys) { - def rets = findPeGteThirdParty(city, 202207) + def rets = findPeGteThirdParty(city, 202208) def total = 0 println city +" "+rets.size() for(def m : rets){ - def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}" + def cityId = cityCodeDatas.get(city) + def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}" sqlInstance236.executeUpdate(sql) } } } def findPeGteThirdParty(cityName, yearmonth){ - def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30" + def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>25" def rets = [] sqlInstance236.eachRow(sql,{r-> def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy new file mode 100644 index 000000000..e8f85a1b7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -0,0 +1,193 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import groovy.sql.Sql +import org.apache.commons.lang.StringUtils + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class FjwPricePkCcbPrice { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new FjwPricePkCcbPrice() +// task.run() + task.updatePrice() + } + + def updatePrice(){ + def index = 0 + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\测试\\ccb.xlsx",{d-> + index++ + println index + def dbo = CommonDatautils.queryDistrictPrice(d['B'], d['C'], d['D'], DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd')) + if(dbo){ + d['G'] = dbo.future_value + } + }) + } + + def run(){ + def ccbPrices = getCommunityBasePriceGj(202208) + println "ccb总量:"+ccbPrices.size() + def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd')) + println "fjw总量:"+ccbPrices.size() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(ccbPrices.size()) + + + def rets = new CopyOnWriteArrayList() + for(def i=0;i + def map = [:] + map.cityname = r.CITYNAME + map.distname = r.DISTNAME + map.commid = r.COMMID + map.commname = r.COMMNAME + map.year_month = r.YEAR_MONTH + map.calc_avg_price = r.CALC_AVG_PRICE + map.avg_price = r.AVG_PRICE + // 计算类型,1:人工估值 2:样本计算 3:神经网络 + if(r.CALC_TYPE){ + if(r.CALC_TYPE == 1){ + map.calc_type = "人工估值" + }else if(r.CALC_TYPE == 2){ + map.calc_type = "样本计算" + }else if(r.CALC_TYPE == 3){ + map.calc_type = "神经网络" + } + } + allRows << map + }); + return allRows + } + + + def batchCreateFile(ret, fileName, headers, charset){ + OutputStream out = new FileOutputStream(fileName); + def sb = new StringBuffer(); +// sb.append(headers.join("@|@")) +// sb.append(System.getProperty("line.separator")); + + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + + def index = 0 + for(def row : ret){ + index++ + sb.append(StringUtils.join(row.toArray(), "@|@")) + sb.append(System.getProperty("line.separator")); + + println index + + if(index%10000==0){ + println index+"开始写入CSV "+new Date().toLocaleString() + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + println index+"结束写入CSV "+new Date().toLocaleString() + } + + } + + if(sb.length()>0){ + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + } + out.close(); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 6838e8e0a..f839de5f5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-07-22", "2022-08-21") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-07-22", "2022-08-24") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index bef978957..bd932d6fd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -30,8 +30,8 @@ import java.util.regex.*; //def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", // "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") -//def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", -// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") //def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", // "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -197,6 +197,79 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business //println "共查到:"+index +def index = 0 +//def commIdThirdMap = findAllThird(202207, sqlInstance236); +def cityCodeDatas = CcbCommonUtils.findCityCodeValue() +//cityCodeDatas.each{city,cityId-> +// +// deleteThirdByCityId(cityId, 202207, sqlInstance236) +// println city + "完成" +//} + + +def lines = org.apache.commons.io.FileUtils.readLines(new File("D:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat")) +for(line in lines) { + def arrs = line.toString().split("\\|@\\|") + if(arrs){ + def sql = "delete from guided_price_gj where id='${arrs[0]}'" + sqlInstance236.executeUpdate(sql) + } +} + +def deleteThirdByCityId(def cityId,def price_date, def sqlInstance236){ + def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and price_date=${price_date}" + sqlInstance236.executeUpdate(sql) +} +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区2022-8-23.xlsx", { d -> +// if (d['C']) { +// index ++T +// def cityId = cityCodeDatas.get(d['A']) +// def sources = findThirdSource(cityId, d['C'], 202207, sqlInstance236) +// if(sources.size()>1){ +// if(sources.contains("安居客") && sources.contains("58同城")){ +// sources.remove("58同城") +// }else if(sources.contains("链家") && sources.contains("贝壳")){ +// sources.remove("贝壳") +// } +// } +// if(sources.size()==1){ +// println index +" 删除 "+ d['C']+" ${d['A']}:"+ cityId +// deleteThirdParty(cityId, d['C'], 202207, sqlInstance236) +// } +// } +//},0) + + +def findAllThird(price_date, def sqlInstance236){ + def sql = "select commid, data_source from bigdata_third_party_price_gj where price_date=${price_date}" + def map = [:] + sqlInstance236.eachRow(sql,{r-> + def sources = map.get(r.commid) + if(!sources){ + sources = [] + map.put(r.commid, sources) + } + sources << r.data_source + }); + return map +} + + +def deleteThirdParty(def cityId, def commid, def price_date, def sqlInstance236){ + def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}" + sqlInstance236.executeUpdate(sql) +} + +def findThirdSource(def cityId, def commid, def price_date, def sqlInstance236){ + def sql = "select data_source from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}" + def sources = [] + sqlInstance236.eachRow(sql,{r-> + sources << r.data_source + }); + return sources +} + + def findAvmData(sqlInstance, def region, def name, def area, def price, def dealdate){ def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=202106 and data_type='成交' and cityname='上海' and distname='${region}' and commname='${name}' and buildarea=${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd') and target_avg_price=${price}" def count = 0 @@ -326,52 +399,52 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def // //},0) -ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> - if(d['A']){ -// def region = d['B'] -// def newRegion = RegionData.formatRegion(d['A'], region) -// if(newRegion) { -// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") - def dist = CommonDatautils.findDistrictById(d['A']) - if(dist){ - def udata = new BasicDBObject() - udata.put("name_cat", 11) - udata.put("status", 1) - udata.put("m_date",new Date()) +//ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> +// if(d['A']){ +//// def region = d['B'] +//// def newRegion = RegionData.formatRegion(d['A'], region) +//// if(newRegion) { +//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// def udata = new BasicDBObject() +// udata.put("name_cat", 11) +// udata.put("status", 1) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", dist._id) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// }else{ +// println d +// } +//// } +// } +//},0) - def q = new BasicDBObject() - q.put("_id", dist._id) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - }else{ - println d - } + +//ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> +// if(d['A']){ +//// def region = d['B'] +//// def newRegion = RegionData.formatRegion(d['A'], region) +//// if(newRegion) { +//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// def udata = new BasicDBObject() +// udata.put("name_cat", 11) +// udata.put("status", 1) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", dist._id) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// }else{ +// println d // } - } -},0) - - -ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> - if(d['A']){ -// def region = d['B'] -// def newRegion = RegionData.formatRegion(d['A'], region) -// if(newRegion) { -// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") - def dist = CommonDatautils.findDistrictById(d['A']) - if(dist){ - def udata = new BasicDBObject() - udata.put("name_cat", 11) - udata.put("status", 1) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", dist._id) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - }else{ - println d - } -// } - } -},0) +//// } +// } +//},0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 94c151d1a..61523694a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -2049,6 +2049,20 @@ class CommonDatautils { } + public static def queryAllDistrictPrice(startDate){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put('weekend', startDate); + DBCursor cur = DIST_STAT_COLL.find(queryCondition) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def rMap = [:]; + while(cur.hasNext()) { + DBObject dbo = cur.next() + rMap.put(dbo.get("city") + "##" + dbo.get("region")+"##"+ dbo.get("district_name"), dbo) + } + return rMap; + } + + public static def queryDistrictPrices(city, startDate){ DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/crawl/SchoolLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/crawl/SchoolLocation.groovy new file mode 100644 index 000000000..534a844e7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/crawl/SchoolLocation.groovy @@ -0,0 +1,159 @@ +package com.fangjia.script.crawl + +import com.alibaba.fastjson.JSONArray +import com.fangjia.core.util.GeocodeUtils +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.lang.StringUtils + +class SchoolLocation { + + DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018) + + static main(args) { + def task = new SchoolLocation() + task.run() + } + + def run(){ + def query = new BasicDBObject() + query.put("province","江苏") + //query.put("lat",null) + query.put("lat",new BasicDBObject('$exists',false)) + def cursor = school_parse_coll.find(query) + def index = 0 + for(def d : cursor){ + index++ + def datas = getSchoolLatAndLng(d.city, d.name, d.address, false) + if(datas){ + for(def ret : datas){ + def addr = d.address + if(d.address.contains(d.city+"市")){ + def cityStr = d.city+"市" + addr = d.address.substring(d.address.indexOf(cityStr)+cityStr.size()) + } + if(ret.name == d.name || (d.address.contains("号") && (ret.address.contains(addr) || addr.contains(ret.address)))){ + def udata = new BasicDBObject() + udata.put("lat", ret.lat.toString().toDouble()) + udata.put("lng", ret.lng.toString().toDouble()) + udata.put("status", 0) + school_parse_coll.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata)) + println index +" "+udata + break; + } + } + } + } + } + + def getSchoolLatAndLng(city, name, address, isFullMatch){ + if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){ + return null; + } + address = address.trim().replace(" ","").replace("?","").replace(" ","") + String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=学校&city=" + city + "&output=json&offset=5&page=1&key=4e9592208336433ee1b7aa4d1d434418&extensions=all" + + List poiList = null; + com.alibaba.fastjson.JSONObject result = null; + try { + HttpRequest request = new HttpRequest(); + Map params = new HashMap(); + String response = request.get(url, params, "UTF-8"); + com.alibaba.fastjson.JSONObject jsonResponse = com.alibaba.fastjson.JSONObject.parseObject(response) + if (jsonResponse != null) { + JSONArray ja = jsonResponse.getJSONArray("pois"); + if(ja != null && ja.size() > 0) { + poiList = new ArrayList<>(); + for(int i=0;i() { +// @Override +// int compare(BasicDBObject o1, BasicDBObject o2) { +// if(o1.finishDate && o2.finishDate){ +// return -o1.finishDate.compareTo(o2.finishDate) +// }else{ +// return 0 +// } +// +// } +// }) + rets.sort {a,b-> + if(a.finishDate && b.finishDate){ + def v1 = a.finishDate + def v2 = b.finishDate + return v1.compareTo(v2) + }else{ + return 0 + } + } + rets = rets.reverse() return rets } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 50470f713..9bd49bc3b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -129,7 +129,7 @@ public class LianjiaCrawlByChrome { def standardRegion(){ def q = new BasicDBObject() - q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd"))) + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd"))) q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) ) def dbcursor = DEAL_PRICE_COLL.find(q) for(def d : dbcursor){ @@ -152,8 +152,9 @@ public class LianjiaCrawlByChrome { def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") lastMonth = lastMonth +"-01" def q = new BasicDBObject() + q.put("city", "福州") q.put("region", null) - q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-05-01","yyyy-MM-dd"))) + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd"))) // q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) def dbcursor = DEAL_PRICE_COLL.find(q) def keys = [] @@ -187,8 +188,19 @@ public class LianjiaCrawlByChrome { DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) cityMap.each{city, cityUrl-> //def detailUrls = getPageUrls(driver, cityUrl) - def detailUrls = getPageDeail(driver, city, cityUrl) - println cityUrl +" 详情链接:"+detailUrls.size() + driver.get(cityUrl+"chengjiao/") + def doc = Jsoup.parse(driver.getPageSource()); + def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) + if(regions){ + for(def r : regions){ + def regionText = r.text() + def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href") + + println "开始抓取【${city}】【${regionText}】【${href}】" + def detailUrls = getPageDeail(driver, city, href, regionText) + println cityUrl +" 详情链接:"+detailUrls.size() + } + } // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) } } @@ -224,12 +236,12 @@ public class LianjiaCrawlByChrome { } - def getPageDeail(driver, city, cityUrl){ + def getPageDeail(driver, city, cityUrl, regionText){ def stop = false def page = 1 def detailUrls = [] while(!stop){ - def url = cityUrl + "chengjiao/pg${page}/" + def url = cityUrl + "pg${page}/" driver.get(url) def doc = Jsoup.parse(driver.getPageSource()); def elements = JQueryUtils.select(".listContent li", doc) @@ -241,7 +253,7 @@ public class LianjiaCrawlByChrome { def endDate = DateUtils.parseFromTightByPattern("2022-06-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl - parse(city, e) + parse(city, e, regionText) }else{ stop = true; break; @@ -259,7 +271,7 @@ public class LianjiaCrawlByChrome { return detailUrls } - def parse(city,e){ + def parse(city,e, regionText){ try { def title = e.select(".title a").text() if (title) { @@ -267,7 +279,11 @@ public class LianjiaCrawlByChrome { def arrs = title.split(" ") detail.city = city detail.district_name = arrs[0] - detail.region = findRegionByThirdPary(detail.city, detail.district_name) + if(regionText){ + detail.region = regionText + }else{ + detail.region = findRegionByThirdPary(detail.city, detail.district_name) + } if(detail.region){ detail.region = RegionData.formatRegion(city, detail.region) } @@ -275,50 +291,61 @@ public class LianjiaCrawlByChrome { def roomStr = arrs[1] if (roomStr) { if (roomStr.indexOf("室") > 0) { - detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() + def room = roomStr.substring(0, roomStr.indexOf("室")) + if(room && room.isNumber()){ + detail.room = room.toInteger() + } + } if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { - detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() + def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + if(hall && hall.isNumber()){ + detail.hall = hall.toInteger() + } } } - def areaStr = arrs[2].replace("平米", "") - if (areaStr) { - detail.area = areaStr.toDouble() - } - - def price = e.select(".totalPrice .number").text() - if (price && price.isNumber()) { - detail.total_price = price.toInteger() * 10000 - } - - if (detail.total_price && detail.area) { - detail.avg_price = (int) (detail.total_price / detail.area) - } - def tradeStr = e.select(".dealDate").text() - if (tradeStr) { - detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") - } - detail.source = "链家在线" - detail.url = e.select(".title a").attr("href") - def dbo = findTrade(detail) - if (!dbo) { - def houseInfo = e.select(".houseInfo").text() - def houseInfos = houseInfo.split("\\|") - detail.direction = houseInfos[0] - detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() - if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { - floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() + if("车位"!=roomStr){ + if(arrs && arrs.size()>=3){ + def areaStr = arrs[2].replace("平米", "") + if (areaStr) { + detail.area = areaStr.toDouble() } } - detail.create_date = new Date() - detail.m_date = new Date() - println detail - DEAL_PRICE_COLL.save(detail) + def price = e.select(".totalPrice .number").text() + if (price && price.isNumber()) { + detail.total_price = price.toInteger() * 10000 + } + + if (detail.total_price && detail.area) { + detail.avg_price = (int) (detail.total_price / detail.area) + } + def tradeStr = e.select(".dealDate").text() + if (tradeStr) { + detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") + } + detail.source = "链家在线" + detail.url = e.select(".title a").attr("href") + def dbo = findTrade(detail) + if (!dbo) { + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + def floorStr = e.select(".positionInfo").text() + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } } } }catch(Exception ex){ @@ -446,9 +473,9 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") -// if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia") && citys.contains(e.text())){ cityMap.put(e.text(), href) -// } + } } } return cityMap diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index 4297a52b8..4c4e698f7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.AMQP +import com.rabbitmq.client.impl.AMQImpl import java.text.SimpleDateFormat import java.util.concurrent.CountDownLatch @@ -33,6 +34,7 @@ class PaimaiDataSend { static main(args) { def task = new PaimaiDataSend() +// task.jdRunByMongo() // task.jdRun() // task.aliZcRun() task.aliZcRun2() @@ -64,7 +66,8 @@ class PaimaiDataSend { def rmfysszcRun(){ def index = 0 def q = new BasicDBObject() -// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/76602.shtml") +// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市"])) + q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml") def dbcursor = rmfysszc_coll.find(q).batchSize(3000) for(def d : dbcursor){ index++ @@ -95,11 +98,22 @@ class PaimaiDataSend { } } - def jdRun(){ + def jdRunByMongo(){ + def pm_q = new BasicDBObject() + pm_q.put("city","宿迁") + pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01"))) + pm_q.put("mappingStage", null) + def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000) + println dbcursor.count() + for(def d : dbcursor){ + sendJd(d.url) + } + } + + def sendJd(url){ def index = 0 def q = new BasicDBObject() - q.put("city","宿迁市") -// q.put("url","https://paimai.jd.com/108245329") + q.put("url", url) def dbcursor = jd_coll.find(q).batchSize(3000) for(def d : dbcursor){ index++ @@ -110,6 +124,45 @@ class PaimaiDataSend { JSON result = new JSONObject() result.put("content", d.desc_content) + result.put("region", d.region) + + def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) + result.put("bid_content", bid_content.toString()) + + result.put("sub_count", d.sub_count) + def data_json = net.sf.json.JSONObject.fromObject(d.json) + result.put("data_json", data_json) + result.put("content_json", d.content) + result.put("item_notice_content", d.item_notice_content) + result.put("notice_detail_content", d.notice_detail_content.content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + + def udata = new BasicDBObject() + udata.put("parser_status", 3) + udata.put("m_date", new Date()) + jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata)) + } + } + def jdRun(){ + def index = 0 + def q = new BasicDBObject() + q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市"])) +// q.put("parser_status", new BasicDBObject('$ne',3)) + q.put("url","https://paimai.jd.com/115669796") + def dbcursor = jd_coll.find(q).batchSize(3000) + for(def d : dbcursor){ + index++ + println "jd:"+index + JSON json = new JSONObject() + json.put("project", "paimai_jd") + json.put("url", d.url) + + JSON result = new JSONObject() + result.put("content", d.desc_content) + result.put("region", d.region) def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) result.put("bid_content", bid_content.toString()) @@ -134,17 +187,19 @@ class PaimaiDataSend { def aliZcRun(){ def pm_q = new BasicDBObject() - pm_q.put("city","苏州") - pm_q.put("region","姑苏") -// pm_q.put("url","https://sf-item.taobao.com/sf_item/596573117068.htm") - pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01"))) +// pm_q.put("city","苏州") +// pm_q.put("region","姑苏") + pm_q.put("url","https://sf-item.taobao.com/sf_item/659117468505.htm") +// pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01"))) // Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE); // pm_q.put("url", new BasicDBObject('$regex', pattern)) -// pm_q.put("status", null) +// pm_q.put("mappingEstateType", null) +// pm_q.put("area", new BasicDBObject('$lt', 30)) def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000) println dbcursor.count() for(def d : dbcursor){ - def flag = sendAliZc(d.url) +// def flag = sendAliZc(d.url) + def flag = sendAli(d.url) if(!flag){ println "不存在:"+d.url } @@ -194,15 +249,59 @@ class PaimaiDataSend { return flag } + def sendAli(def url){ + def index = 0 + def q = new BasicDBObject() + q.put("url",url) + def dbcursor = ali_fp_coll.find(q).batchSize(5000) + def flag = false + for(def d : dbcursor){ + index++ + flag = true + println "ali-fp: "+index + JSON json = new JSONObject() + json.put("project", "paimai_taobao") + json.put("url", d.url) + + JSON result = new JSONObject() + result.put("content", d.content) + def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) + result.put("trades", bid_content) + result.put("province", d.province) + result.put("city", d.city) + result.put("region", d.region) + result.put("sub_count", d.sub_count) + def data_json = net.sf.json.JSONObject.fromObject(d.json) + result.put("data_json", data_json) + result.put("desc_content", d.desc_content) + result.put("item_notice_content", d.item_notice_content) + result.put("notice_detail_content", d.notice_detail_content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + + + def udata = new BasicDBObject() + udata.put("parser_status", 1) + udata.put("m_date", new Date()) + ali_fp_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata)) + } + + return flag + } + def aliFpRun(){ def index = 0 def q = new BasicDBObject() -// q.put("city","宿迁") - q.put("url","https://sf-item.taobao.com/sf_item/665877683620.htm") + q.put("city",new BasicDBObject('$in', ["南京","宿迁"])) +// q.put("parser_status", new BasicDBObject('$ne',2)) +// q.put("url","https://sf-item.taobao.com/sf_item/631662596036.htm") //https://sf-item.taobao.com/sf_item/657131607486.htm // q.put("parser_status", new BasicDBObject('$exists',false)) // q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01"))) - def dbcursor = ali_fp_coll.find(q).batchSize(2000) +// q.put("area", new BasicDBObject('$lt', 30)) + def dbcursor = ali_fp_coll.find(q).batchSize(5000) def rets = [] @@ -229,10 +328,20 @@ class PaimaiDataSend { json.put("result", result) rets << json + if(rets.size()==10000){ + println "共:"+rets.size() + send(rets) + rets.clear() + } } - println "共:"+rets.size() + if(rets.size()>0){ + println "共:"+rets.size() + send(rets) + } + } + def send(def rets){ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(rets.size()) for(def i=0;i0){ + println "共:"+rets.size() + send(rets) } - latch.await() + +// println "共:"+rets.size() +// +// def totalCount = new AtomicLong() +// final CountDownLatch latch = new CountDownLatch(rets.size()) +// for(def i=0;i0){ +// println "共:"+rets.size() +// send(rets, fixedThreadPool, school_coll) +// rets.clear() +// } +//} + + + +def send(def rets, def fixedThreadPool, def school_coll){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i for(def city : citys){ - def rets = findAllEsData(city, esSource, mgSource) + def rets = findAllEsData(city, esSource, mgSource, mdate) println city +" "+rets.size() synchData(city, rets, mgSource) } @@ -46,10 +43,10 @@ class SynchDistrictPrice { } - def findAllEsData(city,esSource, mgSource){ + def findAllEsData(city,esSource, mgSource, mdate){ def names = new HashSet() def rets = [] - ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='2022-07-01' order by m_date", {d-> + ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d-> def s = d.city +"#"+d.region+"#"+d.name if(!names.contains(s)){ names << s diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index a5baaf550..149149fd0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202206); - def count2 = findTotalCount(source, 202207); + def count1 = findTotalCount(source, 202207); + def count2 = findTotalCount(source, 202208); println source +" "+count1 +" "+count2 } }