From 41d099fcb59fb2ca7a7920196d66ea71461d4b0b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 19 Jul 2023 22:32:11 +0800 Subject: [PATCH] updadte --- .../script/ccbevaluation/AddSzDealData.groovy | 2 +- .../script/gj/paimai/BeiKeCrawlByChrome.groovy | 7 ++++--- .../script/gj/paimai/LianjiaCrawlByChrome.groovy | 9 +++++---- .../thirdPriceCrawl/DataStandardBatch.groovy | 16 ++++++++-------- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 1c1fa2788..7ddc3a095 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳5月份二手房成交数据汇总.xls", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳6月份二手房成交数据汇总.xls", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 2691fe7de..3df686d65 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -33,7 +33,7 @@ public class BeiKeCrawlByChrome { def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) - def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM") + def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") //广州,佛山 重庆 @@ -207,9 +207,10 @@ public class BeiKeCrawlByChrome { if (!dbo) { parse(city, e, region, driver, null) - }else if(dbo && !dbo.lift){ - parse(city, e, region, driver, dbo._id) } +// else if(dbo && !dbo.lift){ +// parse(city, e, region, driver, dbo._id) +// } }else{ stop = true; break; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index c87a55634..52e1665f4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -33,7 +33,7 @@ public class LianjiaCrawlByChrome { def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) - def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM") + def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") static main(args) { def task = new LianjiaCrawlByChrome(); @@ -304,9 +304,10 @@ public class LianjiaCrawlByChrome { def dbo = findTrade(durl , "链家在线") if (!dbo) { parse(city, e, regionText, driver, null) - }else if(dbo && !dbo.lift){ - parse(city, e, regionText, driver, dbo._id) } +// else if(dbo && !dbo.lift){ +// parse(city, e, regionText, driver, dbo._id) +// } }else{ stop = true; break; @@ -353,7 +354,7 @@ public class LianjiaCrawlByChrome { detail.region = RegionData.formatRegion(city, detail.region) } if(!detail.region){ - def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+1, detailDoc.toString().size()-1) + def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+18, detailDoc.toString().size()-1) str = str.substring(0, str.indexOf("',")) def locArrs = str.split(",") def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0].toString().toDouble(), locArrs[1].toString().toDouble()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index f79d9ac1a..1d0ca1224 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -65,7 +65,7 @@ class DataStandardBatch extends BaseBatch{ def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7] def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) static { thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); matchErrorLogColl = CrawalDBUtil.getMatchErrorLogCol(); @@ -92,7 +92,7 @@ class DataStandardBatch extends BaseBatch{ cityMap.each{city,alias-> println "开始加载【${city}】" -// if(city == "福州"){ +// if(city == "石家庄"){ def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) districtMap.putAll(detailMap) // } @@ -458,13 +458,13 @@ class DataStandardBatch extends BaseBatch{ def addr = null; if(data.address){ - try{ - addr = DataUtil.formateAddress(data.city, data.address) - }catch (Exception e){ - addr = data.address; - } +// try{ +// addr = DataUtil.formateAddress(data.city, data.address) +// }catch (Exception e){ +// addr = data.address; +// } def copyObj = data - copyObj.address = addr +// copyObj.address = addr district = getMatchAddressFromMap(copyObj) if(district){ type = 8;