Merge branch 'script_gj' into 'master'

updadte

See merge request !688
This commit is contained in:
2023-07-19 22:36:57 +08:00
4 changed files with 18 additions and 16 deletions
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳5月份二手房成交数据汇总.xls", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳6月份二手房成交数据汇总.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -33,7 +33,7 @@ public class BeiKeCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
//广州,佛山 重庆
@@ -207,9 +207,10 @@ public class BeiKeCrawlByChrome {
if (!dbo) {
parse(city, e, region, driver, null)
}else if(dbo && !dbo.lift){
parse(city, e, region, driver, dbo._id)
}
// else if(dbo && !dbo.lift){
// parse(city, e, region, driver, dbo._id)
// }
}else{
stop = true;
break;
@@ -33,7 +33,7 @@ public class LianjiaCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
static main(args) {
def task = new LianjiaCrawlByChrome();
@@ -304,9 +304,10 @@ public class LianjiaCrawlByChrome {
def dbo = findTrade(durl , "链家在线")
if (!dbo) {
parse(city, e, regionText, driver, null)
}else if(dbo && !dbo.lift){
parse(city, e, regionText, driver, dbo._id)
}
// else if(dbo && !dbo.lift){
// parse(city, e, regionText, driver, dbo._id)
// }
}else{
stop = true;
break;
@@ -353,7 +354,7 @@ public class LianjiaCrawlByChrome {
detail.region = RegionData.formatRegion(city, detail.region)
}
if(!detail.region){
def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+1, detailDoc.toString().size()-1)
def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+18, detailDoc.toString().size()-1)
str = str.substring(0, str.indexOf("',"))
def locArrs = str.split(",")
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0].toString().toDouble(), locArrs[1].toString().toDouble())
@@ -65,7 +65,7 @@ class DataStandardBatch extends BaseBatch{
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static {
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
matchErrorLogColl = CrawalDBUtil.getMatchErrorLogCol();
@@ -92,7 +92,7 @@ class DataStandardBatch extends BaseBatch{
cityMap.each{city,alias->
println "开始加载【${city}】"
// if(city == "福州"){
// if(city == "石家庄"){
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
// }
@@ -458,13 +458,13 @@ class DataStandardBatch extends BaseBatch{
def addr = null;
if(data.address){
try{
addr = DataUtil.formateAddress(data.city, data.address)
}catch (Exception e){
addr = data.address;
}
// try{
// addr = DataUtil.formateAddress(data.city, data.address)
// }catch (Exception e){
// addr = data.address;
// }
def copyObj = data
copyObj.address = addr
// copyObj.address = addr
district = getMatchAddressFromMap(copyObj)
if(district){
type = 8;