updadte
This commit is contained in:
+1
-1
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
|||||||
/**
|
/**
|
||||||
* 深圳公众号成交数据录入
|
* 深圳公众号成交数据录入
|
||||||
*/
|
*/
|
||||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳5月份二手房成交数据汇总.xls", 0, 1)
|
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳6月份二手房成交数据汇总.xls", 0, 1)
|
||||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||||
|
|
||||||
|
|||||||
+4
-3
@@ -33,7 +33,7 @@ public class BeiKeCrawlByChrome {
|
|||||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
|
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||||
|
|
||||||
|
|
||||||
//广州,佛山 重庆
|
//广州,佛山 重庆
|
||||||
@@ -207,9 +207,10 @@ public class BeiKeCrawlByChrome {
|
|||||||
|
|
||||||
if (!dbo) {
|
if (!dbo) {
|
||||||
parse(city, e, region, driver, null)
|
parse(city, e, region, driver, null)
|
||||||
}else if(dbo && !dbo.lift){
|
|
||||||
parse(city, e, region, driver, dbo._id)
|
|
||||||
}
|
}
|
||||||
|
// else if(dbo && !dbo.lift){
|
||||||
|
// parse(city, e, region, driver, dbo._id)
|
||||||
|
// }
|
||||||
}else{
|
}else{
|
||||||
stop = true;
|
stop = true;
|
||||||
break;
|
break;
|
||||||
|
|||||||
+5
-4
@@ -33,7 +33,7 @@ public class LianjiaCrawlByChrome {
|
|||||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
|
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new LianjiaCrawlByChrome();
|
def task = new LianjiaCrawlByChrome();
|
||||||
@@ -304,9 +304,10 @@ public class LianjiaCrawlByChrome {
|
|||||||
def dbo = findTrade(durl , "链家在线")
|
def dbo = findTrade(durl , "链家在线")
|
||||||
if (!dbo) {
|
if (!dbo) {
|
||||||
parse(city, e, regionText, driver, null)
|
parse(city, e, regionText, driver, null)
|
||||||
}else if(dbo && !dbo.lift){
|
|
||||||
parse(city, e, regionText, driver, dbo._id)
|
|
||||||
}
|
}
|
||||||
|
// else if(dbo && !dbo.lift){
|
||||||
|
// parse(city, e, regionText, driver, dbo._id)
|
||||||
|
// }
|
||||||
}else{
|
}else{
|
||||||
stop = true;
|
stop = true;
|
||||||
break;
|
break;
|
||||||
@@ -353,7 +354,7 @@ public class LianjiaCrawlByChrome {
|
|||||||
detail.region = RegionData.formatRegion(city, detail.region)
|
detail.region = RegionData.formatRegion(city, detail.region)
|
||||||
}
|
}
|
||||||
if(!detail.region){
|
if(!detail.region){
|
||||||
def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+1, detailDoc.toString().size()-1)
|
def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+18, detailDoc.toString().size()-1)
|
||||||
str = str.substring(0, str.indexOf("',"))
|
str = str.substring(0, str.indexOf("',"))
|
||||||
def locArrs = str.split(",")
|
def locArrs = str.split(",")
|
||||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0].toString().toDouble(), locArrs[1].toString().toDouble())
|
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0].toString().toDouble(), locArrs[1].toString().toDouble())
|
||||||
|
|||||||
+8
-8
@@ -65,7 +65,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
|
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
|
||||||
|
|
||||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||||
static {
|
static {
|
||||||
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||||
matchErrorLogColl = CrawalDBUtil.getMatchErrorLogCol();
|
matchErrorLogColl = CrawalDBUtil.getMatchErrorLogCol();
|
||||||
@@ -92,7 +92,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
|
|
||||||
cityMap.each{city,alias->
|
cityMap.each{city,alias->
|
||||||
println "开始加载【${city}】"
|
println "开始加载【${city}】"
|
||||||
// if(city == "福州"){
|
// if(city == "石家庄"){
|
||||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||||
districtMap.putAll(detailMap)
|
districtMap.putAll(detailMap)
|
||||||
// }
|
// }
|
||||||
@@ -458,13 +458,13 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
|
|
||||||
def addr = null;
|
def addr = null;
|
||||||
if(data.address){
|
if(data.address){
|
||||||
try{
|
// try{
|
||||||
addr = DataUtil.formateAddress(data.city, data.address)
|
// addr = DataUtil.formateAddress(data.city, data.address)
|
||||||
}catch (Exception e){
|
// }catch (Exception e){
|
||||||
addr = data.address;
|
// addr = data.address;
|
||||||
}
|
// }
|
||||||
def copyObj = data
|
def copyObj = data
|
||||||
copyObj.address = addr
|
// copyObj.address = addr
|
||||||
district = getMatchAddressFromMap(copyObj)
|
district = getMatchAddressFromMap(copyObj)
|
||||||
if(district){
|
if(district){
|
||||||
type = 8;
|
type = 8;
|
||||||
|
|||||||
Reference in New Issue
Block a user