updadte
This commit is contained in:
+1
-1
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳5月份二手房成交数据汇总.xls", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳6月份二手房成交数据汇总.xls", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
|
||||
+4
-3
@@ -33,7 +33,7 @@ public class BeiKeCrawlByChrome {
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
|
||||
|
||||
//广州,佛山 重庆
|
||||
@@ -207,9 +207,10 @@ public class BeiKeCrawlByChrome {
|
||||
|
||||
if (!dbo) {
|
||||
parse(city, e, region, driver, null)
|
||||
}else if(dbo && !dbo.lift){
|
||||
parse(city, e, region, driver, dbo._id)
|
||||
}
|
||||
// else if(dbo && !dbo.lift){
|
||||
// parse(city, e, region, driver, dbo._id)
|
||||
// }
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
|
||||
+5
-4
@@ -33,7 +33,7 @@ public class LianjiaCrawlByChrome {
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
@@ -304,9 +304,10 @@ public class LianjiaCrawlByChrome {
|
||||
def dbo = findTrade(durl , "链家在线")
|
||||
if (!dbo) {
|
||||
parse(city, e, regionText, driver, null)
|
||||
}else if(dbo && !dbo.lift){
|
||||
parse(city, e, regionText, driver, dbo._id)
|
||||
}
|
||||
// else if(dbo && !dbo.lift){
|
||||
// parse(city, e, regionText, driver, dbo._id)
|
||||
// }
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
@@ -353,7 +354,7 @@ public class LianjiaCrawlByChrome {
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
if(!detail.region){
|
||||
def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+1, detailDoc.toString().size()-1)
|
||||
def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+18, detailDoc.toString().size()-1)
|
||||
str = str.substring(0, str.indexOf("',"))
|
||||
def locArrs = str.split(",")
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0].toString().toDouble(), locArrs[1].toString().toDouble())
|
||||
|
||||
+8
-8
@@ -65,7 +65,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
|
||||
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
static {
|
||||
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
matchErrorLogColl = CrawalDBUtil.getMatchErrorLogCol();
|
||||
@@ -92,7 +92,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
|
||||
cityMap.each{city,alias->
|
||||
println "开始加载【${city}】"
|
||||
// if(city == "福州"){
|
||||
// if(city == "石家庄"){
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||
districtMap.putAll(detailMap)
|
||||
// }
|
||||
@@ -458,13 +458,13 @@ class DataStandardBatch extends BaseBatch{
|
||||
|
||||
def addr = null;
|
||||
if(data.address){
|
||||
try{
|
||||
addr = DataUtil.formateAddress(data.city, data.address)
|
||||
}catch (Exception e){
|
||||
addr = data.address;
|
||||
}
|
||||
// try{
|
||||
// addr = DataUtil.formateAddress(data.city, data.address)
|
||||
// }catch (Exception e){
|
||||
// addr = data.address;
|
||||
// }
|
||||
def copyObj = data
|
||||
copyObj.address = addr
|
||||
// copyObj.address = addr
|
||||
district = getMatchAddressFromMap(copyObj)
|
||||
if(district){
|
||||
type = 8;
|
||||
|
||||
Reference in New Issue
Block a user