Merge remote-tracking branch 'origin/master'
This commit is contained in:
+1
-1
@@ -450,7 +450,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
|
||||
if(estate_type2 && estate_type2=="普通住宅"){
|
||||
if(seawater.estate_type&& seawater.estate_type.contains("商住")){
|
||||
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
+1
-1
@@ -71,7 +71,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
// }
|
||||
// }
|
||||
|
||||
def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
|
||||
def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"])
|
||||
|
||||
for(def dist : districts){
|
||||
def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
|
||||
|
||||
+1
-1
@@ -112,7 +112,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
if(dbo){
|
||||
def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString())
|
||||
curPrice = dbo.get("avg_price")
|
||||
if(lastMonthPrice && curPrice){
|
||||
if(lastMonthPrice && curPrice && dbo.value_type!=1){
|
||||
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
|
||||
if(Math.abs(markup)>3){
|
||||
def randMarkup = nextDouble(0, 3)
|
||||
|
||||
+99
-25
@@ -23,6 +23,9 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
def charset = "GBK"
|
||||
|
||||
static main(args) {
|
||||
@@ -55,8 +58,12 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name"]);
|
||||
def result = []
|
||||
def index = 0
|
||||
def lastMonthPriceMap = findFzDistrictStat("福州", curDate)
|
||||
|
||||
for(def dist : districts){
|
||||
def rets = findThirdPrice(dist.city, dist.region, dist.name)
|
||||
def key = dist.city +"##"+ dist.region+"##"+ dist.name
|
||||
def rets = findThirdPrice(dist.city, dist.region, dist.name)
|
||||
def curPrice = lastMonthPriceMap.get(key)
|
||||
index ++
|
||||
if(rets){
|
||||
for(def o : rets){
|
||||
@@ -68,46 +75,84 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
row << dist.region
|
||||
row << dist.name
|
||||
row << o.s_date
|
||||
row << o.zhugezhaofang_esf_price
|
||||
if(o.zhugezhaofang_esf_price){
|
||||
def zgf_price = getLastThirdPrice(o.zhugezhaofang_esf_price, curPrice)
|
||||
row << zgf_price
|
||||
if(zgf_price){
|
||||
add = true;
|
||||
}
|
||||
row << o.anjuke_esf_price
|
||||
if(o.anjuke_esf_price){
|
||||
add = true;
|
||||
}
|
||||
row << o.soufang_esf_price
|
||||
if(o.soufang_esf_price){
|
||||
// if(o.zhugezhaofang_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
def ajk_price = getLastThirdPrice(o.anjuke_esf_price, curPrice)
|
||||
row << ajk_price
|
||||
if(ajk_price){
|
||||
add = true;
|
||||
}
|
||||
// if(o.anjuke_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
|
||||
row << o.lianjia_esf_price
|
||||
if(o.lianjia_esf_price){
|
||||
def sf_price = getLastThirdPrice(o.soufang_esf_price, curPrice)
|
||||
row << sf_price
|
||||
if(sf_price){
|
||||
add = true;
|
||||
}
|
||||
// if(o.soufang_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
def lj_price = getLastThirdPrice(o.lianjia_esf_price, curPrice)
|
||||
row << lj_price
|
||||
if(lj_price){
|
||||
add = true;
|
||||
}
|
||||
// if(o.lianjia_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
|
||||
row << o.cityhouse_esf_price
|
||||
if(o.cityhouse_esf_price){
|
||||
def city_price = getLastThirdPrice(o.cityhouse_esf_price, curPrice)
|
||||
row << city_price
|
||||
if(city_price){
|
||||
add = true;
|
||||
}
|
||||
// if(o.cityhouse_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
|
||||
row << o.fangtan_esf_price
|
||||
if(o.fangtan_esf_price){
|
||||
add = true;
|
||||
}
|
||||
row << o.fangdoudou_esf_price
|
||||
if(o.fangdoudou_esf_price){
|
||||
add = true;
|
||||
}
|
||||
row << o.maitian_esf_price
|
||||
if(o.maitian_esf_price){
|
||||
def ft_price = getLastThirdPrice(o.fangtan_esf_price, curPrice)
|
||||
row << ft_price
|
||||
if(ft_price){
|
||||
add = true;
|
||||
}
|
||||
// if(o.fangtan_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
|
||||
row << o.get("58tc_esf_price")
|
||||
if(o.get("58tc_esf_price")){
|
||||
def fdd_price = getLastThirdPrice(o.fangdoudou_esf_price, curPrice)
|
||||
row <<fdd_price
|
||||
if(fdd_price){
|
||||
add = true;
|
||||
}
|
||||
// if(o.fangdoudou_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
|
||||
def mt_price = getLastThirdPrice(o.maitian_esf_price, curPrice)
|
||||
row << mt_price
|
||||
if(mt_price){
|
||||
add = true;
|
||||
}
|
||||
// if(o.maitian_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
|
||||
def tc_price = getLastThirdPrice(o.get("58tc_esf_price"), curPrice)
|
||||
row << tc_price
|
||||
if(tc_price){
|
||||
add = true;
|
||||
}
|
||||
// if(o.get("58tc_esf_price")){
|
||||
// add = true;
|
||||
// }
|
||||
|
||||
if(add){
|
||||
result<< row
|
||||
@@ -121,6 +166,21 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def getLastThirdPrice(otherPrice, fjwPrice){
|
||||
if(otherPrice && fjwPrice){
|
||||
def modPrecent = Math.abs(otherPrice - fjwPrice)*100/fjwPrice
|
||||
if(modPrecent<=30){
|
||||
return otherPrice
|
||||
}
|
||||
}else{
|
||||
if(otherPrice){
|
||||
return otherPrice
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
def findThirdPrice(def city, def region, def name){
|
||||
DBObject obj = new BasicDBObject();
|
||||
obj.put("city", city);
|
||||
@@ -214,4 +274,18 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
return sb.toString()
|
||||
}
|
||||
|
||||
|
||||
def findFzDistrictStat(city,lastMonthDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("month", lastMonthDate)
|
||||
def dbcursor = FZ_DISTRICT_STAT_COLL.find(q)
|
||||
def priceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city +"##"+d.region+"##"+d.name
|
||||
priceMap.put(key, d.avg_price)
|
||||
}
|
||||
return priceMap
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+6
@@ -107,6 +107,12 @@ class FangjiaClientUtils {
|
||||
}
|
||||
|
||||
|
||||
public static def saveToEsPaimai(def data){
|
||||
String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data)
|
||||
HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/paiMai", jsonStr)
|
||||
}
|
||||
|
||||
|
||||
public static Map<String, Object> JsonToMap(JSONObject json) {
|
||||
Map<String, Object> columnValMap = new HashMap<String, Object>();
|
||||
Set<Object> jsonKeys = json.keySet();
|
||||
|
||||
+4
-3
@@ -32,6 +32,7 @@ public class BeiKeCrawlByChrome {
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
|
||||
|
||||
static main(args) {
|
||||
def task = new BeiKeCrawlByChrome();
|
||||
@@ -160,7 +161,7 @@ public class BeiKeCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-05-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
@@ -375,13 +376,13 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
// if(e.text().contains("福州")){
|
||||
if(e.text().contains("福州")){
|
||||
def cityMap = [:]
|
||||
cityMap.city = e.text()
|
||||
cityMap.url = "https:"+href
|
||||
list << cityMap
|
||||
//cityMap.put(e.text(), "https:"+href)
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return list
|
||||
|
||||
+3
-3
@@ -34,7 +34,7 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
// task.run()
|
||||
task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
task.standardRegion()
|
||||
@@ -238,7 +238,7 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-05-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-06-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
@@ -442,7 +442,7 @@ public class LianjiaCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
def citys= ["安庆"]
|
||||
def citys= ["福州"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
|
||||
+82
-1
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.alibaba.fastjson.JSON
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.temp.shenjian.util.MessageUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
@@ -32,13 +33,14 @@ class PaimaiDataSend {
|
||||
|
||||
static main(args) {
|
||||
def task = new PaimaiDataSend()
|
||||
task.jdRun()
|
||||
// task.jdRun()
|
||||
// task.aliZcRun()
|
||||
// task.aliZcRun2()
|
||||
// task.aliFpRun()
|
||||
// task.rmfysszcRun()
|
||||
// task.checkUrl()
|
||||
|
||||
task.paimaiSendToEs()
|
||||
}
|
||||
|
||||
|
||||
@@ -298,6 +300,85 @@ class PaimaiDataSend {
|
||||
ali_zc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
|
||||
def paimaiSendToEs(){
|
||||
def rets = getPaimaiDatas()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def m = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def flag = FangjiaClientUtils.saveToEsPaimai(m)
|
||||
println totalCount.get()+" "+m.city +" "+m.name +" "+flag
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def getPaimaiDatas(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
|
||||
def dbcursor = paimai43_COLL.find(q).batchSize(1000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
def o = [:]
|
||||
o.addPrice = d.addPrice
|
||||
o.agencyName = d.agencyName
|
||||
o.area = d.area
|
||||
o.buildYear = d.buildYear
|
||||
o.callCount = d.callCount
|
||||
o.city = d.city
|
||||
o.court = d.court
|
||||
o.delayCount = d.delayCount
|
||||
o.delayWeekly = d.delayWeekly
|
||||
o.depositPrice = d.depositPrice
|
||||
o.enrollCount = d.enrollCount
|
||||
o.estateType = d.estateType
|
||||
o.evaluatePrice = d.evaluatePrice
|
||||
o.finishDate = d.finishDate
|
||||
o.height = d.height
|
||||
o.houseNum = d.houseNum
|
||||
o.id = d.id
|
||||
o.introduce = d.introduce
|
||||
// o.location = d.location
|
||||
o.lookCount = d.lookCount
|
||||
o.marketPrice = d.marketPrice
|
||||
o.matchAddress = d.matchAddress
|
||||
o.matchCity = d.matchCity
|
||||
o.matchName = d.matchName
|
||||
o.matchRegion = d.matchRegion
|
||||
o.person = d.person
|
||||
o.preemptivePerson = d.preemptivePerson
|
||||
o.pricePkWeekly = d.pricePkWeekly
|
||||
o.province = d.province
|
||||
o.rateDiscount = d.rateDiscount
|
||||
o.region = d.region
|
||||
o.roomNum = d.roomNum
|
||||
o.sellOffAdvancePay = d.sellOffAdvancePay
|
||||
o.source = d.source
|
||||
o.stage = d.stage
|
||||
o.startDate = d.startDate
|
||||
o.startPrice = d.startPrice
|
||||
o.status = d.status
|
||||
o.tel = d.tel
|
||||
o.title = d.title
|
||||
o.tradePrice = d.tradePrice
|
||||
o.type = d.type
|
||||
o.unitNum = d.unitNum
|
||||
o.url = d.url
|
||||
rets << o
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -23,7 +23,7 @@ class SynchDistrictPrice {
|
||||
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
||||
|
||||
//,"城市房产":"城市房产"
|
||||
def esSources = ["365淘房":"365淘房"]//"贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客PC端":"安居客","搜房":"搜房","吉屋":"吉屋"
|
||||
def esSources = ["365淘房":"365淘房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","吉屋":"吉屋"]//
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
// task.run()
|
||||
@@ -46,7 +46,7 @@ class SynchDistrictPrice {
|
||||
def findAllEsData(city,esSource, mgSource){
|
||||
def names = new HashSet()
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>'2022-06-15'", {d->
|
||||
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>'2022-08-01'", {d->
|
||||
def s = d.city +"#"+d.region+"#"+d.name
|
||||
if(!names.contains(s)){
|
||||
names << s
|
||||
|
||||
Reference in New Issue
Block a user