Merge remote-tracking branch 'origin/master'

This commit is contained in:
2022-08-09 18:22:23 +08:00
9 changed files with 199 additions and 37 deletions
@@ -450,7 +450,7 @@ class BigdataForsaleBatch extends BaseBatch{
DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type&& seawater.estate_type.contains("商住")){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
@@ -71,7 +71,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
// }
// }
def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"])
for(def dist : districts){
def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
@@ -112,7 +112,7 @@ class DistrictPriceExportBatch extends BaseBatch{
if(dbo){
def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString())
curPrice = dbo.get("avg_price")
if(lastMonthPrice && curPrice){
if(lastMonthPrice && curPrice && dbo.value_type!=1){
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
if(Math.abs(markup)>3){
def randMarkup = nextDouble(0, 3)
@@ -23,6 +23,9 @@ class ThirdPriceExportBatch extends BaseBatch{
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
def charset = "GBK"
static main(args) {
@@ -55,8 +58,12 @@ class ThirdPriceExportBatch extends BaseBatch{
def districts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name"]);
def result = []
def index = 0
def lastMonthPriceMap = findFzDistrictStat("福州", curDate)
for(def dist : districts){
def rets = findThirdPrice(dist.city, dist.region, dist.name)
def key = dist.city +"##"+ dist.region+"##"+ dist.name
def rets = findThirdPrice(dist.city, dist.region, dist.name)
def curPrice = lastMonthPriceMap.get(key)
index ++
if(rets){
for(def o : rets){
@@ -68,46 +75,84 @@ class ThirdPriceExportBatch extends BaseBatch{
row << dist.region
row << dist.name
row << o.s_date
row << o.zhugezhaofang_esf_price
if(o.zhugezhaofang_esf_price){
def zgf_price = getLastThirdPrice(o.zhugezhaofang_esf_price, curPrice)
row << zgf_price
if(zgf_price){
add = true;
}
row << o.anjuke_esf_price
if(o.anjuke_esf_price){
add = true;
}
row << o.soufang_esf_price
if(o.soufang_esf_price){
// if(o.zhugezhaofang_esf_price){
// add = true;
// }
def ajk_price = getLastThirdPrice(o.anjuke_esf_price, curPrice)
row << ajk_price
if(ajk_price){
add = true;
}
// if(o.anjuke_esf_price){
// add = true;
// }
row << o.lianjia_esf_price
if(o.lianjia_esf_price){
def sf_price = getLastThirdPrice(o.soufang_esf_price, curPrice)
row << sf_price
if(sf_price){
add = true;
}
// if(o.soufang_esf_price){
// add = true;
// }
def lj_price = getLastThirdPrice(o.lianjia_esf_price, curPrice)
row << lj_price
if(lj_price){
add = true;
}
// if(o.lianjia_esf_price){
// add = true;
// }
row << o.cityhouse_esf_price
if(o.cityhouse_esf_price){
def city_price = getLastThirdPrice(o.cityhouse_esf_price, curPrice)
row << city_price
if(city_price){
add = true;
}
// if(o.cityhouse_esf_price){
// add = true;
// }
row << o.fangtan_esf_price
if(o.fangtan_esf_price){
add = true;
}
row << o.fangdoudou_esf_price
if(o.fangdoudou_esf_price){
add = true;
}
row << o.maitian_esf_price
if(o.maitian_esf_price){
def ft_price = getLastThirdPrice(o.fangtan_esf_price, curPrice)
row << ft_price
if(ft_price){
add = true;
}
// if(o.fangtan_esf_price){
// add = true;
// }
row << o.get("58tc_esf_price")
if(o.get("58tc_esf_price")){
def fdd_price = getLastThirdPrice(o.fangdoudou_esf_price, curPrice)
row <<fdd_price
if(fdd_price){
add = true;
}
// if(o.fangdoudou_esf_price){
// add = true;
// }
def mt_price = getLastThirdPrice(o.maitian_esf_price, curPrice)
row << mt_price
if(mt_price){
add = true;
}
// if(o.maitian_esf_price){
// add = true;
// }
def tc_price = getLastThirdPrice(o.get("58tc_esf_price"), curPrice)
row << tc_price
if(tc_price){
add = true;
}
// if(o.get("58tc_esf_price")){
// add = true;
// }
if(add){
result<< row
@@ -121,6 +166,21 @@ class ThirdPriceExportBatch extends BaseBatch{
}
def getLastThirdPrice(otherPrice, fjwPrice){
if(otherPrice && fjwPrice){
def modPrecent = Math.abs(otherPrice - fjwPrice)*100/fjwPrice
if(modPrecent<=30){
return otherPrice
}
}else{
if(otherPrice){
return otherPrice
}
}
return null
}
def findThirdPrice(def city, def region, def name){
DBObject obj = new BasicDBObject();
obj.put("city", city);
@@ -214,4 +274,18 @@ class ThirdPriceExportBatch extends BaseBatch{
return sb.toString()
}
def findFzDistrictStat(city,lastMonthDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("month", lastMonthDate)
def dbcursor = FZ_DISTRICT_STAT_COLL.find(q)
def priceMap = [:]
for(def d : dbcursor){
def key = d.city +"##"+d.region+"##"+d.name
priceMap.put(key, d.avg_price)
}
return priceMap
}
}
@@ -107,6 +107,12 @@ class FangjiaClientUtils {
}
public static def saveToEsPaimai(def data){
String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data)
HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/paiMai", jsonStr)
}
public static Map<String, Object> JsonToMap(JSONObject json) {
Map<String, Object> columnValMap = new HashMap<String, Object>();
Set<Object> jsonKeys = json.keySet();
@@ -32,6 +32,7 @@ public class BeiKeCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
static main(args) {
def task = new BeiKeCrawlByChrome();
@@ -160,7 +161,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-05-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -375,13 +376,13 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
// if(e.text().contains("福州")){
if(e.text().contains("福州")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
//cityMap.put(e.text(), "https:"+href)
// }
}
}
}
return list
@@ -34,7 +34,7 @@ public class LianjiaCrawlByChrome {
static main(args) {
def task = new LianjiaCrawlByChrome();
// task.run()
task.run()
// task.run1()
task.addDealRegion();
task.standardRegion()
@@ -238,7 +238,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-05-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-06-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -442,7 +442,7 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def citys= ["安庆"]
def citys= ["福州"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.paimai
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
@@ -32,13 +33,14 @@ class PaimaiDataSend {
static main(args) {
def task = new PaimaiDataSend()
task.jdRun()
// task.jdRun()
// task.aliZcRun()
// task.aliZcRun2()
// task.aliFpRun()
// task.rmfysszcRun()
// task.checkUrl()
task.paimaiSendToEs()
}
@@ -298,6 +300,85 @@ class PaimaiDataSend {
ali_zc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
}
}
def paimaiSendToEs(){
def rets = getPaimaiDatas()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def m = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def flag = FangjiaClientUtils.saveToEsPaimai(m)
println totalCount.get()+" "+m.city +" "+m.name +" "+flag
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def getPaimaiDatas(){
def q = new BasicDBObject()
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
def dbcursor = paimai43_COLL.find(q).batchSize(1000)
def rets = []
for(def d : dbcursor){
def o = [:]
o.addPrice = d.addPrice
o.agencyName = d.agencyName
o.area = d.area
o.buildYear = d.buildYear
o.callCount = d.callCount
o.city = d.city
o.court = d.court
o.delayCount = d.delayCount
o.delayWeekly = d.delayWeekly
o.depositPrice = d.depositPrice
o.enrollCount = d.enrollCount
o.estateType = d.estateType
o.evaluatePrice = d.evaluatePrice
o.finishDate = d.finishDate
o.height = d.height
o.houseNum = d.houseNum
o.id = d.id
o.introduce = d.introduce
// o.location = d.location
o.lookCount = d.lookCount
o.marketPrice = d.marketPrice
o.matchAddress = d.matchAddress
o.matchCity = d.matchCity
o.matchName = d.matchName
o.matchRegion = d.matchRegion
o.person = d.person
o.preemptivePerson = d.preemptivePerson
o.pricePkWeekly = d.pricePkWeekly
o.province = d.province
o.rateDiscount = d.rateDiscount
o.region = d.region
o.roomNum = d.roomNum
o.sellOffAdvancePay = d.sellOffAdvancePay
o.source = d.source
o.stage = d.stage
o.startDate = d.startDate
o.startPrice = d.startPrice
o.status = d.status
o.tel = d.tel
o.title = d.title
o.tradePrice = d.tradePrice
o.type = d.type
o.unitNum = d.unitNum
o.url = d.url
rets << o
}
return rets;
}
}
@@ -23,7 +23,7 @@ class SynchDistrictPrice {
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
//,"城市房产":"城市房产"
def esSources = ["365淘房":"365淘房"]//"贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客PC端":"安居客","搜房":"搜房","吉屋":"吉屋"
def esSources = ["365淘房":"365淘房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","吉屋":"吉屋"]//
static main(args) {
def task = new SynchDistrictPrice()
// task.run()
@@ -46,7 +46,7 @@ class SynchDistrictPrice {
def findAllEsData(city,esSource, mgSource){
def names = new HashSet()
def rets = []
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>'2022-06-15'", {d->
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>'2022-08-01'", {d->
def s = d.city +"#"+d.region+"#"+d.name
if(!names.contains(s)){
names << s