This commit is contained in:
2022-09-20 22:06:34 +08:00
parent 242671f41b
commit ded7991788
10 changed files with 1182 additions and 1068 deletions
@@ -15,6 +15,7 @@ import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
@@ -32,6 +33,9 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
def startDate = 202206
def endDate = 202208
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
@@ -41,6 +45,7 @@ class BigdataDealBatch extends BaseBatch{
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -102,6 +107,9 @@ class BigdataDealBatch extends BaseBatch{
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
// def lianjiaPriceMap = findThirdPartyPrice("上海")
def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd')
def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
for(def i=0;i<rets.size();i++){
final def trade = rets[i]
fixedThreadPool.execute(new Runnable() {
@@ -109,7 +117,8 @@ class BigdataDealBatch extends BaseBatch{
try {
totalCount.addAndGet(1)
if(source == "youda"){
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap)
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, lianjiaPriceMap)
if(ret){
result << ret
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
@@ -135,147 +144,176 @@ class BigdataDealBatch extends BaseBatch{
return result
}
def findThirdPartyPrice(def city) {
def q = new BasicDBObject()
q.put("city", city)
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap){
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte', endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for (def db : dbcursor) {
def price = null
if (db.lianjia_esf_price && db.lianjia_esf_price > 3000) {
price = db.lianjia_esf_price
}
if (!price && db.beike_esf_price && db.beike_esf_price > 3000) {
price = db.beike_esf_price
}
if(price){
map.put(db.region+"##"+db.name, price)
}
}
return map
}
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def lianjiaPriceMap){
def ret = []
def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
if(!obj){
return ret;
}
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
def data = distCodeMap.get(trade.fjRegion)
if(data){
ret << data.district_code
ret << data.district
}else{
ret << ""
ret << ""
}
def street = ""
def streetNumber = ""
if(trade.address){
if(trade.address.toString().contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
def price = lianjiaPriceMap.get(obj.region+"##"+obj.name)
if(price){
def mod = java.lang.Math.abs((trade.price-price)*100/price)
if(mod<=10){
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
def data = distCodeMap.get(trade.fjRegion)
if(data){
ret << data.district_code
ret << data.district
}else{
ret << ""
ret << ""
}
}
if(!street){
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
def street = ""
def streetNumber = ""
if(trade.address){
if(trade.address.toString().contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}else{
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
if(!street){
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}else{
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
}
}
if(!street){
if(obj.address){
if(obj.address.toString().contains("弄")){
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
if(number){
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
streetNumber = number
if(!street){
if(obj.address){
if(obj.address.toString().contains("弄")){
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
if(number){
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
streetNumber = number
}
}else{
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
if(jsonObj && jsonObj.searchAddress){
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
ret << street
ret << streetNumber
if(obj){
ret << obj._id.toString()
ret << trade.fjName
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
ret << ""//期数
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
if(roomNum && roomNum.contains("0")){
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
}else{
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
if(jsonObj && jsonObj.searchAddress){
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
ret << trade.floor
}
ret << ""
ret << trade.height
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
def buildType = ''
if(arrs){
buildType = arrs[0]
}
ret << buildType
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(buildType, trade.room, trade.hall)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.price
ret << trade.totalPrice * 10000
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
ret << "02"
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << "99"
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.totalPrice * 10000
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
}
}
ret << street
ret << streetNumber
if(obj){
ret << obj._id.toString()
ret << trade.fjName
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
ret << ""//期数
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
if(roomNum && roomNum.contains("0")){
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
}else{
ret << trade.floor
}
ret << ""
ret << trade.height
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
def buildType = ''
if(arrs){
buildType = arrs[0]
}
ret << buildType
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(buildType, trade.room, trade.hall)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.price
ret << trade.totalPrice * 10000
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
ret << "02"
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << "99"
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.totalPrice * 10000
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
@@ -543,11 +581,14 @@ class BigdataDealBatch extends BaseBatch{
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
q.put("weekend", new BasicDBObject('$gte', weekend))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("weekend",-1))
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"@"+d.district_name, d.avg_price)
def price = map.get(d.region+"##"+d.district_name)
if(!price && d.value_type==2){
map.put(d.region+"##"+d.district_name, d.avg_price)
}
}
return map
}
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-07-22", "2022-08-24")
def prices = CcbCommonUtils.findPersonPrice(city, "2022-08-24", "2022-09-21")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
File diff suppressed because it is too large Load Diff
@@ -75,13 +75,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","5月总量","6月总量","7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","6月总量","7月总量","8月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2022-05-01"
def date = "2022-06-01"
def endDate = "2022-07-01"
def newDate = "2022-08-01"
def startDate = "2022-06-01"
def date = "2022-07-01"
def endDate = "2022-08-01"
def newDate = "2022-09-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -754,7 +754,7 @@ class CommonDatautils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","森望芳草绿岸")
// q.put("name","湖畔佳苑")
DBObject keys = new BasicDBObject();
if(fields){
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2022-07-01"
static def statDate = "2022-08-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser();//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2022年9月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -284,8 +284,8 @@ class YouDaDataParser {
}
def esfParser(){
def files = ["D:\\数据\\友达\\2022年8月\\查询1.DBF"]
def esfParser(path){
def files = [path]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -86,7 +86,7 @@ class DistrictIndexBatch extends BaseBatch{
city_avg_total_price = 531.2615913
}
if(city_avg_total_price){
calcute(city, dists, code, city_avg_total_price)
// calcute(city, dists, code, city_avg_total_price)
calcuteIndex(city, dists, code, city_avg_total_price)
}
@@ -250,7 +250,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-01", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-07-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e, regionText)
@@ -469,7 +469,7 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def citys= ["石家庄","贵阳"]
def citys= ["苏州"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
@@ -36,13 +36,13 @@ class PaimaiDataSend {
def task = new PaimaiDataSend()
// task.jdRunByMongo()
// task.jdRun()
// task.aliZcRun()
task.aliZcRun2()
//// task.aliZcRun()
// task.aliZcRun2()
// task.aliFpRun()
// task.rmfysszcRun()
// task.checkUrl()
// task.paimaiSendToEs()
task.paimaiSendToEs()
}
@@ -66,7 +66,8 @@ class PaimaiDataSend {
def rmfysszcRun(){
def index = 0
def q = new BasicDBObject()
q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"]))
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"]))
q.put("province","广东省")
// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml")
def dbcursor = rmfysszc_coll.find(q).batchSize(3000)
for(def d : dbcursor){
@@ -149,10 +150,12 @@ class PaimaiDataSend {
def jdRun(){
def index = 0
def q = new BasicDBObject()
q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"]))
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"]))
q.put("province","广东")
// q.put("parser_status", new BasicDBObject('$ne',3))
// q.put("url","https://paimai.jd.com/115669796")
def dbcursor = jd_coll.find(q).batchSize(3000)
def rets = []
for(def d : dbcursor){
index++
println "jd:"+index
@@ -175,16 +178,49 @@ class PaimaiDataSend {
result.put("notice_detail_content", d.notice_detail_content.content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
rets << json
if(rets.size()==10000){
println "共:"+rets.size()
sendJD(rets)
rets.clear()
}
}
def udata = new BasicDBObject()
udata.put("parser_status", 3)
udata.put("m_date", new Date())
jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
if(rets.size()>0){
println "共:"+rets.size()
sendJD(rets)
}
}
def sendJD(def rets){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def json = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 3)
udata.put("m_date", new Date())
jd_coll.update(new BasicDBObject("_id", json.get("url")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def aliZcRun(){
def pm_q = new BasicDBObject()
pm_q.put("city",new BasicDBObject('$in', ["南京", "宿迁","苏州"]))
@@ -294,7 +330,8 @@ class PaimaiDataSend {
def aliFpRun(){
def index = 0
def q = new BasicDBObject()
q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
q.put("province","广东")
// q.put("parser_status", new BasicDBObject('$ne',2))
// q.put("url","https://sf-item.taobao.com/sf_item/631662596036.htm")
//https://sf-item.taobao.com/sf_item/657131607486.htm
@@ -374,10 +411,10 @@ class PaimaiDataSend {
def index = 0
def q = new BasicDBObject()
// q.put("city",new BasicDBObject('$in', ["苏州","宿迁","南京"]))
q.put("city","上海")
q.put("province","广东")
// q.put("parser_status", new BasicDBObject('$ne',1))
// q.put("region","姑苏")
q.put("url","https://susong-item.taobao.com/auction/652481594083.htm")
// q.put("url","https://susong-item.taobao.com/auction/652481594083.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
@@ -476,8 +513,8 @@ class PaimaiDataSend {
def getPaimaiDatas(){
def q = new BasicDBObject()
q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-15")))
def dbcursor = paimai43_COLL.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
@@ -25,7 +25,7 @@ ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def q = new BasicDBObject()
// q.put("province",new BasicDBObject('$in', ["江苏","江苏省"]))
q.put("status",new BasicDBObject('$ne', 1))
def dbcursor = school_coll.find(q).batchSize(5gdc,rfyhyerhtrbhjfxcs- vvv000)
def dbcursor = school_coll.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
JSON json = new JSONObject()