This commit is contained in:
2023-06-20 11:22:46 +08:00
parent 1fb6796a05
commit 3da40ce36f
15 changed files with 241 additions and 56 deletions
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳4月份二手房成交数据汇总.xls", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳5月份二手房成交数据汇总.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -51,7 +51,19 @@ for(def d : datas){
map.region = d['B']
def roomStr = d['E']
if(roomStr){
map.room = roomStr.toString().toDouble().toInteger()
if(roomStr.contains("-")){
def arrs = roomStr.split("-")
try{
map.room = arrs[0].toString().toInteger()
}catch(Exception e){
println "room error"
}
map.hall = arrs[1].toString().toInteger()
map.toilet = arrs[3].toString().toInteger()
}else{
map.room = roomStr.toString().toDouble().toInteger()
}
}
map.area = d['F'].toString().toDouble()
@@ -59,7 +71,13 @@ for(def d : datas){
map.avg_price = (int)(d['H'].toString().toDouble())
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
def trade_date = d['J']
map.trade_date = trade_date
if(trade_date){
map.trade_date = trade_date
}else{
def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
}
map.source = "深房家族"
map.create_date = new Date()
map.m_date = new Date()
@@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
@@ -77,9 +77,8 @@ class BigdataCommunityBatch extends BaseBatch{
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成->导Oracle"
// delPgData()
// saveToPgSql(rows)
delPgData()
saveToPgSql(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
@@ -61,9 +61,9 @@ class BigdataCitypriceBatch extends BaseBatch{
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
cityRun(citys, startDate, endDate, cityCodeMap)
regionRun(citys, startDate, endDate, cityCodeMap)
// districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
// cityRun(citys, startDate, endDate, cityCodeMap)
// regionRun(citys, startDate, endDate, cityCodeMap)
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
}
def cityRun(citys, startDate, endDate, cityCodeDataMap){
@@ -40,19 +40,25 @@ class HailanTest {
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new HailanTest();
task.run();
}
def run(){
def inputfile = "D:\\数据\\微元素\\排查表.xlsx"
match(inputfile)
def inputfile = "D:\\数据\\微元素\\test.xlsx"
// addThirdPrice(inputfile)
// match(inputfile)
// tradeAvgPrice(inputfile)
// caluateTotalPrice(inputfile)
// addPrice(inputfile)
// addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx")
// addHouse(inputfile)
// stat(inputfile)
// hailanStat(inputfile)
addFzFjwPrice(inputfile)
}
def addHouse(inputfile){
@@ -702,4 +708,60 @@ class HailanTest {
}
return prices;
}
def tradeAvgPrice(inputfile){
ExcelUtils.modify(inputfile, { row ->
if(row['I'] && row['I'] == "4f4dac0a84ae1852a920ad39"){
def dist = CommonDatautils.findDistrictById(row['I'])
if(dist){
def tradeAvgPrice = CommonDatautils.findTrades(dist, "2023-02-01", "2023-06-01");
row['J'] = (int)tradeAvgPrice
}
}
})
}
def addThirdPrice(path){
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
ExcelUtils.multiThreadingModify(path, { row ->
def cityMap = CommonDatautils.findThirdPartyPrice("福州", row['F'], row['G'], startDate, endDate)
def priceMap = cityMap.get("福州"+"##"+row['F']+"##"+row['G'])
if(priceMap){
if(priceMap.containsKey("lianjia_esf_price")){
row['K'] = (int)(priceMap.get("lianjia_esf_price")*0.95)
}
}
})
}
def caluateTotalPrice(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(row['J']){
row['L'] = (int)(row['J'].toString().toDouble()*row['B'].toString().toDouble())
}else if(row['K']){
row['L'] = (int)(row['K'].toString().toDouble()*row['B'].toString().toDouble())
}
})
}
def addFzFjwPrice(inputfile){
ExcelUtils.modify(inputfile, { row ->
if(row['B']) {
def q = new BasicDBObject()
q.put("city", "福州")
q.put("region", row['A'])
q.put("name", row['B'])
q.put("month", 202304)
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
if (obj) {
row['D'] = obj.avg_price
}
}
})
}
}
@@ -1022,7 +1022,7 @@ class CommonDatautils {
q.put("cat", cat);
q.put("region", region)
q.put("name", name);
q.put("status", 0)
q.put("visible", 0)
DBObject keys = new BasicDBObject();
if(fields){
@@ -3801,5 +3801,74 @@ class CommonDatautils {
}
static def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.avg_price){
total_price = total_price + d.avg_price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
return avg_price
}
static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.city+"##"+db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("lianjia_esf_price", db.beike_esf_price)
}
}
return map
}
}
@@ -25,7 +25,7 @@ class FzDataResultStat {
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
// task.regionPriceTrendRun()
task.regionPriceTrendRun()
//// task.margeExcel()
// task.saveCsvData()
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202305.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-04-01"
static def statDate = "2023-05-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年5月\\查询1.DBF");//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年6月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -65,7 +65,7 @@ class AddDistrictElevatorBatch extends BaseBatch{
}
def newHouses = analyElevator(haveHouses, dist.estate_type2)
if(newHouses && newHouses.size()>=3 && newHouses.size()>=haveHouses.size()*0.6){
println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 10层以上:"+newHouses.size()
println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 筛选样本量:"+newHouses.size()
BasicDBObject udata = new BasicDBObject()
if(dist.estate_type2 == "别墅"){
udata.put("is_elevator", false)
@@ -289,20 +289,26 @@ public class BeiKeCrawlByChrome {
}
detail.source = "贝壳找房"
def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","")
def detailDiv1 = JQueryUtils.select(".introContent .base", detailDoc)
def detailDiv2 = JQueryUtils.select(".introContent .transaction", detailDoc)
def liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","")
if(liftStr){
detail.lift = DataFormatUtils.parseLift(liftStr)
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
}
detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","")
detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构",""))
detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属",""))
detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限",""))
detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯",""))
detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构",""))
detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代",""))
detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型",""))
def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("㎡",""))
detail.room_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(户型结构)").text().replace("户型结构",""))
detail.elevator = DataFormatUtils.getValue(detailDiv1.select("li:contains(配备电梯)").text().replace("配备电梯",""))
detail.build_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑结构)").text().replace("建筑结构",""))
detail.build_year = DataFormatUtils.getValue(detailDiv1.select("li:contains(建成年代)").text().replace("建成年代",""))
detail.build_type = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑类型)").text().replace("建筑类型",""))
def floorStr = DataFormatUtils.getValue(detailDiv1.select("li:contains(所在楼层)").text().replace("㎡",""))
detail.trade_property = detailDiv2.select("li:contains(交易权属)").text().replace("交易权属","")
detail.property_ownership = DataFormatUtils.getValue(detailDiv2.select("li:contains(房权所属)").text().replace("房权所属",""))
detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途",""))
detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途",""))
detail.housing_age = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋年限)").text().replace("房屋年限",""))
if(detail.total_price && detail.avg_price){
detail.url = detailUrl
@@ -483,7 +489,7 @@ public class BeiKeCrawlByChrome {
elemPassword.sendKeys("12qwaszx");
btn.submit(); // 点击行为;
Thread.sleep(20000)
Thread.sleep(10000)
}catch(Exception e){
Thread.sleep(15000)
}
@@ -297,16 +297,23 @@ public class LianjiaCrawlByChrome {
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
def dbo = findTrade(durl , "链家在线")
if (!dbo) {
parse(city, e, regionText, driver)
if(dealDate){
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
println "1..."+new Date()
def dbo = findTrade(durl , "链家在线")
println "2..."+new Date()
if (!dbo) {
parse(city, e, regionText, driver)
}
}else{
stop = true;
break;
}
}else{
stop = true;
break;
println url +" 成交时间有误!"
}
}
if(page>=100){
stop = true;
@@ -326,8 +333,9 @@ public class LianjiaCrawlByChrome {
if(price){
def detailUrl = e.select(".title a").attr("href")
driver.get(detailUrl)
println "3..."+new Date()
def detailDoc = checkDocument(driver, detailUrl);
println "4..."+new Date()
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
@@ -386,20 +394,22 @@ public class LianjiaCrawlByChrome {
detail.source = "链家在线"
detail.url = detailUrl
def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","")
def detailDiv = JQueryUtils.select(".introContent", detailDoc)
def liftStr = detailDiv.select("li:contains(梯户比例)").text().replace("梯户比例","")
if(liftStr){
detail.lift = DataFormatUtils.parseLift(liftStr)
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
}
detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","")
detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构",""))
detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属",""))
detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限",""))
detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯",""))
detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构",""))
detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代",""))
detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型",""))
def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("所在楼层",""))
detail.trade_property = detailDiv.select("li:contains(交易权属)").text().replace("交易权属","")
detail.room_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(户型结构)").text().replace("户型结构",""))
detail.property_ownership = DataFormatUtils.getValue( detailDiv.select("li:contains(房权所属)").text().replace("房权所属",""))
detail.housing_age = DataFormatUtils.getValue( detailDiv.select("li:contains(房屋年限)").text().replace("房屋年限",""))
detail.elevator = DataFormatUtils.getValue( detailDiv.select("li:contains(配备电梯)").text().replace("配备电梯",""))
detail.build_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑结构)").text().replace("建筑结构",""))
detail.build_year = DataFormatUtils.getValue( detailDiv.select("li:contains(建成年代)").text().replace("建成年代",""))
detail.build_type = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑类型)").text().replace("建筑类型",""))
def floorStr = DataFormatUtils.getValue( detailDiv.select("li:contains(所在楼层)").text().replace("所在楼层",""))
if(detail.total_price && detail.avg_price){
def houseInfo = e.select(".houseInfo").text()
@@ -417,7 +427,9 @@ public class LianjiaCrawlByChrome {
detail.create_date = new Date()
detail.m_date = new Date()
println detail
println "5..."+new Date()
DEAL_PRICE_COLL.save(detail)
println "6..."+new Date()
}
}
}
@@ -28,9 +28,18 @@ class DayUpdateDistRefreshBatch extends BaseBatch{
if(dists){
for(def d : dists){
def distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields)
def distObj = null
if(d.comm_id){
distObj = CommonDatautils.findDistrictById(d.comm_id)
}else{
distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields)
}
if(distObj && distObj._id){
FzDataUtils.converDistrict(distObj, null, null);
if(d.type && d.type==1){
FzDataUtils.removeById(d.comm_id)
}else{
FzDataUtils.converDistrict(distObj, null, null);
}
println new Date().toString()+" 【处理完成】 "+d.city +" "+d.region+" "+d.name
updateTaskStatus(d.id, 2);
}
@@ -57,7 +66,7 @@ class DayUpdateDistRefreshBatch extends BaseBatch{
def q = new BasicDBObject()
q.put("status", 5);//未处理
TASK_COLL.find(q).each {d->
rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name]
rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name, "type":d.type, "comm_id":d.comm_id]
}
return rets;
}
@@ -119,6 +119,13 @@ class FzDataUtils {
}
}
static def removeById(id){
def del_data = new BasicDBObject()
del_data.put("num_id", id)
DIST_COLL.remove(del_data)
}
static def saveDistrict(def m){
DBObject q = new BasicDBObject();
q.put("num_id",m.num_id)
@@ -92,8 +92,11 @@ class DataStandardBatch extends BaseBatch{
cityMap.each{city,alias->
println "开始加载【${city}】"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
// if(city == "福州"){
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
// }
}
while(flag){
@@ -833,7 +836,7 @@ class DataStandardBatch extends BaseBatch{
}
}
}
if(!dbo){
if(!dbo && isStandardRoad(copyObj.address)){
result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address);
if(result.size()==1){
dbo = result[0];
@@ -920,7 +923,7 @@ class DataStandardBatch extends BaseBatch{
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", SHORT_DATE)
// qdata.put("city","福州")
qdata.put("city","福州")
// qdata.put("source","安居客")
// qdata.put("name",Pattern.compile("龚家宅.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
@@ -42,7 +42,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
}
@Override
void process(Object args) {
println SHORT_DATE
def citys = CommonDatautils.findAllCity();
def weekend = DateUtils.getFirstDayOfWeek(new Date());