Merge branch 'script_gj' into 'master'

update

See merge request !720
This commit is contained in:
2023-10-07 16:49:18 +08:00
25 changed files with 529 additions and 128 deletions
@@ -1,5 +1,7 @@
package com.fangjia.script.ana
import com.fangjia.script.util.ExcelUtils
import java.util.Date;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -40,9 +42,46 @@ public class Test {
// //test.run("district", "上海");
// test.analyDistrictPk("district", "天津");//小区统计与小区的对比
// println new Date(new ObjectId('5469bb4cda29ee73010fa577').getTime())
test.addTestData();
// test.addTestData();
test.findRepeatGuidingPrice()
}
def findRepeatGuidingPrice(){
def q = new BasicDBObject();
q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2023-03-01", "yyyy-MM-dd")))
def dbcursor = GUIDING_PRICE_COLL.find(q)
def map = [:]
for(def d : dbcursor){
def key = d.city +"##"+d.region+"##"+d.name
def rets = map.get(key)
if(!rets){
rets = []
map.put(key, rets)
}
rets << d
}
def datas = []
datas << ["id", "城市","区域","名称","添加人","指导价","日期"]
map.each{key,list->
if(list.size()>1){
def userNames = []
for(def ret : list){
userNames << ret.guiding_person
}
if(userNames.contains("付贵生")){
for(def r : list){
if(r.guiding_person!="付贵生"){
GUIDING_PRICE_COLL.remove(new BasicDBObject("_id", r._id))
}
}
}
}
}
// ExcelUtils.write("d:\\数据\\福州\\指导价.xlsx",datas, "datas")
}
public void addTestData(){
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳7月份二手房成交数据汇总.xls", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳8月份二手房成交数据汇总.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -74,7 +74,7 @@ for(def d : datas){
if(trade_date){
map.trade_date = trade_date
}else{
def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31")
def trade_date1 = DateUtils.getRandomDate("2023-08-01", "2023-08-31")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
}
@@ -39,7 +39,7 @@ class BigdataDealBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com", "幸福里":"xflapp.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
@@ -37,7 +37,7 @@ class CcbCommonUtils {
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42","xflapp.com":"43"]
public static def specialCityMap = ["阿坝州":"阿坝","阿拉善盟":"阿拉善","黄南州":"黄南","巴音郭楞":"巴州","兴安盟":"兴安","锡林郭勒盟":"锡林郭勒","凉山州":"凉山","西双版纳州":"西双版纳","德宏州":"德宏","迪庆州":"迪庆","海北州":"海北","海西州":"海西","博尔塔拉":"博州","甘孜州":"甘孜","玉树州":"玉树","果洛州":"果洛","文山州":"文山","怒江州":"怒江","克孜勒苏":"克州"]
@@ -392,12 +392,12 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all
def month = 202307
def month = 202308
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty2(city, 202308)
def rets = findPeGteThirdParty2(city, 202309)
def total = 0
println city +" "+rets.size()
if(rets){
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-07-26", "2023-08-25")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-08-25", "2023-09-27")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -2,6 +2,7 @@ package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.RandomUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
@@ -1287,10 +1288,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\9.25人工指导价.xlsx"
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['A']){
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['A']){
// def region = RegionData.formatRegion(d['B'], d['C'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
@@ -1299,18 +1300,18 @@ def fileName = "D:\\
// d['H'] = dist._id.toString()
// }
// }
//// if(!d['F']){
//// def dist = CommonDatautils.findDistrictById(d['A'])
//////
//// if(dist){
////// d['H'] = dist.region
////// d['G'] = dist.name
//// d['F'] = dist.status
//// }
//// }
//
// }
//},0)
if(!d['K']){
def dist = CommonDatautils.findDistrictById(d['E'])
if(dist){
d['K'] = dist.region
// d['I'] = dist.name
// d['J'] = dist.status
// def val = d['F'].toString().toDouble().toInteger() + RandomUtil.random(-100,500)
// d['K'] = val.toInteger()
}
}
}
},0)
@@ -1373,30 +1374,30 @@ def findPgDealPrice(c, pgsqlInstance, startDate){
return count
}
citys = CcbCommonUtils.citys_all
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
def rows = []
def headers = []
headers << "城市"
headers.addAll(dates)
rows << headers
for(def c : citys){
def row = []
row << c
def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c))
// for(def d : dates){
def startDate = "2022-07-01"
def count = findPgDealPrice(c, pgsqlInstance, startDate)
row << count*100/disCount
// def map = findSampleRate(pgsqlInstance, c, d)
// row << map.third
// }
rows << row
}
ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方")
//citys = CcbCommonUtils.citys_all
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
////def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
//def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
//def rows = []
//def headers = []
//headers << "城市"
//headers.addAll(dates)
//rows << headers
//for(def c : citys){
// def row = []
// row << c
// def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c))
//// for(def d : dates){
// def startDate = "2022-07-01"
// def count = findPgDealPrice(c, pgsqlInstance, startDate)
// row << count*100/disCount
//
//// def map = findSampleRate(pgsqlInstance, c, d)
//// row << map.third
//// }
// rows << row
//}
//ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方")
@@ -302,7 +302,11 @@ class CommonDatautils {
q.put("cat", cat);
q.put("city", city);
if(startDate){
q.put("m_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def datequery = new BasicDBObject('$gte', startDate)
if(endDate){
datequery.append('$lt',endDate)
}
q.put("m_date", datequery)
}
DBObject keys = new BasicDBObject();
@@ -97,7 +97,7 @@ class DistrictInfoExportBatch extends BaseBatch{
saveOrUpdateFzDist(districts)
if(day == 15){
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, DateUtils.parseFromTightYYYYMMDD("2022-01-01"), endMDate);
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, DateUtils.parseFromTightYYYYMMDD("2022-01-01"), null);
remarkRemoveDist(removeDists)
}else{
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, startMDate, endMDate);
@@ -201,6 +201,9 @@ class DistrictInfoExportBatch extends BaseBatch{
def udate = new BasicDBObject()
udate.put("status",1)
udate.put("m_date", rd.m_date)
udate.put("region", rd.region)
udate.put("name", rd.name)
udate.put("estate_type2", rd.estate_type2)
FZ_DIST_COLL.update(new BasicDBObject('_id',rd.get('_id')), new BasicDBObject('$set', udate));
}
}
@@ -131,6 +131,7 @@ class DistrictPriceExportBatch extends BaseBatch{
for(def rMap : districts){
def curPrice = 0
def dbo = null
if(curPriceMap.containsKey(rMap.get("_id").toString())){
dbo = curPriceMap.get(rMap.get("_id").toString())
curPrice = dbo.avg_price
@@ -99,8 +99,10 @@ class DistrictSellStatBatch extends BaseBatch {
println index+" "+city +" "+districts.size()+" "+ (new Date()).toString()
def startDate = DateUtils.getFirstDayOfMonth(new Date(), -1);
def endDate = DateUtils.getLastDayOfMonth(new Date(), -1);
if(districts.size()>0){
def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价
def weekendPriceMap = findPriceMap(city, startDate, endDate)//上个月小区均价
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价
def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价
@@ -245,11 +247,12 @@ class DistrictSellStatBatch extends BaseBatch {
}
def findPriceMap(city, weekend){
def findPriceMap(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DISTRICT_STAT.find(q)
// q.put("district_name","金福花园")
q.put("weekend", new BasicDBObject('$lte', endDate).append('$gte', startDate))
def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject().append("weekend", -1))
def priceMap = [:]
for(def d : dbcursor){
if(city == "福州"){
@@ -257,7 +260,9 @@ class DistrictSellStatBatch extends BaseBatch {
d.avg_price = d.avg_price_2b
}
}
priceMap.put(d.region +"##"+d.district_name, d)
if(!priceMap.containsKey(d.region +"##"+d.district_name)){
priceMap.put(d.region +"##"+d.district_name, d)
}
}
return priceMap
}
@@ -22,6 +22,10 @@ class FzDataResultStat {
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
@@ -55,7 +59,8 @@ class FzDataResultStat {
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
// def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
def dists = getFzDistricts("福州", ["status"])
def idMaps = [:]
for(def d : dists){
idMaps.put(d._id.toString(), d.status==0?"有效":"无效")
@@ -82,7 +87,7 @@ class FzDataResultStat {
map2.put(key, obj)
}
def rows = []
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅","计算方式", "是否有效", "涨跌绝对值"]
def index = 0
map1.each {k,obj1->
def keys = k.split("##")
@@ -102,7 +107,12 @@ class FzDataResultStat {
if(obj1){
flag = idMaps.get(obj1.comm_id)
}
def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag]
def markup_abs = null
if(markup){
markup_abs = Math.abs(markup)
}
def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag, markup_abs]
rows << row
index++
println row
@@ -263,4 +273,24 @@ class FzDataResultStat {
q.put("month", lastMonth)
return FZ_DISTRICT_STAT_COLL.find(q).toList()
}
def getFzDistricts(def city, fields){
def q = new BasicDBObject()
q.put("city", city)
q.put("cat", "district")
def rets = []
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
FZ_DIST_COLL.find(q, keys).each{it->
rets << it
}
return rets
}
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202307.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202308.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-07-01"
static def statDate = "2023-08-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年8月\\查询1.DBF");//友达二手房存入mongo
task.esfParser("D:\\数据\\友达\\2023年9月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -14,12 +14,12 @@ class FieldStat {
static main(args) {
def task = new FieldStat();
// task.run()
// task.run1()
task.run1()
// task.run2()
// task.run3()
// task.offircRun()
task.distRun()
task.shopRun()
// task.distRun()
// task.shopRun()
}
def distRun(){
@@ -203,11 +203,15 @@ class FieldStat {
alias_total = alias_total + alias.size()
}
}
def precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
def precent = 0
if(districts){
precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
}
rets << [city, alias_total, alias_index, districts.size(), precent+"%"]
}
ExcelUtils.write("d:\\数据\\92个城市别名统计.xlsx", rets, "统计")
ExcelUtils.write("d:\\数据\\城市别名统计.xlsx", rets, "统计")
// println dist_total
// println alias_total
// println addr_alias_total
@@ -5,24 +5,38 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
//
//ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d->
// def opt = d['J']
// if(opt && opt.contains("别名")){
// def dist = CommonDatautils.findDistrictById(d['H'])
// if(dist){
// def alias = []
// if(dist.alias){
// alias = dist.alias
// }
// if(!alias.contains(d['C'])){
// alias << d['C']
// def udata = new BasicDBObject()
// udata.put("alias", alias)
// DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
// }
//
// }
// }
//})
ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d->
ExcelUtils.modify("D:\\数据\\厦门\\厦门户数(曹和付)-结果.xlsx",{d->
def opt = d['J']
if(opt && opt.contains("别名")){
def dist = CommonDatautils.findDistrictById(d['H'])
if(opt && opt.toString().toDouble().toInteger()==1){
def dist = CommonDatautils.findDistrict('厦门',d['G'],d['H'], "district")
if(dist){
def alias = []
if(dist.alias){
alias = dist.alias
}
if(!alias.contains(d['C'])){
alias << d['C']
def udata = new BasicDBObject()
udata.put("alias", alias)
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
}
def udata = new BasicDBObject()
udata.put("household_count", d['F'].toString().toDouble().toInteger())
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
}
}
})
@@ -39,17 +39,22 @@ class DistrictFieldAddBatch extends BaseBatch{
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
//
def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
def sourceMap = ["安居客":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
//"alias":"alias",
//ownership_type:权属类型 为商品房、经适房、动迁房等
//house_property 产权性质:国有产权 "house_property":"网站没有",
def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households",
"car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure",
"build_type":"build_type", "land_property":"ownership_type","live_date":"live_date",
"loop_line_info":"loop_line","loudong_count":"loudong_cnt"]//key:seaweed表字段 value:es表字段
// def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
// "virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households",
// "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure",
// "build_type":"build_type", "land_property":"ownership_type","live_date":"live_date",
// "loop_line_info":"loop_line","loudong_count":"loudong_cnt"]//key:seaweed表字段 value:es表字段
def fieldsMap = ["household_count":"households"]
def fieldsNameMap = ["estate_charge":"物业费","block":"板块","build_date":"建筑年代","estate_company":"物业公司","developer":"开发商",
"virescence_percent":"绿化率","cubage_percent":"容积率","build_area":"建筑面积","all_area":"占地面积","household_count":"户型",
@@ -88,8 +93,8 @@ class DistrictFieldAddBatch extends BaseBatch{
for(def c : citys){
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
if(totalCount){
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)
analysRets = addFieldFromMatch(c, totalCount, analysRets)
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)//融合
analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
}
}
println analysRets.toString()
@@ -106,11 +111,11 @@ class DistrictFieldAddBatch extends BaseBatch{
def matchRets2 = selectRestByCredit(matchRets1)
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
def updateRets = processFieldsFromMatch(c , matchRets2)
def updateRets = processFieldsFromMatch(c , matchRets2, esWebSource)
println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%"
def precentMap = fieldUpdate(totalCount, updateRets)
println c +" "+precentMap.toString()
updateDB(c, updateRets)
//updateDB(c, updateRets)
println "------------------------------------------------------------------------------"
def pre1 = 0
def pre2 = 0
@@ -199,11 +204,13 @@ class DistrictFieldAddBatch extends BaseBatch{
}
def processFieldsFromMatch(c,matchRets){
def processFieldsFromMatch(c,matchRets, esWebSource){
def fields = fieldsMap.keySet() as List
fields << "city"
fields << "estate_type2"
def fieldValueMap = [:]
def rets = []
rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"]
for(def matchDist : matchRets){
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
if(!dist){
@@ -212,11 +219,13 @@ class DistrictFieldAddBatch extends BaseBatch{
}
def newValues = strandField(c, dist, matchDist)
if(newValues){
rets << [matchDist.city, matchDist.region, matchDist.name, matchDist.address,newValues.get("household_count"), matchDist.match_region, matchDist.match_name, matchDist.credit, matchDist.flag]
println dist._id.toString()+" "+newValues.toString()
fieldValueMap.put(dist._id.toString(), newValues)
}
}
ExcelUtils.write("D:\\数据\\小区\\${c}户数.xlsx",rets,esWebSource)
return fieldValueMap;
}
@@ -332,7 +341,7 @@ class DistrictFieldAddBatch extends BaseBatch{
def findEsDatas(c, esWebSource){
def list = []
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r->
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}' ", {r->
list << r
})
return list
@@ -379,6 +388,7 @@ class DistrictFieldAddBatch extends BaseBatch{
try{
totalCount.addAndGet(1)
def ret = matching(data)
// def ret = newMatching(data)
if(ret){
// println totalCount.get() +" "+data.city+" "+data.name+" [Match] "+data.match_region+" "+data.match_name +" "+data.flag+" "+data.credit
rets << ret
@@ -399,7 +409,7 @@ class DistrictFieldAddBatch extends BaseBatch{
def selectRestByCredit(matchRets){
def matchRets1 = []
for(def r : matchRets){
if(r.credit && r.credit>=0.9){
if(r.credit && r.credit>0.9){
matchRets1 << r
}
}
@@ -458,6 +468,58 @@ class DistrictFieldAddBatch extends BaseBatch{
}
def newMatching(data){
def name = data.name
if(data.name && data.name.contains("·")){
name = data.name.replace("·", "")
}
def flag = "名称匹配"
def dist = matchDist(data.city, data.region, name)
if(!dist){
dist = CommonDatautils.findSeaweedByMappingAlias(data.city, data.region, name,"district")
flag = "映射别名匹配"
}
if(!dist) {
def rets = CommonDatautils.findDistrictByAlias(data.city, data.region, name, "district")
if(rets && rets.size()==1){
flag = "别名匹配"
dist = rets[0]
}
}
if(dist){
data.flag = flag
data.match_id = dist._id.toString()
data.match_region = dist.region
data.match_name = dist.name
data.credit = 0.95
}else{
def jsonObject = FangjiaClientUtils.newMatch(data.city, data.region+name, "property", false)
if(jsonObject!=null && jsonObject.credit>0){
if(data.address && isStandardRoad(data.address)){
jsonObject = FangjiaClientUtils.newMatch(data.city, data.address, "property", false)
}else{
data.flag = "匹配接口-地址"
}
}else{
data.flag = "匹配接口-名称"
}
if(jsonObject != null && jsonObject.credit>0) {
data.match_id = jsonObject.get("id")
data.match_region = jsonObject.get("district")
data.match_name = jsonObject.get("name")
data.credit = jsonObject.get("credit")
}
}
if(data.match_id){
return data
}
return null
}
def updateDB(c, updateRets){
for(def rm : updateRets) {
@@ -200,4 +200,22 @@ class FangjiaClientUtils {
return jsonObj;
}
public static def newMatch(city,address, property, useMap){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
// def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&category=${property}&useMap=${useMap}"
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
}
@@ -0,0 +1,42 @@
package com.fangjia.script.gj.district
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def city = "常州"
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"])
println districts.size()
def rets = []
rets << ["城市", "区域", "名称", "房价网ToC", "房价网ToB", "近三个月成交均价"]
def index = 0
for(def d : districts){
index++
def ret = []
def dbo = CommonDatautils.queryDistrictPrice(d.city, d.region, d.name, DateUtils.parseFromTightByPattern("2023-09-17","yyyy-MM-dd"))
ret << d.city
ret << d.region
ret << d.name
if(dbo){
ret << dbo.avg_price
ret << dbo.avg_price_2b
}
def map = CcbCommonUtils.findTrades(d, "2023-05-01","2023-09-01")
if(map){
ret << map.avg_price
}
rets << ret
println index
}
ExcelUtils.write("d:\\数据\\江南农商行\\常州小区价格.xlsx", rets, "数据")
@@ -15,7 +15,7 @@ class DistrictTypeUpdate {
}
def updateType(){
ExcelUtils.modify("D:\\数据\\厦门\\厦门银行涉及城市(物业类型可疑小区).xlsx",{d->
ExcelUtils.modify("D:\\数据\\厦门\\物业类型可疑小区(第二批).xlsx",{d->
def flag = d['F']
if(flag && flag.toString().toDouble().toInteger()==2){
def name = d['C']
@@ -43,6 +43,7 @@ public class LianjiaCrawlByChrome {
// task.addDealRegion();
// task.standardRegion()
// task.updadteDealPriceField()
}
def updadteDealPriceField(){
@@ -263,26 +264,27 @@ public class LianjiaCrawlByChrome {
}
def checkDocument(driver, url){
driver.get(url)
def s = driver.getPageSource()
def flag = true
while(flag){
try{
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
driver = getLogin(driver, url)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
}
}catch(Exception e){
driver.close()
driver = new ChromeDriver()
Thread.sleep(15000)
driver = getLogin(driver, url)
println "异常Url:"+url
try{
driver.get(url)
def s = driver.getPageSource()
def flag = true
while(flag){
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
driver = getLogin(driver, url)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
}
}
}catch(Exception e){
driver.close()
driver = new ChromeDriver()
Thread.sleep(15000)
driver = getLogin(driver, url)
println "异常Url:"+url
}
return driver;
}
@@ -581,11 +583,11 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityList = []
def citys= ["衢州"]
def citys= ["福州"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
if (!href.contains("fang.lianjia")){
cityList << [e.text(), href]
}
}
@@ -31,23 +31,24 @@ class StreetProcess {
task.run()
}
def run(){
def region = RegionData.formatRegion("三明市", "三元区")
println region
//
// def region = RegionData.formatRegion("三明市", "三元区")
// println region
// standard()
// matchStreet()
// synchTownship()
synchTownship()
}
def synchTownship(){
def q = new BasicDBObject()
// q.put("city_name","¸£ÖÝÊÐ")
q.put("shape",new BasicDBObject('$exists', true))
// q.put("shape",new BasicDBObject('$exists', true))
def dbcursor = TOWNSHIP.find(q)
for(def d : dbcursor){
def udata = new BasicDBObject()
udata.put("shape", d.shape)
// udata.put("shape", d.shape)
udata.put("area_station", d.area_station)
TOWNSHIP100.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
}
}
@@ -0,0 +1,158 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class PoiWash {
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_test", MAP_POI_URL)
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", MAP_POI_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
public static def townsMap = [:]
static main(args) {
def task = new PoiWash()
task.run()
}
def run(){
//"三明市",
def citys = ["宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"]
for(def city : citys){
townsMap = findAllTownship(city)
def datas = findPois(city)
println city +' '+datas.size()
process(city, datas)
println city +" 完成"
}
}
def process(city, results){
// def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
def updateTotalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
def count = results.size()
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def newAddr = washing(map)
if(map.address!=newAddr){
updateTotalCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("address", newAddr)
MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
}
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
// return rets;
}
def washing(data){
if(data.address){
def addr = data.address.replace('[]', '').replace(' ','').replace(data.province,'')
if(data.province){
addr = addr.replace(data.province,'')
}
if(data.city){
addr = addr.replace(data.city,'')
}
if(data.district){
addr = addr.replace(data.district,'')
}
if(data.township){
addr = addr.replace(data.township,'')
}
if(addr && addr.startsWith('-')){
addr = addr.replace('---','').replace('--','').replace('-','')
}
if(addr && addr.isNumber()){
return null;
}
if(addr){
def towns = townsMap.get(data.fjw_city)
for(def t : towns){
if(addr.contains(t)){
addr = addr.replace(t, "")
continue
}
}
}
return addr
}
return null;
}
def findPois(def city){
def q = new BasicDBObject()
q.put("city", city)
def keys = new BasicDBObject()
keys.put("_id", 1)
keys.put("province", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("township", 1)
keys.put("address", 1)
keys.put("fjw_city", 1)
def dbcursor = MAPPOI_CONN.find(q, keys)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
def findAllTownship(cityName){
def q = new BasicDBObject()
q.put("city_name", cityName)
def dbcursor = TOWNSHIP_CONN.find(q)
def map = [:]
for(def d : dbcursor){
def city = d.fangjia_city
def towns = map.get(city)
if(!towns){
towns = []
map.put(city, towns)
}
def alias = d.alias
towns << d.street_name
if(alias){
towns.addAll(alias)
}
towns.unique()
}
return map
}
}
@@ -25,6 +25,10 @@ class SeaweedDataSynch {
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
public static def provinceMap = CommonDatautils.findAllCityProvince()
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
public static def priceMap = null
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new SeaweedDataSynch()
@@ -35,7 +39,7 @@ class SeaweedDataSynch {
}
def run(){
priceMap = findAllDistrictSellPrice()
def citys = CommonDatautils.findAllCity()
def index = 0
for(def c : citys){
@@ -188,7 +192,20 @@ class SeaweedDataSynch {
if(dist.main_type_area && dist.main_type_area.toString().isNumber()){
map.main_house_area = DataHandleUtils.getDoubleRound(2, dist.main_type_area.toString().toFloat())
}
def dataMap = priceMap.get(dist._id.toString())
if(dataMap){
map.avg_price = dataMap.avg_price
}
// map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
return map
}
def findAllDistrictSellPrice(){
def q = new BasicDBObject()
q.put("date", lastMonthDate)
def priceMap = [:]
DISTRICT_SELL_STAT.find(q).each{
priceMap.put(it.commid, it)
}
return priceMap
}
}
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20230801", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230901", "yyyyMMdd")
def startDate = DateUtil.parseDate("20230901", "yyyyMMdd")
def endDate = DateUtil.parseDate("20231001", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -23,7 +23,7 @@ while (startDate.before(endDate)) {
def nextDate = calendar.getTime()
count++
///property/addressEvaluate /address/financialRating /v2/property/detail
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/address/financialRating"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/v2/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
// if (!hashSet.contains(result)) {
def item = [doc._id.toString(), "ÁÉÄþ׿ÐÅÆÕ»ÝÕ÷ÐÅ", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
@@ -39,6 +39,6 @@ while (startDate.before(endDate)) {
startDate = nextDate
}
ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信-202309.xlsx", data, "财力评级")
ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信-202310.xlsx", data, "楼盘评情")