update
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
package com.fangjia.script.ana
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
@@ -40,9 +42,46 @@ public class Test {
|
||||
// //test.run("district", "上海");
|
||||
// test.analyDistrictPk("district", "天津");//小区统计与小区的对比
|
||||
// println new Date(new ObjectId('5469bb4cda29ee73010fa577').getTime())
|
||||
test.addTestData();
|
||||
// test.addTestData();
|
||||
test.findRepeatGuidingPrice()
|
||||
}
|
||||
|
||||
def findRepeatGuidingPrice(){
|
||||
def q = new BasicDBObject();
|
||||
q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2023-03-01", "yyyy-MM-dd")))
|
||||
def dbcursor = GUIDING_PRICE_COLL.find(q)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city +"##"+d.region+"##"+d.name
|
||||
def rets = map.get(key)
|
||||
if(!rets){
|
||||
rets = []
|
||||
map.put(key, rets)
|
||||
}
|
||||
rets << d
|
||||
}
|
||||
|
||||
def datas = []
|
||||
datas << ["id", "城市","区域","名称","添加人","指导价","日期"]
|
||||
map.each{key,list->
|
||||
if(list.size()>1){
|
||||
def userNames = []
|
||||
for(def ret : list){
|
||||
userNames << ret.guiding_person
|
||||
}
|
||||
if(userNames.contains("付贵生")){
|
||||
for(def r : list){
|
||||
if(r.guiding_person!="付贵生"){
|
||||
GUIDING_PRICE_COLL.remove(new BasicDBObject("_id", r._id))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ExcelUtils.write("d:\\数据\\福州\\指导价.xlsx",datas, "datas")
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void addTestData(){
|
||||
|
||||
+2
-2
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳7月份二手房成交数据汇总.xls", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳8月份二手房成交数据汇总.xls", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
@@ -74,7 +74,7 @@ for(def d : datas){
|
||||
if(trade_date){
|
||||
map.trade_date = trade_date
|
||||
}else{
|
||||
def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31")
|
||||
def trade_date1 = DateUtils.getRandomDate("2023-08-01", "2023-08-31")
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com", "幸福里":"xflapp.com"]
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
+1
-1
@@ -37,7 +37,7 @@ class CcbCommonUtils {
|
||||
|
||||
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
|
||||
|
||||
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
|
||||
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42","xflapp.com":"43"]
|
||||
|
||||
public static def specialCityMap = ["阿坝州":"阿坝","阿拉善盟":"阿拉善","黄南州":"黄南","巴音郭楞":"巴州","兴安盟":"兴安","锡林郭勒盟":"锡林郭勒","凉山州":"凉山","西双版纳州":"西双版纳","德宏州":"德宏","迪庆州":"迪庆","海北州":"海北","海西州":"海西","博尔塔拉":"博州","甘孜州":"甘孜","玉树州":"玉树","果洛州":"果洛","文山州":"文山","怒江州":"怒江","克孜勒苏":"克州"]
|
||||
|
||||
|
||||
+2
-2
@@ -392,12 +392,12 @@ class DealPriceDataProcess {
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def month = 202307
|
||||
def month = 202308
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def set = new HashSet()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty2(city, 202308)
|
||||
def rets = findPeGteThirdParty2(city, 202309)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
if(rets){
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-07-26", "2023-08-25")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-08-25", "2023-09-27")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
|
||||
+40
-39
@@ -2,6 +2,7 @@ package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.RandomUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
@@ -1287,10 +1288,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx"
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\9.25人工指导价.xlsx"
|
||||
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['A']){
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['A']){
|
||||
// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
@@ -1299,18 +1300,18 @@ def fileName = "D:\\
|
||||
// d['H'] = dist._id.toString()
|
||||
// }
|
||||
// }
|
||||
//// if(!d['F']){
|
||||
//// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
//////
|
||||
//// if(dist){
|
||||
////// d['H'] = dist.region
|
||||
////// d['G'] = dist.name
|
||||
//// d['F'] = dist.status
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
// }
|
||||
//},0)
|
||||
if(!d['K']){
|
||||
def dist = CommonDatautils.findDistrictById(d['E'])
|
||||
if(dist){
|
||||
d['K'] = dist.region
|
||||
// d['I'] = dist.name
|
||||
// d['J'] = dist.status
|
||||
// def val = d['F'].toString().toDouble().toInteger() + RandomUtil.random(-100,500)
|
||||
// d['K'] = val.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
|
||||
|
||||
@@ -1373,30 +1374,30 @@ def findPgDealPrice(c, pgsqlInstance, startDate){
|
||||
return count
|
||||
}
|
||||
|
||||
citys = CcbCommonUtils.citys_all
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
//def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
|
||||
def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
|
||||
def rows = []
|
||||
def headers = []
|
||||
headers << "城市"
|
||||
headers.addAll(dates)
|
||||
rows << headers
|
||||
for(def c : citys){
|
||||
def row = []
|
||||
row << c
|
||||
def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c))
|
||||
// for(def d : dates){
|
||||
def startDate = "2022-07-01"
|
||||
def count = findPgDealPrice(c, pgsqlInstance, startDate)
|
||||
row << count*100/disCount
|
||||
|
||||
// def map = findSampleRate(pgsqlInstance, c, d)
|
||||
// row << map.third
|
||||
// }
|
||||
rows << row
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方")
|
||||
//citys = CcbCommonUtils.citys_all
|
||||
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
////def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
|
||||
//def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
|
||||
//def rows = []
|
||||
//def headers = []
|
||||
//headers << "城市"
|
||||
//headers.addAll(dates)
|
||||
//rows << headers
|
||||
//for(def c : citys){
|
||||
// def row = []
|
||||
// row << c
|
||||
// def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c))
|
||||
//// for(def d : dates){
|
||||
// def startDate = "2022-07-01"
|
||||
// def count = findPgDealPrice(c, pgsqlInstance, startDate)
|
||||
// row << count*100/disCount
|
||||
//
|
||||
//// def map = findSampleRate(pgsqlInstance, c, d)
|
||||
//// row << map.third
|
||||
//// }
|
||||
// rows << row
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方")
|
||||
|
||||
|
||||
|
||||
|
||||
+5
-1
@@ -302,7 +302,11 @@ class CommonDatautils {
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
if(startDate){
|
||||
q.put("m_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def datequery = new BasicDBObject('$gte', startDate)
|
||||
if(endDate){
|
||||
datequery.append('$lt',endDate)
|
||||
}
|
||||
q.put("m_date", datequery)
|
||||
}
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
|
||||
+4
-1
@@ -97,7 +97,7 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
saveOrUpdateFzDist(districts)
|
||||
|
||||
if(day == 15){
|
||||
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, DateUtils.parseFromTightYYYYMMDD("2022-01-01"), endMDate);
|
||||
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, DateUtils.parseFromTightYYYYMMDD("2022-01-01"), null);
|
||||
remarkRemoveDist(removeDists)
|
||||
}else{
|
||||
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, startMDate, endMDate);
|
||||
@@ -201,6 +201,9 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
def udate = new BasicDBObject()
|
||||
udate.put("status",1)
|
||||
udate.put("m_date", rd.m_date)
|
||||
udate.put("region", rd.region)
|
||||
udate.put("name", rd.name)
|
||||
udate.put("estate_type2", rd.estate_type2)
|
||||
FZ_DIST_COLL.update(new BasicDBObject('_id',rd.get('_id')), new BasicDBObject('$set', udate));
|
||||
}
|
||||
}
|
||||
|
||||
+1
@@ -131,6 +131,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
for(def rMap : districts){
|
||||
def curPrice = 0
|
||||
def dbo = null
|
||||
|
||||
if(curPriceMap.containsKey(rMap.get("_id").toString())){
|
||||
dbo = curPriceMap.get(rMap.get("_id").toString())
|
||||
curPrice = dbo.avg_price
|
||||
|
||||
+10
-5
@@ -99,8 +99,10 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
|
||||
println index+" "+city +" "+districts.size()+" "+ (new Date()).toString()
|
||||
|
||||
def startDate = DateUtils.getFirstDayOfMonth(new Date(), -1);
|
||||
def endDate = DateUtils.getLastDayOfMonth(new Date(), -1);
|
||||
if(districts.size()>0){
|
||||
def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价
|
||||
def weekendPriceMap = findPriceMap(city, startDate, endDate)//上个月小区均价
|
||||
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
|
||||
def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价
|
||||
def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价
|
||||
@@ -245,11 +247,12 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
|
||||
def findPriceMap(city, weekend){
|
||||
def findPriceMap(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def dbcursor = DISTRICT_STAT.find(q)
|
||||
// q.put("district_name","金福花园")
|
||||
q.put("weekend", new BasicDBObject('$lte', endDate).append('$gte', startDate))
|
||||
def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject().append("weekend", -1))
|
||||
def priceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(city == "福州"){
|
||||
@@ -257,7 +260,9 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
d.avg_price = d.avg_price_2b
|
||||
}
|
||||
}
|
||||
priceMap.put(d.region +"##"+d.district_name, d)
|
||||
if(!priceMap.containsKey(d.region +"##"+d.district_name)){
|
||||
priceMap.put(d.region +"##"+d.district_name, d)
|
||||
}
|
||||
}
|
||||
return priceMap
|
||||
}
|
||||
|
||||
+33
-3
@@ -22,6 +22,10 @@ class FzDataResultStat {
|
||||
|
||||
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new FzDataResultStat()
|
||||
task.priceMarkupRun();
|
||||
@@ -55,7 +59,8 @@ class FzDataResultStat {
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
|
||||
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||
def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
|
||||
// def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
|
||||
def dists = getFzDistricts("福州", ["status"])
|
||||
def idMaps = [:]
|
||||
for(def d : dists){
|
||||
idMaps.put(d._id.toString(), d.status==0?"有效":"无效")
|
||||
@@ -82,7 +87,7 @@ class FzDataResultStat {
|
||||
map2.put(key, obj)
|
||||
}
|
||||
def rows = []
|
||||
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
|
||||
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅","计算方式", "是否有效", "涨跌绝对值"]
|
||||
def index = 0
|
||||
map1.each {k,obj1->
|
||||
def keys = k.split("##")
|
||||
@@ -102,7 +107,12 @@ class FzDataResultStat {
|
||||
if(obj1){
|
||||
flag = idMaps.get(obj1.comm_id)
|
||||
}
|
||||
def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag]
|
||||
|
||||
def markup_abs = null
|
||||
if(markup){
|
||||
markup_abs = Math.abs(markup)
|
||||
}
|
||||
def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag, markup_abs]
|
||||
rows << row
|
||||
index++
|
||||
println row
|
||||
@@ -263,4 +273,24 @@ class FzDataResultStat {
|
||||
q.put("month", lastMonth)
|
||||
return FZ_DISTRICT_STAT_COLL.find(q).toList()
|
||||
}
|
||||
|
||||
def getFzDistricts(def city, fields){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("cat", "district")
|
||||
def rets = []
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
FZ_DIST_COLL.find(q, keys).each{it->
|
||||
rets << it
|
||||
}
|
||||
return rets
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202307.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202308.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+2
-2
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2023-07-01"
|
||||
static def statDate = "2023-08-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年8月\\查询1.DBF");//友达二手房存入mongo
|
||||
task.esfParser("D:\\数据\\友达\\2023年9月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
|
||||
@@ -14,12 +14,12 @@ class FieldStat {
|
||||
static main(args) {
|
||||
def task = new FieldStat();
|
||||
// task.run()
|
||||
// task.run1()
|
||||
task.run1()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
// task.offircRun()
|
||||
task.distRun()
|
||||
task.shopRun()
|
||||
// task.distRun()
|
||||
// task.shopRun()
|
||||
}
|
||||
|
||||
def distRun(){
|
||||
@@ -203,11 +203,15 @@ class FieldStat {
|
||||
alias_total = alias_total + alias.size()
|
||||
}
|
||||
}
|
||||
def precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
|
||||
def precent = 0
|
||||
if(districts){
|
||||
precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
|
||||
}
|
||||
|
||||
rets << [city, alias_total, alias_index, districts.size(), precent+"%"]
|
||||
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\92个城市别名统计.xlsx", rets, "统计")
|
||||
ExcelUtils.write("d:\\数据\\城市别名统计.xlsx", rets, "统计")
|
||||
// println dist_total
|
||||
// println alias_total
|
||||
// println addr_alias_total
|
||||
|
||||
+29
-15
@@ -5,24 +5,38 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
//
|
||||
//ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d->
|
||||
// def opt = d['J']
|
||||
// if(opt && opt.contains("别名")){
|
||||
// def dist = CommonDatautils.findDistrictById(d['H'])
|
||||
// if(dist){
|
||||
// def alias = []
|
||||
// if(dist.alias){
|
||||
// alias = dist.alias
|
||||
// }
|
||||
// if(!alias.contains(d['C'])){
|
||||
// alias << d['C']
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("alias", alias)
|
||||
// DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d->
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\厦门\\厦门户数(曹和付)-结果.xlsx",{d->
|
||||
def opt = d['J']
|
||||
if(opt && opt.contains("别名")){
|
||||
def dist = CommonDatautils.findDistrictById(d['H'])
|
||||
if(opt && opt.toString().toDouble().toInteger()==1){
|
||||
def dist = CommonDatautils.findDistrict('厦门',d['G'],d['H'], "district")
|
||||
if(dist){
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
alias = dist.alias
|
||||
}
|
||||
if(!alias.contains(d['C'])){
|
||||
alias << d['C']
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("household_count", d['F'].toString().toDouble().toInteger())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
|
||||
+76
-14
@@ -39,17 +39,22 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
|
||||
|
||||
//
|
||||
def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
|
||||
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
|
||||
def sourceMap = ["安居客":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
|
||||
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
|
||||
|
||||
|
||||
|
||||
//"alias":"alias",
|
||||
//ownership_type:权属类型 为商品房、经适房、动迁房等
|
||||
//house_property 产权性质:国有产权 "house_property":"网站没有",
|
||||
def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
|
||||
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households",
|
||||
"car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure",
|
||||
"build_type":"build_type", "land_property":"ownership_type","live_date":"live_date",
|
||||
"loop_line_info":"loop_line","loudong_count":"loudong_cnt"]//key:seaweed表字段 value:es表字段
|
||||
// def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
|
||||
// "virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households",
|
||||
// "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure",
|
||||
// "build_type":"build_type", "land_property":"ownership_type","live_date":"live_date",
|
||||
// "loop_line_info":"loop_line","loudong_count":"loudong_cnt"]//key:seaweed表字段 value:es表字段
|
||||
|
||||
|
||||
def fieldsMap = ["household_count":"households"]
|
||||
|
||||
def fieldsNameMap = ["estate_charge":"物业费","block":"板块","build_date":"建筑年代","estate_company":"物业公司","developer":"开发商",
|
||||
"virescence_percent":"绿化率","cubage_percent":"容积率","build_area":"建筑面积","all_area":"占地面积","household_count":"户型",
|
||||
@@ -88,8 +93,8 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
for(def c : citys){
|
||||
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
|
||||
if(totalCount){
|
||||
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)
|
||||
analysRets = addFieldFromMatch(c, totalCount, analysRets)
|
||||
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)//融合
|
||||
analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
|
||||
}
|
||||
}
|
||||
println analysRets.toString()
|
||||
@@ -106,11 +111,11 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
def matchRets2 = selectRestByCredit(matchRets1)
|
||||
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
|
||||
|
||||
def updateRets = processFieldsFromMatch(c , matchRets2)
|
||||
def updateRets = processFieldsFromMatch(c , matchRets2, esWebSource)
|
||||
println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%"
|
||||
def precentMap = fieldUpdate(totalCount, updateRets)
|
||||
println c +" "+precentMap.toString()
|
||||
updateDB(c, updateRets)
|
||||
//updateDB(c, updateRets)
|
||||
println "------------------------------------------------------------------------------"
|
||||
def pre1 = 0
|
||||
def pre2 = 0
|
||||
@@ -199,11 +204,13 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
def processFieldsFromMatch(c,matchRets){
|
||||
def processFieldsFromMatch(c,matchRets, esWebSource){
|
||||
def fields = fieldsMap.keySet() as List
|
||||
fields << "city"
|
||||
fields << "estate_type2"
|
||||
def fieldValueMap = [:]
|
||||
def rets = []
|
||||
rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"]
|
||||
for(def matchDist : matchRets){
|
||||
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
|
||||
if(!dist){
|
||||
@@ -212,11 +219,13 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
}
|
||||
def newValues = strandField(c, dist, matchDist)
|
||||
if(newValues){
|
||||
rets << [matchDist.city, matchDist.region, matchDist.name, matchDist.address,newValues.get("household_count"), matchDist.match_region, matchDist.match_name, matchDist.credit, matchDist.flag]
|
||||
println dist._id.toString()+" "+newValues.toString()
|
||||
fieldValueMap.put(dist._id.toString(), newValues)
|
||||
}
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\小区\\${c}户数.xlsx",rets,esWebSource)
|
||||
return fieldValueMap;
|
||||
}
|
||||
|
||||
@@ -332,7 +341,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
|
||||
def findEsDatas(c, esWebSource){
|
||||
def list = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r->
|
||||
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}' ", {r->
|
||||
list << r
|
||||
})
|
||||
return list
|
||||
@@ -379,6 +388,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = matching(data)
|
||||
// def ret = newMatching(data)
|
||||
if(ret){
|
||||
// println totalCount.get() +" "+data.city+" "+data.name+" [Match] "+data.match_region+" "+data.match_name +" "+data.flag+" "+data.credit
|
||||
rets << ret
|
||||
@@ -399,7 +409,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
def selectRestByCredit(matchRets){
|
||||
def matchRets1 = []
|
||||
for(def r : matchRets){
|
||||
if(r.credit && r.credit>=0.9){
|
||||
if(r.credit && r.credit>0.9){
|
||||
matchRets1 << r
|
||||
}
|
||||
}
|
||||
@@ -458,6 +468,58 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def newMatching(data){
|
||||
def name = data.name
|
||||
if(data.name && data.name.contains("·")){
|
||||
name = data.name.replace("·", "")
|
||||
}
|
||||
|
||||
def flag = "名称匹配"
|
||||
def dist = matchDist(data.city, data.region, name)
|
||||
|
||||
if(!dist){
|
||||
dist = CommonDatautils.findSeaweedByMappingAlias(data.city, data.region, name,"district")
|
||||
flag = "映射别名匹配"
|
||||
}
|
||||
if(!dist) {
|
||||
def rets = CommonDatautils.findDistrictByAlias(data.city, data.region, name, "district")
|
||||
if(rets && rets.size()==1){
|
||||
flag = "别名匹配"
|
||||
dist = rets[0]
|
||||
}
|
||||
}
|
||||
|
||||
if(dist){
|
||||
data.flag = flag
|
||||
data.match_id = dist._id.toString()
|
||||
data.match_region = dist.region
|
||||
data.match_name = dist.name
|
||||
data.credit = 0.95
|
||||
}else{
|
||||
def jsonObject = FangjiaClientUtils.newMatch(data.city, data.region+name, "property", false)
|
||||
if(jsonObject!=null && jsonObject.credit>0){
|
||||
if(data.address && isStandardRoad(data.address)){
|
||||
jsonObject = FangjiaClientUtils.newMatch(data.city, data.address, "property", false)
|
||||
}else{
|
||||
data.flag = "匹配接口-地址"
|
||||
}
|
||||
}else{
|
||||
data.flag = "匹配接口-名称"
|
||||
}
|
||||
if(jsonObject != null && jsonObject.credit>0) {
|
||||
data.match_id = jsonObject.get("id")
|
||||
data.match_region = jsonObject.get("district")
|
||||
data.match_name = jsonObject.get("name")
|
||||
data.credit = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
if(data.match_id){
|
||||
return data
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def updateDB(c, updateRets){
|
||||
|
||||
for(def rm : updateRets) {
|
||||
|
||||
+18
@@ -200,4 +200,22 @@ class FangjiaClientUtils {
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
public static def newMatch(city,address, property, useMap){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
// def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&category=${property}&useMap=${useMap}"
|
||||
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
def city = "常州"
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"])
|
||||
println districts.size()
|
||||
def rets = []
|
||||
rets << ["城市", "区域", "名称", "房价网ToC", "房价网ToB", "近三个月成交均价"]
|
||||
def index = 0
|
||||
for(def d : districts){
|
||||
index++
|
||||
def ret = []
|
||||
def dbo = CommonDatautils.queryDistrictPrice(d.city, d.region, d.name, DateUtils.parseFromTightByPattern("2023-09-17","yyyy-MM-dd"))
|
||||
ret << d.city
|
||||
ret << d.region
|
||||
ret << d.name
|
||||
if(dbo){
|
||||
ret << dbo.avg_price
|
||||
ret << dbo.avg_price_2b
|
||||
}
|
||||
|
||||
|
||||
def map = CcbCommonUtils.findTrades(d, "2023-05-01","2023-09-01")
|
||||
if(map){
|
||||
ret << map.avg_price
|
||||
}
|
||||
rets << ret
|
||||
println index
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\江南农商行\\常州小区价格.xlsx", rets, "数据")
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ class DistrictTypeUpdate {
|
||||
}
|
||||
|
||||
def updateType(){
|
||||
ExcelUtils.modify("D:\\数据\\厦门\\厦门银行涉及城市(物业类型可疑小区).xlsx",{d->
|
||||
ExcelUtils.modify("D:\\数据\\厦门\\物业类型可疑小区(第二批).xlsx",{d->
|
||||
def flag = d['F']
|
||||
if(flag && flag.toString().toDouble().toInteger()==2){
|
||||
def name = d['C']
|
||||
|
||||
+23
-21
@@ -43,6 +43,7 @@ public class LianjiaCrawlByChrome {
|
||||
// task.addDealRegion();
|
||||
// task.standardRegion()
|
||||
// task.updadteDealPriceField()
|
||||
|
||||
}
|
||||
|
||||
def updadteDealPriceField(){
|
||||
@@ -263,26 +264,27 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
def checkDocument(driver, url){
|
||||
driver.get(url)
|
||||
def s = driver.getPageSource()
|
||||
def flag = true
|
||||
while(flag){
|
||||
try{
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
}
|
||||
}catch(Exception e){
|
||||
driver.close()
|
||||
driver = new ChromeDriver()
|
||||
Thread.sleep(15000)
|
||||
driver = getLogin(driver, url)
|
||||
println "异常Url:"+url
|
||||
try{
|
||||
driver.get(url)
|
||||
def s = driver.getPageSource()
|
||||
def flag = true
|
||||
while(flag){
|
||||
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
driver.close()
|
||||
driver = new ChromeDriver()
|
||||
Thread.sleep(15000)
|
||||
driver = getLogin(driver, url)
|
||||
println "异常Url:"+url
|
||||
}
|
||||
return driver;
|
||||
}
|
||||
@@ -581,11 +583,11 @@ public class LianjiaCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityList = []
|
||||
def citys= ["衢州"]
|
||||
def citys= ["福州"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
if (!href.contains("fang.lianjia")){
|
||||
cityList << [e.text(), href]
|
||||
}
|
||||
}
|
||||
|
||||
+7
-6
@@ -31,23 +31,24 @@ class StreetProcess {
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
|
||||
def region = RegionData.formatRegion("三明市", "三元区")
|
||||
println region
|
||||
//
|
||||
// def region = RegionData.formatRegion("三明市", "三元区")
|
||||
// println region
|
||||
|
||||
// standard()
|
||||
// matchStreet()
|
||||
// synchTownship()
|
||||
synchTownship()
|
||||
}
|
||||
|
||||
def synchTownship(){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city_name","¸£ÖÝÊÐ")
|
||||
q.put("shape",new BasicDBObject('$exists', true))
|
||||
// q.put("shape",new BasicDBObject('$exists', true))
|
||||
def dbcursor = TOWNSHIP.find(q)
|
||||
for(def d : dbcursor){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", d.shape)
|
||||
// udata.put("shape", d.shape)
|
||||
udata.put("area_station", d.area_station)
|
||||
TOWNSHIP100.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class PoiWash {
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_test", MAP_POI_URL)
|
||||
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", MAP_POI_URL)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
|
||||
public static def townsMap = [:]
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiWash()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
//"三明市",
|
||||
def citys = ["宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"]
|
||||
for(def city : citys){
|
||||
townsMap = findAllTownship(city)
|
||||
def datas = findPois(city)
|
||||
println city +' '+datas.size()
|
||||
process(city, datas)
|
||||
println city +" 完成"
|
||||
}
|
||||
}
|
||||
|
||||
def process(city, results){
|
||||
// def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
def updateTotalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
|
||||
def count = results.size()
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def newAddr = washing(map)
|
||||
if(map.address!=newAddr){
|
||||
updateTotalCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("address", newAddr)
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
|
||||
}
|
||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
latch.await()
|
||||
|
||||
// return rets;
|
||||
}
|
||||
|
||||
def washing(data){
|
||||
if(data.address){
|
||||
def addr = data.address.replace('[]', '').replace(' ','').replace(data.province,'')
|
||||
|
||||
if(data.province){
|
||||
addr = addr.replace(data.province,'')
|
||||
}
|
||||
if(data.city){
|
||||
addr = addr.replace(data.city,'')
|
||||
}
|
||||
if(data.district){
|
||||
addr = addr.replace(data.district,'')
|
||||
}
|
||||
if(data.township){
|
||||
addr = addr.replace(data.township,'')
|
||||
}
|
||||
if(addr && addr.startsWith('-')){
|
||||
addr = addr.replace('---','').replace('--','').replace('-','')
|
||||
}
|
||||
if(addr && addr.isNumber()){
|
||||
return null;
|
||||
}
|
||||
|
||||
if(addr){
|
||||
def towns = townsMap.get(data.fjw_city)
|
||||
for(def t : towns){
|
||||
if(addr.contains(t)){
|
||||
addr = addr.replace(t, "")
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
return addr
|
||||
}
|
||||
return null;
|
||||
|
||||
}
|
||||
|
||||
def findPois(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
keys.put("province", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("township", 1)
|
||||
keys.put("address", 1)
|
||||
keys.put("fjw_city", 1)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q, keys)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findAllTownship(cityName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_name", cityName)
|
||||
def dbcursor = TOWNSHIP_CONN.find(q)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def city = d.fangjia_city
|
||||
def towns = map.get(city)
|
||||
if(!towns){
|
||||
towns = []
|
||||
map.put(city, towns)
|
||||
}
|
||||
def alias = d.alias
|
||||
towns << d.street_name
|
||||
if(alias){
|
||||
towns.addAll(alias)
|
||||
}
|
||||
towns.unique()
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
}
|
||||
+18
-1
@@ -25,6 +25,10 @@ class SeaweedDataSynch {
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
public static def provinceMap = CommonDatautils.findAllCityProvince()
|
||||
|
||||
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
public static def priceMap = null
|
||||
|
||||
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new SeaweedDataSynch()
|
||||
@@ -35,7 +39,7 @@ class SeaweedDataSynch {
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
priceMap = findAllDistrictSellPrice()
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
@@ -188,7 +192,20 @@ class SeaweedDataSynch {
|
||||
if(dist.main_type_area && dist.main_type_area.toString().isNumber()){
|
||||
map.main_house_area = DataHandleUtils.getDoubleRound(2, dist.main_type_area.toString().toFloat())
|
||||
}
|
||||
def dataMap = priceMap.get(dist._id.toString())
|
||||
if(dataMap){
|
||||
map.avg_price = dataMap.avg_price
|
||||
}
|
||||
// map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||
return map
|
||||
}
|
||||
def findAllDistrictSellPrice(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("date", lastMonthDate)
|
||||
def priceMap = [:]
|
||||
DISTRICT_SELL_STAT.find(q).each{
|
||||
priceMap.put(it.commid, it)
|
||||
}
|
||||
return priceMap
|
||||
}
|
||||
}
|
||||
|
||||
+4
-4
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20230801", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230901", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20230901", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20231001", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
@@ -23,7 +23,7 @@ while (startDate.before(endDate)) {
|
||||
def nextDate = calendar.getTime()
|
||||
count++
|
||||
///property/addressEvaluate /address/financialRating /v2/property/detail
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/address/financialRating"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/v2/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
|
||||
def result = JSONObject.fromObject(doc.res_map.result)
|
||||
// if (!hashSet.contains(result)) {
|
||||
def item = [doc._id.toString(), "ÁÉÄþ׿ÐÅÆÕ»ÝÕ÷ÐÅ", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
@@ -39,6 +39,6 @@ while (startDate.before(endDate)) {
|
||||
|
||||
startDate = nextDate
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信-202309.xlsx", data, "财力评级")
|
||||
ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信-202310.xlsx", data, "楼盘评情")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user