From e1fbc1ee6a5daea2fea1b11f89a0e7a01b467ee1 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:32:18 +0800 Subject: [PATCH] update --- .../ccbevaluation/AddThirdPriceSample.groovy | 82 +++++++++++-- .../fangjia/script/ccbevaluation/Test.groovy | 100 ++++++++-------- .../script/ccbevaluation/TradeStatLog.groovy | 21 +++- .../ccbevaluation/data/AddressCut.groovy | 97 +++++++++------ .../script/commData/CommonDatautils.groovy | 75 ++++++++++-- .../com/fangjia/script/gj/FiledStrand.groovy | 24 ++++ .../com/fangjia/script/gj/TradeMatch.groovy | 96 ++++++++++++--- .../script/gj/data_handle/DevProcess.groovy | 4 +- .../gj/data_handle/DevUpdateTestBatch.groovy | 10 +- .../gj/data_handle/GXDdistrictMatch.groovy | 28 ++++- .../data_handle/ThirdPriceMappingBatch.groovy | 9 +- .../gj/data_handle/TownMiddleProcess.groovy | 112 +++++++++++++----- .../script/trade/ImportIndexStats.groovy | 85 +++++++++---- 13 files changed, 556 insertions(+), 187 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy index 2684f90c2..a36d9d715 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy @@ -13,19 +13,24 @@ class AddThirdPriceSample { static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + public static final DBCollection NO_PRICE_DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com") + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + static main(args) { def task = new AddThirdPriceSample() - task.run() + task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据 +// task.addTrade() //添加成交 } def run(){ - for(def i=1;i<19;i++){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d-> - if(d['A'] && d['C']){ + for(def i=0;i<19;i++){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达参考价作为第三方入库.xlsx",{d-> + if(d['A'] && d['B']){ def dist = CommonDatautils.findDistrictById(d['A'].toString().trim()) if(dist){ - def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407) - def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger()) - + def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202408) +// def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger()) + def udata = new BasicDBObject("fccs_esf_price", d['B'].toString().toDouble().toInteger()) AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false) }else{ println d['A'] +" 没查到" @@ -37,4 +42,67 @@ class AddThirdPriceSample { } } + + def addTrade(){ + def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"] + def rets = findTrades(citys) +// DEAL_PRICE_COLL.insert(rets) + } + + def findTrades(citys){ + def q = new BasicDBObject() + q.put("city", new BasicDBObject('$in', citys)) + q.put("flag", 1) + def dbcursor = NO_PRICE_DEAL_PRICE_COLL.find(q) + def rets = [] + def index = 0 + for(def db : dbcursor){ + index++ + def udata = new BasicDBObject() + + udata.put("_id",db._id) + udata.put("url",db.url) + udata.put("area",db.area) + udata.put("avg_price",db.avg_price.toString().toDouble().toInteger()) + udata.put("build_structure",db.build_structure) + udata.put("build_type",db.build_type) + udata.put("build_year",db.build_year) + udata.put("city",db.city) + udata.put("direction", db.direction) + udata.put("district_name",db.district_name) + udata.put("elevator",db.elevator) + udata.put("fitment",db.fitment) + udata.put("floor_info",db.floor_info) + udata.put("hall",db.hall.toString().toInteger()) + udata.put("height",db.height.toString().toInteger()) + udata.put("housing_age",db.housing_age) + udata.put("lift",db.lift.toString().toInteger()) + udata.put("lift_household",db.lift_household.toString().toInteger()) + udata.put("property_ownership",db.property_ownership) + udata.put("region",db.region) + udata.put("room", db.room.toString().toInteger()) + udata.put("room_structure",db.room_structure) + udata.put("source",db.source) + udata.put("total_price",db.total_price.toString().toInteger()) + udata.put("trade_date",db.trade_date) + udata.put("trade_property",db.trade_property) + udata.put("house_using",db.house_using) + udata.put("heating_type",db.heating_type) + udata.put("community_id",db.community_id) + udata.put("create_date",db.create_time) + udata.put("m_date", new Date()) + udata.put("address",db.address) + udata.put("match_city",db.match_city) + udata.put("match_region",db.match_region) + udata.put("match_name",db.match_name) + udata.put("match_id",db.match_id) + udata.put("price_flag", "国信达成交-推导") + + DEAL_PRICE_COLL.save(udata) + println index +// rets << udata + } + return rets + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 0d36523e6..fed8e9757 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -631,36 +631,36 @@ def findAllData(sqlInstance236, citycode){ //},0) -ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d-> - if(d['A'] && d['B']){ - def dbo = CommonDatautils.findDistrictById(d['A']) - if(dbo){ - def alias = null - if(!dbo.alias){ - alias = [] - }else{ - alias = dbo.alias - } - if(!alias.contains(d['B'])){ - alias << d['B'] - } - if(alias){ - def udata = new BasicDBObject() - udata.put("alias", alias) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", dbo._id) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - }else{ - println d['A']+" "+"别名已存在" - } - - }else{ - println dbo._id.toString() +" 没找到" - } - } -}) +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d-> +// if(d['A'] && d['B']){ +// def dbo = CommonDatautils.findDistrictById(d['A']) +// if(dbo){ +// def alias = null +// if(!dbo.alias){ +// alias = [] +// }else{ +// alias = dbo.alias +// } +// if(!alias.contains(d['B'])){ +// alias << d['B'] +// } +// if(alias){ +// def udata = new BasicDBObject() +// udata.put("alias", alias) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", dbo._id) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// }else{ +// println d['A']+" "+"别名已存在" +// } +// +// }else{ +// println dbo._id.toString() +" 没找到" +// } +// } +//}) //def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] //for(def c : citys){ @@ -1331,34 +1331,34 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx" +def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx" // -//ExcelUtils.multiThreadingModify(fileName, {d-> -// if(d['J']){ -//// def region = RegionData.formatRegion(d['A'], d['J']) -//// if(region){ -//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") -//// if(dist){ -//// d['B'] = dist.region -//// d['C'] = dist.name -//// d['L'] = dist.status-dist.name_cat -//// } -//// } -//// if(d['J'] instanceof String){ -// def dist = CommonDatautils.findDistrictById(d['J']) +ExcelUtils.multiThreadingModify(fileName, {d-> + if(d['G']){ +// def region = RegionData.formatRegion(d['A'], d['J']) +// if(region){ +// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") // if(dist){ +// d['B'] = dist.region +// d['C'] = dist.name +// d['L'] = dist.status-dist.name_cat +// } +// } + if(d['G'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['G']) + if(dist){ // d['A'] = dist.city // d['B'] = dist.region // d['C'] = dist.name // d['K'] = dist.estate_type2 // d['L'] = dist.status // d['M'] = dist.name_cat -// -//// d['H'] = dist.name -// } -//// } -// } -//},0) + + d['H'] = dist.name + } + } + } +},0) //def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 9fc014cca..c2caf475f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -47,7 +47,8 @@ class TradeStatLog { // task.run()//统计最近3个月抓取的成交数据量 // task.exportTrade()//导出未入库成交 - task.exportTrade2()//导出未入库成交 +// task.exportTrade2()//导出未入库成交 + task.updateDealPriceRegion()//修改成交区域数据 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -597,7 +598,7 @@ class TradeStatLog { // def endDate = "2024-07-01" def ids = findAllDealIds(); println ids.size() - ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\廊坊.xlsx",{d-> if(ids.contains(d['A'])){ d['V'] = "供" } @@ -674,7 +675,7 @@ class TradeStatLog { } def findAllDealIds(){ - def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'" + def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='廊坊'" def ids = [] pssqlInstance.eachRow(sql,{r-> if(!ids.contains(r.gujia_bds_mdl_id)){ @@ -801,6 +802,20 @@ class TradeStatLog { },0) } + def updateDealPriceRegion(){ + ExcelUtils.modify("D:\\数据\\建行\\建行成交\\成交待修改区域数据.xlsx",{d-> + if(d['A'] && d['G']){ + def region = RegionData.formatRegion(d['D'], d['G']) + if(region){ + def udata = new BasicDBObject() + udata.put("region", region) + DEAL_PRICE_COLL.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata)) + } + + } + },0) + } + def findTradeById(id){ def q = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy index af276a9c9..10a3cc392 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy @@ -24,6 +24,13 @@ class AddressCut extends BaseBatch{ final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + final DBCollection thirdPartyPriceBkColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price_history_bk_new", "192.168.0.235", 27018) + + + final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018) + + final DBCollection dataProcessBkColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process_history_bk", "192.168.0.235", 27018) + public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" static { @@ -39,50 +46,66 @@ class AddressCut extends BaseBatch{ @Override public void process(Object args) { -<<<<<<< HEAD // sameAddr() - seaweedStreetUpdate() + def dates = [202408] + for(def date : dates){ +// synch(date) + synchDataProcess(date) + println date + "同步完成! " + } + } - def seaweedStreetUpdate(){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区地址拆分\\已核实-路2.xlsx",{d-> - if(d['A']){ - def dbo = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") - if(dbo){ - def udate = new BasicDBObject() - def street = null - def address = "" - if(d['E']){ - street = d['E'] - udate.put("street", street) - address = street - } - def streetNum = null - if(d['F']){ - streetNum = d['F'] - udate.put("street_number", streetNum) - address = address + streetNum - } - if(dbo.address!=address){ - udate.put("address", address) - } - if(udate){ - udate.put("m_date", new Date()) - DBUtils.seaweed.update(new BasicDBObject('_id', dbo.get("_id")), new BasicDBObject('$set', udate)) - } - } + def synch(date){ + def q = new BasicDBObject() + q.put("s_date", date) + def dbcursor = thirdPartyPriceColl.find(q).batchSize(5000) + def index = 0 + def rets = [] + def num = 0 + for(def d : dbcursor){ + num++ + index++ + rets << d + if(index%10000==0){ + thirdPartyPriceBkColl.insert(rets) + println(num) + rets.clear() } - - }) -======= -// sameAddr() //相同地址小区 -// roadMatch() - addAnjukeRoad() - + } + if(rets.size()>0){ + thirdPartyPriceBkColl.insert(rets) + rets.clear() + } } + def synchDataProcess(date){ + def q = new BasicDBObject() + q.put("s_date", date) + def dbcursor = dataProcessColl.find(q).batchSize(5000) + def index = 0 + def rets = [] + def num = 0 + for(def d : dbcursor){ + num++ + index++ + rets << d + if(index%10000==0){ + dataProcessBkColl.insert(rets) + println(num) + rets.clear() + } + } + if(rets.size()>0){ + dataProcessBkColl.insert(rets) + rets.clear() + } + } + + + def addAnjukeRoad(){ ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d -> def roads = findThirdPartyPrice(d['A'], d['B'], d['C'], "链家") @@ -130,8 +153,6 @@ class AddressCut extends BaseBatch{ } }) - ->>>>>>> c1a5ab8c6aa973a91e6be5547ec0925db3fa500f } def sameAddr() { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 94668a4df..5e2937768 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -122,6 +122,8 @@ class CommonDatautils { public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com") + public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com") + public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { return house_dataStoreConnection @@ -820,6 +822,7 @@ class CommonDatautils { DBObject q = new BasicDBObject(); q.put("status",0) +// q.put("status",new BasicDBObject('$ne', 0)) q.put("cat", cat); q.put("city", city) @@ -829,7 +832,7 @@ class CommonDatautils { keys.put(field, 1); } } - def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000) + def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000) def rets = [] for(def d : dbsour){ rets << d @@ -959,13 +962,7 @@ class CommonDatautils { keys.put(field, 1); } } - def cursor = getDataStoreConnection().find(q, keys).batchSize(2000) - if(cursor){ - for(def obj : cursor){ - return obj; - } - } - return null; + return SEAWEED_COLL.findOne(q, keys) } @@ -1132,6 +1129,10 @@ class CommonDatautils { q.put("city", city); q.put("cat", cat); q.put("name", name); + if(region){ + q.put("region", region); + } + q.put("status",new BasicDBObject('$ne', 0)) DBObject keys = new BasicDBObject(); if(fields){ @@ -1680,7 +1681,7 @@ class CommonDatautils { DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); queryCondition.put("category","region"); - queryCondition.put("show",1); +// queryCondition.put("show",1); def dbCursor = REGION_BLOCK_COLL.find(queryCondition) def regions = []; for(DBObject obj : dbCursor){ @@ -2385,6 +2386,9 @@ class CommonDatautils { q.put("city", city); if(region){ q.put("region", region); + q.put("cat", "区域") + }else{ + q.put("cat", "城市") } q.put('weekend', weekend); def dbCursor = REGION_STAT_COLL.find(q).sort(new BasicDBObject().append("weekend", -1)).limit(1); @@ -3735,9 +3739,9 @@ class CommonDatautils { if(!region){ if(city){ q.put("cat", "city") - }else{ - q.put("cat", "region") } + }else{ + q.put("cat", "region") } } @@ -4095,6 +4099,21 @@ class CommonDatautils { return retMap; } + + static def findHouseCount(def city, def region, def name){ + + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + .must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("district_name", name)).must(QueryBuilders.termQuery("cat", "sell")) + + + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(5); + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + return houses.size(); + } + + /** * 是否是别墅 * @return @@ -5060,4 +5079,38 @@ class CommonDatautils { } + static def findStatIndexBySort(type, city, region, sort){ + def q = new BasicDBObject() + q.put("cat", "district") + q.put("type", type) + if(region){ + q.put("district", region) + } + if(city){ + q.put("city", city) + } + + def cur = STATS_INDEX_COLL.find(q).sort(new BasicDBObject("yearmonth", sort)).limit(1) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def obj = 0; + while(cur.hasNext()) { + obj = cur.next() + } + return obj; + } + + static def findStatIndexByMonth(type, city, region, month){ + def q = new BasicDBObject() + q.put("cat", "district") + q.put("type", type) + if(region){ + q.put("district", region) + } + if(city){ + q.put("city", city) + } + q.put("yearmonth", month) + return STATS_INDEX_COLL.findOne(q) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index 1b11ee16a..a3c119bec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -2,8 +2,10 @@ package com.fangjia.script.gj import com.fangjia.core.BlockData import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.DateUtil import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils @@ -531,6 +533,28 @@ class FiledStrand { return null } + + static def processLngAndLat(val) { + if (val) { + def map = [:] + def lng = DataHandleUtils.getDoubleRound(6, val.x) + def lat = DataHandleUtils.getDoubleRound(6, val.y) + Long gd_lat = lat * 10000000000L + Long gd_lng = lng * 10000000000L + map.lat = gd_lat + map.lng = gd_lng + + def baidu = BaiDuApiUtils.geoconv(lng, lat, 3) + Long bd_lng = baidu.x * 10000000000L + Long bd_lat = baidu.y * 10000000000L + map.lat2 = bd_lat + map.lng2 = bd_lng + return map + + } + return null + } + static def isRoad(def s){ def pattern = Pattern.compile("[路|街|大道|巷]"); def matcher = pattern.matcher(s) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy index 082e2f170..a496ed93f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy @@ -2,33 +2,97 @@ package com.fangjia.script.gj import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.DataUtil import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection -ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d-> - if(d['A'] && !d['E']){ -// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district") -// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district") - println d['A'] +" "+d['C'] -// def total = findDatas(d['A'], d['B'], d['C']) -// d['D'] = total - def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·","")) - d['E'] = c - } -}) +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +//ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d-> +// if(d['A'] && !d['E']){ +//// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district") +//// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district") +// println d['A'] +" "+d['C'] +//// def total = findDatas(d['A'], d['B'], d['C']) +//// d['D'] = total +// def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·","")) +// d['E'] = c +// } +//}) + +def URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; +DBCollection seaweed_coll = MongoUtils.getCollectionByName2("fangjia", "seaweed", URL) +def q = new BasicDBObject() +q.put("city", "上海市") +q.put("category", "district") +q.put("status", 0) +def ret = [] +def list = seaweed_coll.find(q).batchSize(5000).toList() +println "总计:"+list.size() +//for(def l : list){ +// def obj = getAmapLocation(l.city, l.region, l.name) +// if(obj){ +// ret << [l.city, l.region, l.name, obj.cityname, obj.adname, obj.name,obj.type] +// } +//} + +ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) +def addCount = new AtomicLong() +final CountDownLatch latch = new CountDownLatch(list.size()) +def total = list.size() +for(def i=0;i - if(d.name.contains(name) && d.type.contains("住宅")){ - total ++ + if(d.name.contains(name) && d.type.contains("住宅") && d.shape){ + obj = d } }) - return total; + return obj; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 56915936b..f722db94c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -46,7 +46,7 @@ class DevProcess { def task = new DevProcess(); // task.run() // task.parseCityAndDist() - task.devSplitRoad() +// task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() // task.matchDev() @@ -63,7 +63,7 @@ class DevProcess { // task.createDevMapping() // task.deleteAddr() // task.deleteAddr2() -// task.synchDevTo100() + task.synchDevTo100() // task.addNewDev2() // task.addAlias() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index d56afba8a..cf1b49203 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -20,6 +20,10 @@ class DevUpdateTestBatch { def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection development_zone_43_CONN = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") + + + DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) @@ -30,8 +34,8 @@ class DevUpdateTestBatch { static main(args) { def task = new DevUpdateTestBatch() def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx -// task.addDev(path) - task.addDevAddress(path) + task.addDev(path) +// task.addDevAddress(path) // // task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是") @@ -42,7 +46,7 @@ class DevUpdateTestBatch { def addDev(path){ ExcelUtils.modify(path,{ d-> if(!d['I'] && d['A']){ - def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A'])) + def obj = development_zone_43_CONN.findOne(new BasicDBObject("name", d['A'])) if(!obj){ def o = new BasicDBObject() o.put("province", d['B']) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy index 35c1fa9db..ee54817e6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy @@ -28,7 +28,7 @@ class GXDdistrictMatch { static main(args) { def task = new GXDdistrictMatch(); // task.run() - task.addOldMatch() + task.addOldMatch2() // task.runDb() // task.updateDistStatus()//恢复name_cat不正常数据 @@ -68,6 +68,32 @@ class GXDdistrictMatch { } + def addOldMatch2(){ + //老匹配 + def fileName = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx"//11城重新匹配.xlsx + ExcelUtils.multiThreadingModify(fileName, { d -> + println d['C'] + if(!d['AJ']){ + def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D']) + if (jsonObject != null && !jsonObject.containsKey("result")) { + d['AJ'] = jsonObject.get("id") + d['AK'] = jsonObject.get("city") + d['AL'] = jsonObject.get("district") + d['AM'] = jsonObject.get("name") + d['AN'] = jsonObject.get("address") + + def dbo = CommonDatautils.findDistrictById(jsonObject.get("id")) + if(dbo){ + d['AO'] = dbo.status + d['AP'] = dbo.name_cat + d['AQ'] = dbo.estate_type2 + } + } + } + }); + + } + def runDb(){ def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx" ExcelUtils.modify(fileName, { d -> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy index 0fbf352f7..41d940274 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy @@ -35,7 +35,7 @@ class ThirdPriceMappingBatch extends BaseBatch{ final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018) - def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); static main(args) { runBatch(new ThirdPriceMappingBatch(), args); @@ -44,9 +44,9 @@ class ThirdPriceMappingBatch extends BaseBatch{ @Override public void process(Object args) { // //将新增数据同步过来 -// addNemCommunity() + addNemCommunity() // //接口进行匹配 - match() +// match() // addOldMatchResult() } @@ -151,8 +151,9 @@ class ThirdPriceMappingBatch extends BaseBatch{ sd.each {k,v-> udata.put(k,v) } + udata.removeField("_id") udata.put("m_date", new Date()) - thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata)) + thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata),false, false) }else{ sd.put("m_date", new Date()) thirdSeaweedMappingColl.save(sd) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy index 0f61101ea..9d5283a6b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.GeocodeUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils @@ -900,41 +901,92 @@ class TownMiddleProcess { } def updateTownPolineBatch(){ + def i = 0 ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx",{ row-> if(row['R'] && row['R']=="新增围栏"){ - def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D']) - println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q) - def d = TOWNPOLYLINE_COLL.findOne(q) - if(d){ - def resultMap = [:] - resultMap.type = "polygon" - def arrs = d.polyline.split("\\|") - def rets = [] - for(def s : arrs){ - def points = s.split(";") - def pointList = [] - for(def pointStr : points){ - def pointArr = pointStr.split(",") - def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble()) - def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble()) - pointList << [lon, lat] + i++ + if(i>=134){ + def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D']) + println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q) + def d = TOWNPOLYLINE_COLL.findOne(q) + if(d){ + def resultMap = [:] + def bd_resultMap = [:] + + resultMap.type = "polygon" + bd_resultMap.type = "polygon" + + def arrs = d.polyline.split("\\|") + def rets = [] + def bd_rets = [] + + println row['N'] +" 共有:"+arrs.size()+" 个围栏" + def index = 1 + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + def bd_pointList = [] + + for(def pointStr : points){ + def pointArr = pointStr.split(",") + def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble()) + def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble()) + pointList << [lon, lat] + + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(lon, lat, 3) + def bd_lng = baidu.x + def bd_lat = baidu.y + bd_pointList << [bd_lng, bd_lat] + } + + if(index<=1){ + resultMap.coordinates = pointList + bd_resultMap.coordinates = bd_pointList + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + udata.put("operator", "高娟") + udata.put("m_date", new Date()) + udata.put("bd_shape",bd_resultMap) + + def updateQuery = new BasicDBObject("name",row['N']) + DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata)) + }else{ + def dbo = DEVELOPMENT_ZONE_43_COLL.findOne(new BasicDBObject("name",row['N'])) + def obj = new BasicDBObject() + dbo.each {k,v-> + if(k!='_id'){ + obj.put(k,v) + } + } + + resultMap.coordinates = pointList + bd_resultMap.coordinates = bd_pointList + obj.put("shape", resultMap) + obj.put("operator", "高娟") + obj.put("m_date", new Date()) + obj.put("bd_shape",bd_resultMap) + obj.put("create_time", new Date()) + obj.put("block", dbo.name+"-地块"+index) + DEVELOPMENT_ZONE_43_COLL.save(obj) + } + +// rets << pointList +// bd_rets << bd_pointList + index++ } - rets << pointList + + +// if(rets.size()>1) { +// resultMap.type = "MultiPolygon" +// bd_resultMap.type = "MultiPolygon" +// } +// println d.city+" "+d.region+" "+d.town+" "+rets.size() + row['S'] = "已完成" } - resultMap.coordinates = rets - if(rets.size()>1) { - resultMap.type = "MultiPolygon" - } - println d.city+" "+d.region+" "+d.town+" "+rets.size() - - def udata = new BasicDBObject() - udata.put("shape", resultMap) - - - def updateQuery = new BasicDBObject("name",row['N']) - DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata)) - row['S'] = "已完成" } + } }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy index c51b3f825..b5de80cae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy @@ -53,7 +53,7 @@ class ImportIndexStats extends BaseBatch{ - public static final def dingjiYear = 2020; + public static def dingjiYear = 2020; private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final EsDataSource dataSource = new EsDataSource() @@ -76,22 +76,23 @@ class ImportIndexStats extends BaseBatch{ // synch() // synchTjjIndex()//同步统计局抓取过来的指数 -// Calendar calendar=Calendar.getInstance(); -// def month= calendar.get(Calendar.MONTH); -// def year= calendar.get(Calendar.YEAR); -// def date = createDateString(year, month, 0, ""); -// calculatePriceIndex(year, month, date); + Calendar calendar=Calendar.getInstance(); + def month= calendar.get(Calendar.MONTH); + def year= calendar.get(Calendar.YEAR); + def date = createDateString(year, month, 0, ""); + calculatePriceIndex(year, month, date); // addLostIndex() - - addHistoryIndex() +// addHistoryIndex() } def addLostIndex(){ def citys = CommonDatautils.findAllCity(); + def index = 0 for(def city : citys){ + index++ if(cityList.contains(city)){ - println city +" 数据来源于统计局" + println "[${index}] "+ city +" 数据来源于统计局" continue; } def start = findStartOrEndIndex(city, 1) @@ -117,21 +118,48 @@ class ImportIndexStats extends BaseBatch{ if(nextObj && nextObj.chain){ indexs << nextObj.chain } - udata.put("chain",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size())) - udata.put("remark","推导") + if(indexs){ + udata.put("chain",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size())) + udata.put("remark","推导") + } + } + if(!dbo.yeartoyear){ + def indexs = [] + if(lastMonthObj && lastMonthObj.yeartoyear){ + indexs << lastMonthObj.yeartoyear + } + if(nextObj && nextObj.yeartoyear){ + indexs << nextObj.yeartoyear + } + if(indexs){ + udata.put("yeartoyear",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size())) + udata.put("remark","推导") + } + } + if(!dbo.fixedbase){ + def indexs = [] + if(lastMonthObj && lastMonthObj.fixedbase){ + indexs << lastMonthObj.fixedbase + } + if(nextObj && nextObj.fixedbase){ + indexs << nextObj.fixedbase + } + if(indexs){ + udata.put("fixedbase",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size())) + udata.put("remark","推导") + } + + } + if(udata){ + println city +" 补充" STATS_INDEX_COLL.update(new BasicDBObject("_id", dbo._id), new BasicDBObject('$set', udata),false, true); } -// if(!dbo.yeartoyear){ -// -// } -// if(!dbo.fixedbase){ -// -// } - } } } + + println "[${index}] "+ city +" 已处理完" } } @@ -154,7 +182,7 @@ class ImportIndexStats extends BaseBatch{ } def addHistoryIndex(){ - def s = "202101" + def s = "202001" def startDate = null; def stop = false; def runDates = [] @@ -171,7 +199,7 @@ class ImportIndexStats extends BaseBatch{ runDates << map s = DateUtils.addMonth(date, 1, "yyyyMM") - if(year==2024 && month==9){ + if(year==2024 && month==8){ stop = true; } } @@ -182,6 +210,7 @@ class ImportIndexStats extends BaseBatch{ //计算统计局没有发布的城市的价格指数 def calculatePriceIndex(def year, def month, def date){ +// def citys = ["五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]// def citys = CommonDatautils.findAllCity(); // def date = createDateString(year, month, 0, ""); def haveIndexCitys = findPriceIndexCitys(date.toInteger()); @@ -193,7 +222,7 @@ class ImportIndexStats extends BaseBatch{ def newCitys = [] for(def i=0;i=201601){ + dingjiYear = 2015 + }else if(date.toInteger()>=202101){ + dingjiYear = 2020 + }else if(date.toInteger()>=202601){ + dingjiYear = 2025 + }else if(date.toInteger()>=203101){ + dingjiYear = 2030 + } fixedThreadPool.execute(new Runnable() { void run() { try{ @@ -220,7 +258,7 @@ class ImportIndexStats extends BaseBatch{ } }); } - latch.await(20, TimeUnit.SECONDS); + latch.await(1, TimeUnit.MINUTES); } def calculating(city, year, month, date){ @@ -647,6 +685,7 @@ class ImportIndexStats extends BaseBatch{ DBObject q = new BasicDBObject(); q.put("yearmonth", date); q.put("cat", "district") + q.put("type", "城市") DBObject keys = new BasicDBObject(); keys.put("city", 1) @@ -666,6 +705,7 @@ class ImportIndexStats extends BaseBatch{ q.put("yearmonth", date); q.put("city", city); q.put("cat", "district"); + q.put("type", "城市") DBObject keys = new BasicDBObject(); keys.put("city", 1) @@ -684,6 +724,7 @@ class ImportIndexStats extends BaseBatch{ dbo.put("source",result.source); dbo.put("remark",result.remark); dbo.put("cat",result.cat); + dbo.put("type",result.type); dbo.put("c_date",new Date()); if(result.markup){ dbo.put("chain", result.markup.toDouble());