diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 974df17ee..0bf6ec4b9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年7月深圳二手房成交.xlsx", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年9月深圳二手房成交.xlsx", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy index 2684f90c2..d3df8510d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy @@ -1,11 +1,18 @@ package com.fangjia.script.ccbevaluation import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtil +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + /** * 将国信达及成交对标数据加入到第三方均价表,做为均价参考 */ @@ -13,19 +20,30 @@ class AddThirdPriceSample { static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + public static final DBCollection NO_PRICE_DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com") + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + static main(args) { def task = new AddThirdPriceSample() - task.run() +// task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据 + task.addTrade() //添加成交 + +// task.matchOldData() +// task.noInCcbCity() + } def run(){ - for(def i=1;i<19;i++){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d-> - if(d['A'] && d['C']){ + for(def i=0;i<1;i++){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达指导价录入-1024.xlsx",{d-> + if(d['A'] && d['B']){ def dist = CommonDatautils.findDistrictById(d['A'].toString().trim()) if(dist){ - def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407) - def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger()) - + def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202409) +// def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger()) + def udata = new BasicDBObject("fccs_esf_price", d['B'].toString().toDouble().toInteger()) AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false) }else{ println d['A'] +" 没查到" @@ -37,4 +55,142 @@ class AddThirdPriceSample { } } + + def addTrade(){ +// def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"] + def rets = findTrades() + DEAL_PRICE_COLL.insert(rets) + } + + def findTrades(){ + def q = new BasicDBObject() + q.put("ccb_price_remark","建行无此城市成交") +// q.put("city", new BasicDBObject('$in', citys)) +// q.put("flag", 1) + def dbcursor = NO_PRICE_DEAL_PRICE_COLL.find(q) + def rets = [] + def index = 0 + for(def db : dbcursor){ + index++ + def udata = new BasicDBObject() + + udata.put("_id",db._id) + udata.put("url",db.url) + udata.put("area",db.area.toString().toDouble()) + udata.put("avg_price",db.avg_price.toString().toDouble().toInteger()) + udata.put("build_structure",db.build_structure) + udata.put("build_type",db.build_type) + udata.put("build_year",db.build_year) + udata.put("city",db.city) + udata.put("direction", db.direction) + udata.put("district_name",db.district_name) + udata.put("elevator",db.elevator) + udata.put("fitment",db.fitment) + udata.put("floor_info",db.floor_info) + if(db.hall){ + udata.put("hall",db.hall.toString().toInteger()) + } + + if(db.height){ + udata.put("height",db.height.toString().toInteger()) + } + + udata.put("housing_age",db.housing_age) + udata.put("lift",db.lift.toString().toInteger()) + udata.put("lift_household",db.lift_household.toString().toInteger()) + udata.put("property_ownership",db.property_ownership) + udata.put("region",db.region) + udata.put("room", db.room.toString().toInteger()) + udata.put("room_structure",db.room_structure) + udata.put("source",db.source) + udata.put("total_price",db.total_price.toString().toDouble().toInteger()) + udata.put("trade_date",db.trade_date) + udata.put("trade_property",db.trade_property) + udata.put("house_using",db.house_using) + udata.put("heating_type",db.heating_type) + udata.put("community_id",db.community_id) + udata.put("create_date",db.create_time) + udata.put("m_date", new Date()) + udata.put("address",db.address) + udata.put("match_city",db.match_city) + udata.put("match_region",db.match_region) + udata.put("match_name",db.match_name) + udata.put("match_id",db.match_id) + udata.put("price_flag", "国信达成交-推导") + +// DEAL_PRICE_COLL.save(udata) + println index + rets << udata + } + return rets + } + + def matchOldData(){ + def q = new BasicDBObject("source","贝壳找房").append("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-09-01","yyyy-MM-dd"))) + def list = DEAL_PRICE_COLL.find(q).toList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(list.size()) + for(def i=0;i - total = r.count - }) - if(total>0){ - println city +" "+m.commid+" (按价格)删除:"+total - def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" - pgsqlInstance.executeUpdate(sql) - }else{ - - querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " - pgsqlInstance.eachRow(querySql, { r -> - total = r.count - }) - if(total>0){ - println city +" "+m.commid+" (按小区)删除:"+total - def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " - pgsqlInstance.executeUpdate(sql) - } - } - } + processDetele(city, cityid, month, rets) } println set.toString() } + + def processDetele(city, cityid, month, rets){ + + final CountDownLatch latch = new CountDownLatch(rets.size()) + + for(def i=0;i + total = r.count + }) + if(total>0){ + println city +" "+m.commid+" (按价格)删除:"+total + def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" + pgsqlInstance.executeUpdate(sql) + }else{ + + querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " + pgsqlInstance.eachRow(querySql, { r -> + total = r.count + }) + if(total>0){ + println city +" "+m.commid+" (按小区)删除:"+total + def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " + pgsqlInstance.executeUpdate(sql) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }); + } + + latch.await() + } + + + + + def findThirdCount(sqlInstance236, sql){ def count = 0 sqlInstance236.eachRow(sql,{r-> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 0d36523e6..3ccdc43ef 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -631,36 +631,36 @@ def findAllData(sqlInstance236, citycode){ //},0) -ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d-> - if(d['A'] && d['B']){ - def dbo = CommonDatautils.findDistrictById(d['A']) - if(dbo){ - def alias = null - if(!dbo.alias){ - alias = [] - }else{ - alias = dbo.alias - } - if(!alias.contains(d['B'])){ - alias << d['B'] - } - if(alias){ - def udata = new BasicDBObject() - udata.put("alias", alias) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", dbo._id) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - }else{ - println d['A']+" "+"别名已存在" - } - - }else{ - println dbo._id.toString() +" 没找到" - } - } -}) +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d-> +// if(d['A'] && d['B']){ +// def dbo = CommonDatautils.findDistrictById(d['A']) +// if(dbo){ +// def alias = null +// if(!dbo.alias){ +// alias = [] +// }else{ +// alias = dbo.alias +// } +// if(!alias.contains(d['B'])){ +// alias << d['B'] +// } +// if(alias){ +// def udata = new BasicDBObject() +// udata.put("alias", alias) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", dbo._id) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// }else{ +// println d['A']+" "+"别名已存在" +// } +// +// }else{ +// println dbo._id.toString() +" 没找到" +// } +// } +//}) //def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] //for(def c : citys){ @@ -1234,15 +1234,7 @@ def findThirdCommCount(cityid, sqlInstance){ //} // -//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\无价格小区(3)2.23.xlsx" -//ExcelUtils.multiThreadingModify(fileName, { d -> -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// d['G'] = dist.city -// d['H'] = dist.region -// d['I'] = dist.name -// } -//}) + @@ -1331,10 +1323,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx" -// +//def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx" +//// //ExcelUtils.multiThreadingModify(fileName, {d-> -// if(d['J']){ +// if(d['G']){ //// def region = RegionData.formatRegion(d['A'], d['J']) //// if(region){ //// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") @@ -1344,23 +1336,34 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ //// d['L'] = dist.status-dist.name_cat //// } //// } -//// if(d['J'] instanceof String){ -// def dist = CommonDatautils.findDistrictById(d['J']) +// if(d['G'] instanceof String){ +// def dist = CommonDatautils.findDistrictById(d['G']) // if(dist){ -// d['A'] = dist.city -// d['B'] = dist.region -// d['C'] = dist.name -// d['K'] = dist.estate_type2 -// d['L'] = dist.status -// d['M'] = dist.name_cat +//// d['A'] = dist.city +//// d['B'] = dist.region +//// d['C'] = dist.name +//// d['K'] = dist.estate_type2 +//// d['L'] = dist.status +//// d['M'] = dist.name_cat // -//// d['H'] = dist.name +// d['H'] = dist.name // } -//// } +// } // } //},0) +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20241025人工指导价.xlsx" +ExcelUtils.multiThreadingModify(fileName, { d -> + def dist = CommonDatautils.findDistrictById(d['J']) + if(dist){ + d['A'] = dist.city + d['B'] = dist.region + d['C'] = dist.name + } +}) + + //def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx" //ExcelUtils.multiThreadingModify(file, { d -> // if(d['A']){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 9fc014cca..c2caf475f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -47,7 +47,8 @@ class TradeStatLog { // task.run()//统计最近3个月抓取的成交数据量 // task.exportTrade()//导出未入库成交 - task.exportTrade2()//导出未入库成交 +// task.exportTrade2()//导出未入库成交 + task.updateDealPriceRegion()//修改成交区域数据 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -597,7 +598,7 @@ class TradeStatLog { // def endDate = "2024-07-01" def ids = findAllDealIds(); println ids.size() - ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\廊坊.xlsx",{d-> if(ids.contains(d['A'])){ d['V'] = "供" } @@ -674,7 +675,7 @@ class TradeStatLog { } def findAllDealIds(){ - def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'" + def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='廊坊'" def ids = [] pssqlInstance.eachRow(sql,{r-> if(!ids.contains(r.gujia_bds_mdl_id)){ @@ -801,6 +802,20 @@ class TradeStatLog { },0) } + def updateDealPriceRegion(){ + ExcelUtils.modify("D:\\数据\\建行\\建行成交\\成交待修改区域数据.xlsx",{d-> + if(d['A'] && d['G']){ + def region = RegionData.formatRegion(d['D'], d['G']) + if(region){ + def udata = new BasicDBObject() + udata.put("region", region) + DEAL_PRICE_COLL.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata)) + } + + } + },0) + } + def findTradeById(id){ def q = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy index af276a9c9..10a3cc392 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy @@ -24,6 +24,13 @@ class AddressCut extends BaseBatch{ final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + final DBCollection thirdPartyPriceBkColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price_history_bk_new", "192.168.0.235", 27018) + + + final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018) + + final DBCollection dataProcessBkColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process_history_bk", "192.168.0.235", 27018) + public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" static { @@ -39,50 +46,66 @@ class AddressCut extends BaseBatch{ @Override public void process(Object args) { -<<<<<<< HEAD // sameAddr() - seaweedStreetUpdate() + def dates = [202408] + for(def date : dates){ +// synch(date) + synchDataProcess(date) + println date + "同步完成! " + } + } - def seaweedStreetUpdate(){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区地址拆分\\已核实-路2.xlsx",{d-> - if(d['A']){ - def dbo = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") - if(dbo){ - def udate = new BasicDBObject() - def street = null - def address = "" - if(d['E']){ - street = d['E'] - udate.put("street", street) - address = street - } - def streetNum = null - if(d['F']){ - streetNum = d['F'] - udate.put("street_number", streetNum) - address = address + streetNum - } - if(dbo.address!=address){ - udate.put("address", address) - } - if(udate){ - udate.put("m_date", new Date()) - DBUtils.seaweed.update(new BasicDBObject('_id', dbo.get("_id")), new BasicDBObject('$set', udate)) - } - } + def synch(date){ + def q = new BasicDBObject() + q.put("s_date", date) + def dbcursor = thirdPartyPriceColl.find(q).batchSize(5000) + def index = 0 + def rets = [] + def num = 0 + for(def d : dbcursor){ + num++ + index++ + rets << d + if(index%10000==0){ + thirdPartyPriceBkColl.insert(rets) + println(num) + rets.clear() } - - }) -======= -// sameAddr() //相同地址小区 -// roadMatch() - addAnjukeRoad() - + } + if(rets.size()>0){ + thirdPartyPriceBkColl.insert(rets) + rets.clear() + } } + def synchDataProcess(date){ + def q = new BasicDBObject() + q.put("s_date", date) + def dbcursor = dataProcessColl.find(q).batchSize(5000) + def index = 0 + def rets = [] + def num = 0 + for(def d : dbcursor){ + num++ + index++ + rets << d + if(index%10000==0){ + dataProcessBkColl.insert(rets) + println(num) + rets.clear() + } + } + if(rets.size()>0){ + dataProcessBkColl.insert(rets) + rets.clear() + } + } + + + def addAnjukeRoad(){ ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d -> def roads = findThirdPartyPrice(d['A'], d['B'], d['C'], "链家") @@ -130,8 +153,6 @@ class AddressCut extends BaseBatch{ } }) - ->>>>>>> c1a5ab8c6aa973a91e6be5547ec0925db3fa500f } def sameAddr() { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index c611aa10f..1521d4e0b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -1457,45 +1457,45 @@ class ExportDistrictRongHe extends BaseBatch{ println "community_p_describe_gj 上传Ftp完成......" } - def exportGuideStatistics(time){ - - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" - } - def headers = [] - def allRows = [] - - for(def city : citys) { - def cityCode = cityCodeMap.get(city) - def sql = null; - if(CcbCommonUtils.citys_92.contains(city)){ - sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}' and year_month=${time}"; - }else{ - sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}'"; - } - - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.MODEL_TYPE - data << r.CITYID - data << r.CITYNAME - data << r.YEAR_MONTH - data << r.COMM_SUM_COUNT - data << r.COMM_COUNT - data << r.RATIO - data << r.TYPE - allRows << data - }); - } - println allRows.size() - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - - println "guide_statistics 数据文件输出完成......" - CcbFtpUtils.ftpUpload("rh", fileName, curDate) - println "guide_statistics 上传Ftp完成......" - } +// def exportGuideStatistics(time){ +// +// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" +// def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" +// } +// def headers = [] +// def allRows = [] +// +// for(def city : citys) { +// def cityCode = cityCodeMap.get(city) +// def sql = null; +// if(CcbCommonUtils.citys_92.contains(city)){ +// sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}' and year_month=${time}"; +// }else{ +// sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}'"; +// } +// +// sqlInstance.eachRow(sql, { r -> +// def data = [] +// data << r.MODEL_TYPE +// data << r.CITYID +// data << r.CITYNAME +// data << r.YEAR_MONTH +// data << r.COMM_SUM_COUNT +// data << r.COMM_COUNT +// data << r.RATIO +// data << r.TYPE +// allRows << data +// }); +// } +// println allRows.size() +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); +// +// println "guide_statistics 数据文件输出完成......" +// CcbFtpUtils.ftpUpload("rh", fileName, curDate) +// println "guide_statistics 上传Ftp完成......" +// } def exportPgGuideStatistics(time){ @@ -1514,7 +1514,6 @@ class ExportDistrictRongHe extends BaseBatch{ pgsqlInstance.eachRow(sql, { r -> def data = [] - data << r.gujia_model_type data << r.urbn_id data << r.urbn_nm data << r.gujia_clc_yrmo @@ -1522,6 +1521,7 @@ class ExportDistrictRongHe extends BaseBatch{ data << r.gujia_comm_count data << r.gujia_ratio data << r.gujia_type + data << r.gujia_model_type allRows << data }); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 94668a4df..15c76f043 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1,43 +1,22 @@ package com.fangjia.script.commData import com.fangjia.core.RegionData -import com.fangjia.datastore.driver.FacetResult -import com.fangjia.httpclient.UserAgents -import com.fangjia.script.common.DataHandleUtils -import com.fangjia.script.looplineinfo.InsidePolygon -import com.fangjia.script.util.DbUtils -import com.fangjia.script.util.ElasticUtils -import com.fangjia.script.util.JacksonUtil -import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils -import com.mongodb.AggregationOutput -import com.mongodb.client.MongoCursor -import com.mongodb.client.model.Accumulators -import com.mongodb.client.model.Aggregates -import com.rabbitmq.client.impl.AMQImpl -import groovy.json.JsonSlurper - -import net.sf.json.JSONArray -import net.sf.json.JSONObject -import org.apache.commons.lang.StringUtils -import org.apache.poi.ss.util.RegionUtil -import org.bson.Document -import org.bson.types.ObjectId - -import com.fangjia.core.util.HttpRequest; +import com.fangjia.core.util.HttpRequest import com.fangjia.datastore.driver.DSCollection import com.fangjia.datastore.driver.DSCursor import com.fangjia.datastore.driver.DataStore +import com.fangjia.httpclient.UserAgents +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.newDistrict.InvokeHttpRequest -import com.fangjia.script.util.DateUtils -import com.fangjia.script.util.GeoHelper -import com.fangjia.script.util.GeoUtils -import com.fangjia.script.util.MongoUtils -import com.mongodb.BasicDBObject -import com.mongodb.Bytes -import com.mongodb.DBCollection -import com.mongodb.DBCursor -import com.mongodb.DBObject -import com.mongodb.QueryOperators +import com.fangjia.script.util.* +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.* +import groovy.json.JsonSlurper +import net.sf.json.JSONArray +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +import org.bson.types.ObjectId import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders @@ -122,6 +101,8 @@ class CommonDatautils { public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com") + public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com") + public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { return house_dataStoreConnection @@ -820,6 +801,7 @@ class CommonDatautils { DBObject q = new BasicDBObject(); q.put("status",0) +// q.put("status",new BasicDBObject('$ne', 0)) q.put("cat", cat); q.put("city", city) @@ -829,7 +811,7 @@ class CommonDatautils { keys.put(field, 1); } } - def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000) + def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000) def rets = [] for(def d : dbsour){ rets << d @@ -959,13 +941,7 @@ class CommonDatautils { keys.put(field, 1); } } - def cursor = getDataStoreConnection().find(q, keys).batchSize(2000) - if(cursor){ - for(def obj : cursor){ - return obj; - } - } - return null; + return SEAWEED_COLL.findOne(q, keys) } @@ -1132,6 +1108,10 @@ class CommonDatautils { q.put("city", city); q.put("cat", cat); q.put("name", name); + if(region){ + q.put("region", region); + } + q.put("status",new BasicDBObject('$ne', 0)) DBObject keys = new BasicDBObject(); if(fields){ @@ -1680,7 +1660,7 @@ class CommonDatautils { DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); queryCondition.put("category","region"); - queryCondition.put("show",1); +// queryCondition.put("show",1); def dbCursor = REGION_BLOCK_COLL.find(queryCondition) def regions = []; for(DBObject obj : dbCursor){ @@ -2385,6 +2365,9 @@ class CommonDatautils { q.put("city", city); if(region){ q.put("region", region); + q.put("cat", "区域") + }else{ + q.put("cat", "城市") } q.put('weekend', weekend); def dbCursor = REGION_STAT_COLL.find(q).sort(new BasicDBObject().append("weekend", -1)).limit(1); @@ -3542,6 +3525,20 @@ class CommonDatautils { return prices; } + static def queryCityPriceByWeekend(city, weekend){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("cat", "城市"); + + queryCondition.put('weekend', DateUtils.parseFromTightYYYYMMDD(weekend)); + def obj = REGION_STAT_COLL.findOne(queryCondition) + def prices = null; + if(obj) { + prices = obj.get("avg_price"); + } + return prices; + } + @@ -3735,9 +3732,9 @@ class CommonDatautils { if(!region){ if(city){ q.put("cat", "city") - }else{ - q.put("cat", "region") } + }else{ + q.put("cat", "region") } } @@ -4095,6 +4092,21 @@ class CommonDatautils { return retMap; } + + static def findHouseCount(def city, def region, def name){ + + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + .must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("district_name", name)).must(QueryBuilders.termQuery("cat", "sell")) + + + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(5); + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + return houses.size(); + } + + /** * 是否是别墅 * @return @@ -4766,7 +4778,6 @@ class CommonDatautils { def q = new BasicDBObject() q.put("type","区域") q.put("district_diff",1) - q.put("city",new BasicDBObject('$in', ["深圳市"])) def keys = new BasicDBObject() keys.put("province",1) @@ -4795,6 +4806,90 @@ class CommonDatautils { return regionMap } + + static def queryAllRegionNameByCity(){ + + def q = new BasicDBObject() + q.put("type","区域") + q.put("district_diff",1) + + def keys = new BasicDBObject() + keys.put("city_short",1) + keys.put("district_short",1) + keys.put("alias",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def retMap = [:] + for(def d : dbcursor){ + def regionMap = retMap.get(d.city_short) + if(!regionMap){ + regionMap = [:] + retMap.put(d.city_short, regionMap) + } + regionMap.put(d.district_short, d.alias) + } + + return retMap + } + + static def getRegionShortName(city){ + def q = new BasicDBObject() + q.put("type","区域") + q.put("district_diff",1) + q.put("city", city) + + def keys = new BasicDBObject() + keys.put("district",1) + keys.put("district_short", 1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def shortNameMap = [:] + for(def d : dbcursor){ + shortNameMap.put(d.district_short, d.district) + } + return shortNameMap + } + + + static def queryAllRegionFullNameByCity(){ + def q = new BasicDBObject() + q.put("type","区域") + q.put("district_diff",1) + + def keys = new BasicDBObject() + keys.put("city",1) + keys.put("district",1) + keys.put("alias",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def retMap = [:] + for(def d : dbcursor){ + def regionMap = retMap.get(d.city) + if(!regionMap){ + regionMap = [:] + retMap.put(d.city, regionMap) + } + regionMap.put(d.district, d.alias) + } + + return retMap + } + + static def queryAllStreetNameByCity(city){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + q.put("city", city) + + def keys = new BasicDBObject() + keys.put("street",1) + keys.put("alias",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def retMap = [:] + for(def d : dbcursor){ + retMap.put(d.street, d.alias) + } + + return retMap + } + static def queryCommunityByCity(){ def q = new BasicDBObject() @@ -4825,7 +4920,72 @@ class CommonDatautils { return townMap } + static def queryMzbRegion(city){ + def q = new BasicDBObject() + q.put("type","区域") + q.put("city",city) + def keys = new BasicDBObject() + keys.put("district",1) + keys.put("mzb_district_name",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def regionMap = [:] + for(def d : dbcursor){ + regionMap.put(d.district, d.mzb_district_name) + } + return regionMap + } + + static def queryTownByCity(city, fields, mzbRegionMap){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + q.put("city",city) + + def keys = new BasicDBObject() + for(def key : fields){ + keys.put(key,1) + } + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def townMap = [:] + for(def d : dbcursor){ + def key = mzbRegionMap.get(d.district) + def list = townMap.get(key) + if(!list){ + list = [] + } + list << d + townMap.put(key, list) + } + + println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss") + return townMap + } + + static def queryTownByCity(city, fields){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + q.put("city",city) + + def keys = new BasicDBObject() + for(def key : fields){ + keys.put(key,1) + } + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def townMap = [:] + for(def d : dbcursor){ + def list = townMap.get(d.district) + if(!list){ + list = [] + } + list << d + townMap.put(d.district, list) + } + + println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss") + return townMap + } static def queryTownByCity(city){ @@ -4861,7 +5021,7 @@ class CommonDatautils { def q = new BasicDBObject() q.put("type","乡镇街道") - q.put("city",new BasicDBObject('$in', ["深圳"])) + def keys = new BasicDBObject() keys.put("city",1) @@ -5060,4 +5220,38 @@ class CommonDatautils { } + static def findStatIndexBySort(type, city, region, sort){ + def q = new BasicDBObject() + q.put("cat", "district") + q.put("type", type) + if(region){ + q.put("district", region) + } + if(city){ + q.put("city", city) + } + + def cur = STATS_INDEX_COLL.find(q).sort(new BasicDBObject("yearmonth", sort)).limit(1) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def obj = 0; + while(cur.hasNext()) { + obj = cur.next() + } + return obj; + } + + static def findStatIndexByMonth(type, city, region, month){ + def q = new BasicDBObject() + q.put("cat", "district") + q.put("type", type) + if(region){ + q.put("district", region) + } + if(city){ + q.put("city", city) + } + q.put("yearmonth", month) + return STATS_INDEX_COLL.findOne(q) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index 1b11ee16a..a3c119bec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -2,8 +2,10 @@ package com.fangjia.script.gj import com.fangjia.core.BlockData import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.DateUtil import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils @@ -531,6 +533,28 @@ class FiledStrand { return null } + + static def processLngAndLat(val) { + if (val) { + def map = [:] + def lng = DataHandleUtils.getDoubleRound(6, val.x) + def lat = DataHandleUtils.getDoubleRound(6, val.y) + Long gd_lat = lat * 10000000000L + Long gd_lng = lng * 10000000000L + map.lat = gd_lat + map.lng = gd_lng + + def baidu = BaiDuApiUtils.geoconv(lng, lat, 3) + Long bd_lng = baidu.x * 10000000000L + Long bd_lat = baidu.y * 10000000000L + map.lat2 = bd_lat + map.lng2 = bd_lng + return map + + } + return null + } + static def isRoad(def s){ def pattern = Pattern.compile("[路|街|大道|巷]"); def matcher = pattern.matcher(s) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy index 082e2f170..a496ed93f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy @@ -2,33 +2,97 @@ package com.fangjia.script.gj import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.DataUtil import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection -ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d-> - if(d['A'] && !d['E']){ -// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district") -// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district") - println d['A'] +" "+d['C'] -// def total = findDatas(d['A'], d['B'], d['C']) -// d['D'] = total - def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·","")) - d['E'] = c - } -}) +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +//ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d-> +// if(d['A'] && !d['E']){ +//// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district") +//// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district") +// println d['A'] +" "+d['C'] +//// def total = findDatas(d['A'], d['B'], d['C']) +//// d['D'] = total +// def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·","")) +// d['E'] = c +// } +//}) + +def URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; +DBCollection seaweed_coll = MongoUtils.getCollectionByName2("fangjia", "seaweed", URL) +def q = new BasicDBObject() +q.put("city", "上海市") +q.put("category", "district") +q.put("status", 0) +def ret = [] +def list = seaweed_coll.find(q).batchSize(5000).toList() +println "总计:"+list.size() +//for(def l : list){ +// def obj = getAmapLocation(l.city, l.region, l.name) +// if(obj){ +// ret << [l.city, l.region, l.name, obj.cityname, obj.adname, obj.name,obj.type] +// } +//} + +ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) +def addCount = new AtomicLong() +final CountDownLatch latch = new CountDownLatch(list.size()) +def total = list.size() +for(def i=0;i - if(d.name.contains(name) && d.type.contains("住宅")){ - total ++ + if(d.name.contains(name) && d.type.contains("住宅") && d.shape){ + obj = d } }) - return total; + return obj; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy index cc42938d7..954e0402b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy @@ -6,21 +6,33 @@ import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.MongoClient +import com.rabbitmq.client.AMQP class AddCityRegionPrice { public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command') + + public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; + public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl) + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", mongoDbUrl) + static main(args) { def task = new AddCityRegionPrice() // task.addRegionPrice() -// task.addCityprice() + task.addCityprice() // task.addDistrictprice() // task.addCityRentPrice() - task.addDistrictRentprice() +// task.addRegionRentPrice() +// task.addDistrictRentprice() +// task.updateRegionRentData() +// task.updateDealPriceAreaType() +// task.checkCityImport() } def addRegionPrice(){ @@ -54,7 +66,7 @@ class AddCityRegionPrice { def addCityprice(){ InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch() - def citys = CommonDatautils.findAllCity().reverse() + def citys = ["乐东"]// CommonDatautils.findAllCity().reverse() def add = 0 for(def c : citys){ def startObj = CommonDatautils.queryCityOrRegionPrice(c, null, 1) @@ -125,6 +137,35 @@ class AddCityRegionPrice { } } + def addRegionRentPrice(){ + InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch(); + def citys = CommonDatautils.findAllCity() + for(def c : citys){ + + def regions = CommonDatautils.getCityRegions(c) + for(def r : regions){ +// if(r!="南川"){ +// continue +// } + def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, r, null, 1) + def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, r, null, -1) + if(startObj && endObj){ + def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409") + months.each {m-> +// println m + def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd"))) + def obj = CommonDatautils.findSqmetreRent(c, r, null, lastDay) + if(!obj){ + inserBatch.saveCityCalcCommand(c, r, null, lastDay, "fangjia-priceanalysis-0") + println "[add] "+c+" "+r+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd") + } + } + } + } + + } + } + def addDistrictRentprice(){ InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch(); def citys = CommonDatautils.findAllCity() @@ -158,4 +199,72 @@ class AddCityRegionPrice { } } + + def updateRegionRentData(){ + def q = new BasicDBObject("cat", "region") + def keys = new BasicDBObject() + keys.put("markup", 1) + def dbcursor = SQMETRE_RENT_COLL.find(q, keys).batchSize(5000) + def index = 1 + for(def d : dbcursor){ + if(!(d.markup instanceof Double)){ + def udata = new BasicDBObject() + double markup = 1.0 + markup = d.markup * markup + udata.put("markup", markup) + SQMETRE_RENT_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + println index + index++ + } + } + } + + + def updateDealPriceAreaType(){ + def index = 1 + def citys = ["凉山州","福州","南京","佛山","沈阳","苏州"] + citys.each {c-> + def q = new BasicDBObject() + q.put("city", c) +// q.put("price_flag", "国信达成交-推导") + def dbcursor = DEAL_PRICE_COLL.find(q) + for(def d : dbcursor){ +// if(!(d.area instanceof Double)){ +// def udata = new BasicDBObject() +// double area = 1.0 +// area = d.area * area +// udata.put("area", area) +// DEAL_PRICE_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) +// println c+" "+index +// index++ +// } + if(!(d.total_price instanceof Integer)){ + def udata = new BasicDBObject() + udata.put("total_price", d.total_price.toString().toDouble().toInteger()) + DEAL_PRICE_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + println c+" "+index + index++ + } + } + } + } + + def checkCityImport(){ + def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] + for(def c : citys){ + def startObj = CommonDatautils.findStatIndexBySort("城市", c, null, 1) + def endObj = CommonDatautils.findStatIndexBySort("城市", c,null, -1) + if(startObj && endObj){ + def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202409") + months.each {m-> + def obj = CommonDatautils.findStatIndexByMonth("城市", c,null, m.toInteger()) + if(!obj){ + println "[add] "+c+" "+m + } + } + }else{ + println c +" 一个也没有" + } + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy new file mode 100644 index 000000000..d250b8944 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy @@ -0,0 +1,376 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl +import groovy.json.JsonSlurper +import net.sf.json.JSONObject + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 添加高德地图,百度地图围栏 + */ +@BatchOwner(Watcher.GaoJuan) +class AddDistrictShapeBatch extends BaseBatch{ + + def fullCityMap = CommonDatautils.queryFullNameCity() + def mcaDistMap = CommonDatautils.queryDistNameCityByMapping() + + DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) + public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com") + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + static main(args) { + runBatch(new AddDistrictShapeBatch(), args) + } + + @Override + public void process(Object args) { + +// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" ) + def citys = CommonDatautils.findAllCity() + def index = 1 + for(def c : citys){ + println c +" "+index + index ++ + def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"]) +// baiduMatch(c, districts) + gaodeMatch(c, districts) + + } + } + + def baiduMatch(city,districts){ + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + final CountDownLatch latch = new CountDownLatch(districts.size()) + + if(fullCityName){ + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + for(def i=0;i1) { +// resultMap.type = "MultiPolygon" +// }else{ + resultMap.type = "polygon" +// } + o.put("shape", resultMap) + + def q = new BasicDBObject("comm_id", d._id.toString()) + seaweed_fence_coll.update(q, new BasicDBObject('$set', o),true,false) +// seaweed_fence_coll.save(o) + println "[save] 围栏 ( "+d.city +" "+d.region +" "+d.name+" )" + return fenceMap.data_id + } + return null + } + + def findBaiduDatas(city, district, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("district", district) + q.put("name", name) + q.put('std_tag', '房地产;住宅区') + q.put("guoke_geo", new BasicDBObject('$ne', null)) + def dbcursor = BAIDU_POI_COLL.find(q) + def map = [:] + println dbcursor.count() + for(def d : dbcursor){ + def arrs = d.gcj02_guoke_geo.split("\\|") + + def coordinatesStr = arrs[2].toString().substring(2, arrs[2].toString().size()-1 ) + // 将字符串分割成列表 + def coordinatePairs = coordinatesStr.split(',') + def coordinatesList = [] + for (int i = 0; i < coordinatePairs.size(); i += 2) { + // 将字符串坐标转换为浮点数 + double longitude = coordinatePairs[i] as double + double latitude = coordinatePairs[i + 1] as double + + // 将坐标点作为一个元组添加到集合中 + coordinatesList << [longitude, latitude] + } + if(coordinatePairs){ + map.data_id = d._id + map.coordinatePairs = coordinatesList + } + } + return map; + } + + + def gaodeMatch(city,districts){ + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + final CountDownLatch latch = new CountDownLatch(districts.size()) + + if(fullCityName){ + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + for(def i=0;i + def newName = d.name.replace("-","").replace("·","") + if(newName == name && d.type.contains("住宅") && d.shape){ + obj = d + } + }) + return obj; + } + + + def findGaoDeDatas2(city, region, name){ + def header = [:] + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString() + String seawaterUrl = "http://192.168.0.206:9200/amap_poi_detail/_search" + def body = """ + { + "_source": { + "includes": ["name", "shape", "id"] + }, + "query": { + "bool": { + "must": [ + { + "term": { + "cityname" : "${city}" + } + }, + { + "term": { + "adname" : "${region}" + } + }, + { + "match": { + "name": {"query": "${name}","operator": "and"} + } + }, + { + "match": { + "type": "住宅" + } + }, + { + "exists": { + "field": "shape" + } + } + ] + } + } + } + """ + HttpRequest request = new HttpRequest() + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + def response = new JsonSlurper().parseText(res) + def obj = null + def fenceMap = [:] + if(response && response.hits){ + def hits = response.hits.hits + if(hits){ + def d = hits[0] + def source = d._source + def newName = source.name.replace("-","").replace("·","").replace("(建设中)","") + if(newName == name){ + fenceMap.id = source.id + def list = [] + def points_arr = source.shape.coordinates + for(def points : points_arr){ + def rets = [] + for(def point : points){ + def lng = DataHandleUtils.getDoubleRound(6, point[0].toString().toDouble()) + def lat = DataHandleUtils.getDoubleRound(6, point[1].toString().toDouble()) + rets << [lng, lat] + } + list << rets + } + if(list.size()==1){ + fenceMap.coordinates = list[0] + }else{ + fenceMap.coordinates = list + } + + } + } + } + return fenceMap + + } + + + + + + def findSeaweedFenceExists(commid){ + def q = new BasicDBObject() + q.put("comm_id", commid) + return seaweed_fence_coll.findOne(q) + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy new file mode 100644 index 000000000..f6a8c1208 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy @@ -0,0 +1,234 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 小区自动入库(新增小区,每月运行一次) + */ +class AutoAddDistrictBatch extends BaseBatch{ + + final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018) + public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + def fullCityMap = CommonDatautils.queryFullNameCity() + def mcaDistMap = CommonDatautils.queryDistNameCityByMapping() + + static main(args) { + runBatch(new AutoAddDistrictBatch(), args) + } + + @Override + void process(Object args) { + //查找第三方匹配没有匹配上的小区 + def citys = CommonDatautils.findAllCity() + def list = [] + list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"] + for(def c : citys){ + def noMatchDatas = findNoMatchResult(c) + println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size() + def rets = checkDatas(noMatchDatas.keySet()) + list.addAll(rets) + } + + def fileName = "/home/goojia/gj/add_dist_info.xlsx" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx" + } + ExcelUtils.write(fileName, list, "待入库数据") + } + + def checkDatas(keysets){ + def noMatchCount = keysets.size() + final CountDownLatch latch = new CountDownLatch(noMatchCount) + + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + for(def i=0;i + if(d.name == name && d.type.contains("住宅") && d.shape){ + obj = d + } + }) + return obj; + } + + def findTrades(city, region, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + return DEAL_PRICE_COLL.count(q) + } + + def findNoMatchResult(def c){ + def q = new BasicDBObject() + q.put("match_id", new BasicDBObject('$exists',false)) + q.put("old_match_id", new BasicDBObject('$exists',false)) + q.put("city", c) + def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000) + def rMap = [:] + for(def d : dbcursor){ + if(d.estate_type){ + if(d.estate_type.contains("自建房")){ + continue + } + if(d.estate_type.contains("购物中心")){ + continue + } + if(d.estate_type.contains("住宅底商") || d.estate_type.contains("酒店") || d.estate_type.contains("商铺")){ + continue + } + if(d.estate_type.contains("办公") || d.estate_type.contains("写字楼")){ + continue + } + }else{ + continue + } + def key = d.city + "##"+d.district+"##"+d.name + def rets = rMap.get(key) + if(!rMap.containsKey(key)){ + rets = [] + rMap.put(key, rets) + } + rets << d + } + return rMap + } + + def createData(city, region, name, address, lng, lat){ + def udata = new BasicDBObject() + udata.put("city", city) + udata.put("region", region) + name = name.replaceAll('[·\\(\\)()]|·', '').replace("建设中", "") + udata.put("name", name) + + def addr = AddressUtil.formatAddress(city, address) + if(addr){ + udata.put("address", addr) + } + + udata.put("status", "0") + udata.put("name_cat", "0") + udata.put("cat", "district") + udata.put("username", "高娟") + + def estateType2 = "普通住宅" + def type = "" + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + udata.put("estateType2", estateType2) + if(lng){ + Long gd_lng = lng*10000000000 + udata.put("lng", gd_lng) + } + if(lat){ + Long gd_lat = lat * 10000000000 + udata.put("lat", gd_lat) + } + FangjiaClientUtils.saveOrUpdateDistrict(udata) + } + + def findBaiduDatas(city, district, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("district", district) + q.put("name", name) + q.put('std_tag', '房地产;住宅区') + def count = BAIDU_POI_COLL.count(q) + return count; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 56915936b..f722db94c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -46,7 +46,7 @@ class DevProcess { def task = new DevProcess(); // task.run() // task.parseCityAndDist() - task.devSplitRoad() +// task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() // task.matchDev() @@ -63,7 +63,7 @@ class DevProcess { // task.createDevMapping() // task.deleteAddr() // task.deleteAddr2() -// task.synchDevTo100() + task.synchDevTo100() // task.addNewDev2() // task.addAlias() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index d56afba8a..49d2547b7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -20,6 +20,10 @@ class DevUpdateTestBatch { def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection development_zone_43_CONN = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") + + + DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) @@ -42,7 +46,7 @@ class DevUpdateTestBatch { def addDev(path){ ExcelUtils.modify(path,{ d-> if(!d['I'] && d['A']){ - def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A'])) + def obj = development_zone_43_CONN.findOne(new BasicDBObject("name", d['A'])) if(!obj){ def o = new BasicDBObject() o.put("province", d['B']) @@ -118,6 +122,8 @@ class DevUpdateTestBatch { q.put("address", d['C']) }else if(d['B'] == "园区"){ q.put("park", d['C']) + }else if(d['B'] == "Poi"){ + q.put("poi", d['C']) } def mappingObj = DEV_MAPPING_100_CONN.findOne(q) if(!mappingObj){ @@ -133,6 +139,8 @@ class DevUpdateTestBatch { o.put("address", d['C']) }else if(d['B'] == "园区"){ o.put("park", d['C']) + }else if(d['B'] == "Poi"){ + o.put("poi", d['C']) } def date = new Date() o.put("c_date", date) @@ -160,7 +168,7 @@ class DevUpdateTestBatch { println d['A'] +" 不存在" } } - },1) + },2) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy index 35c1fa9db..ee54817e6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy @@ -28,7 +28,7 @@ class GXDdistrictMatch { static main(args) { def task = new GXDdistrictMatch(); // task.run() - task.addOldMatch() + task.addOldMatch2() // task.runDb() // task.updateDistStatus()//恢复name_cat不正常数据 @@ -68,6 +68,32 @@ class GXDdistrictMatch { } + def addOldMatch2(){ + //老匹配 + def fileName = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx"//11城重新匹配.xlsx + ExcelUtils.multiThreadingModify(fileName, { d -> + println d['C'] + if(!d['AJ']){ + def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D']) + if (jsonObject != null && !jsonObject.containsKey("result")) { + d['AJ'] = jsonObject.get("id") + d['AK'] = jsonObject.get("city") + d['AL'] = jsonObject.get("district") + d['AM'] = jsonObject.get("name") + d['AN'] = jsonObject.get("address") + + def dbo = CommonDatautils.findDistrictById(jsonObject.get("id")) + if(dbo){ + d['AO'] = dbo.status + d['AP'] = dbo.name_cat + d['AQ'] = dbo.estate_type2 + } + } + } + }); + + } + def runDb(){ def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx" ExcelUtils.modify(fileName, { d -> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy new file mode 100644 index 000000000..7536d908f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -0,0 +1,350 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.RegionData +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.update.UpdateRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +/** + * 区域标准化 + * 1.小区库区域标准化,包含小区,商铺,写字楼 + * 2.poi数据库中 区域标准化 + */ +class RegionStandardBatch extends BaseBatch{ + + def fullCityMap = CommonDatautils.queryFullNameCity() + def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() + def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() + + static main(args) { + runBatch(new RegionStandardBatch(), args) + } + + @Override + void process(Object args) { +// seaweedRegionStandard() + poiRegionStandard() + } + + def seaweedRegionStandard(){ + seaweedRegionStandardByCat("district") + } + def poiRegionStandard(){ + def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"] + def index = 0 + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + index ++ + if(index<32){ + continue + } + if(runCitys.contains(city)){ + continue + } + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + if(fullCityName){ + + def rMap = regionFullNameMap.get(fullCityName) + if(rMap){ + def set = rMap.keySet() + if(set.size()>1){ + def rets = findEsPoiByCity(fullCityName) + println city +" "+rets.size() + poiHaveRegionProcess(city, set, rMap, rets) + }else{ +// haveStreetProcess(city, set, cat) + } + }else{ + println "没找区域:"+city + } + }else{ + println "没找到相应的全称:"+city + } + + } + + } + + + def seaweedRegionStandardByCat(cat){ + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + + def rMap = regionMap.get(city) + if(rMap){ + def set = rMap.keySet() + if(set.size()>1){ + haveRegionProcess(city, set, rMap, cat) + }else{ + haveStreetProcess(city, set, cat) + } + } + } + + } + + def haveRegionProcess(city, set, rMap, cat){ + def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"]) + for(def d : districts){ + if(set.contains(d.region)){ + continue + } + def updateRegion = null + for(def key : set){ + def alias = rMap.get(key) + if(alias.contains(d.region)){ + updateRegion = key + break; + } + } + if(updateRegion){ + def udata = new BasicDBObject() + udata.put("region", updateRegion) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion + DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + }else{ + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name +" 状态修改为:删除" + DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + } + } + } + + def haveStreetProcess(city, set, cat){ + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName) + if(streetMap){ + set = streetMap.keySet() + def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"]) + for(def d : districts){ + if(set.contains(d.region)){ + continue + } + + def updateRegion = null + def flag = null + for(def key : set){ + def alias = streetMap.get(key) + if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){ + def newRegion = RegionData.formatRegion(d.city, key) + if(newRegion!=d.region){ + updateRegion = newRegion + } + flag = "pass" + break; + } + } + if(updateRegion){ + def udata = new BasicDBObject() + udata.put("region", updateRegion) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion +// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + }else{ + if(flag!="pass"){ + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name +" 状态修改为:删除" +// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + } + } + } + } + } + + def findEsPoiByCity(city){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + def rets = [] + + String[] includes = ["city", "district", "name", "township", "status", "id"] + def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder,includes) + for(def d: datas){ + def o = new BasicDBObject() + o.put("city", d.city) + o.put("district", d.district) + o.put("name", d.name) + o.put("township", d.township) + o.put("location", d.location) + o.put("status", d.status) + o.put("id", d.id) + rets << o + } + return rets + } + + + def poiHaveRegionProcess(city, set, rMap, datas){ + def results = [] + for(def d : datas){ + if(set.contains(d.district)){ + continue + } + def updateRegion = null + for(def key : set){ + def alias = rMap.get(key) + if(alias.contains(d.district)){ + updateRegion = key + break; + } + } + if(updateRegion){ + def m = [:] + m.put(d.id, ["district":updateRegion]) + results << m + if(results.size()%100==0){ + updateRegionByEs(results) + results.clear() + } + println d.city +" "+d.district+" "+d.name +" "+d.status+" 区域修改为:"+updateRegion + + }else{ + def m = [:] + m.put(d.id, ["status":1]) + results << m + if(results.size()%100==0){ + updateRegionByEs(results) + results.clear() + } + + println d.city +" "+d.district+" "+d.name +" "+d.status+" 状态修改为:删除" + + } + } + + if(results.size()>0){ + updateRegionByEs(results) + results.clear() + } + println city +" 处理完成" + } + + + + def updateRegionByEs(rows){ + def client = ElasticUtils89.getClient("192.168.0.89") + def bulkRequest = new BulkRequest() + + rows.each { row -> + def id = row.keySet().iterator().next() + def doc = row.get(id) + + def updateRequest = new UpdateRequest("didata_map_poi", id) + updateRequest.doc(doc, XContentType.JSON) + bulkRequest.add(updateRequest) + } + + try { + // 执行批量更新请求 + def bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT) + + // 检查响应 + if (bulkResponse.hasFailures()) { + println("批量更新中有失败的操作:") + bulkResponse.buildFailureMessage().each { println(it) } + } else { + println("批量更新成功!") + } + } catch (Exception e) { + e.printStackTrace() + } finally { + // 关闭客户端 + try { + client.close() + } catch (Exception e) { + e.printStackTrace() + } + } + + } + +// def queryPoiDatasByCity(){ +// def esUrl = "http://localhost:9200/your_index/_search" +// def pageSize = 1000 // 每页返回的记录数 +// def scrollSize = '1m' // scroll上下文的生存时间 +// +//// 初始化HttpClient和ObjectMapper +// CloseableHttpClient httpClient = HttpClients.createDefault() +// ObjectMapper objectMapper = new ObjectMapper() +// +//// 初始化scroll请求 +// def scrollRequest = """ +//{ +// "size": $pageSize, +// "query": { +// "match_all": {} +// }, +// "scroll": "$scrollSize" +//} +//""" +// +//// 发送scroll初始化请求 +// HttpGet scrollInitRequest = new HttpGet(esUrl) +// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8")) +// scrollInitRequest.setHeader("Content-Type", "application/json") +// +// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest) +// HttpEntity entity = scrollInitResponse.getEntity() +// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString() +// +//// 存储所有结果 +// def allResults = [] +// +//// 发送scroll请求获取数据,直到没有更多数据为止 +// while (true) { +// def scrollRequestWithId = """ +// { +// "scroll_id": "$scrollId", +// "scroll": "$scrollSize" +// } +// """ +// +// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll") +// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8")) +// scrollGetRequest.setHeader("Content-Type", "application/json") +// +// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest) +// entity = scrollGetResponse.getEntity() +// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class) +// +// def hits = responseMap.get("hits").get("hits") +// if (hits.isEmpty()) { +// break // 没有更多数据,跳出循环 +// } +// +// // 合并当前页的数据 +// allResults.addAll(hits.map({ it.get("_source") })) +// +// // 更新scrollId以进行下一次请求 +// scrollId = responseMap.get("_scroll_id").toString() +// } +// +// // 关闭HttpClient +// httpClient.close() +// +// // 输出或处理全量数据 +// println(allResults) +// +// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。 +// } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionUpdateBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionUpdateBatch.groovy new file mode 100644 index 000000000..6fce88fa6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionUpdateBatch.groovy @@ -0,0 +1,65 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +//更新第三方均价表中小区区域 +class RegionUpdateBatch { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new RegionUpdateBatch() + task.run() + + } + def run(){ + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + def districtMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["city","region", "name"]) + def keys = new BasicDBObject() + keys.put("city", 1) + keys.put("region", 1) + keys.put("name", 1) + keys.put("seaweed_id", 1) + def statistics = AVERAGE_STATISTICS.find(new BasicDBObject("city", city), keys) + def total = statistics.size() + final CountDownLatch latch = new CountDownLatch(total) + def totalCount = new AtomicLong() + for(def i=0;i udata.put(k,v) } + udata.removeField("_id") udata.put("m_date", new Date()) - thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata)) + thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata),false, false) }else{ sd.put("m_date", new Date()) thirdSeaweedMappingColl.save(sd) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy new file mode 100644 index 000000000..b776efc53 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy @@ -0,0 +1,111 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 乡镇街道 判断 中心点是否在围栏或一个围栏是否包含多个中心点 + */ +@BatchOwner(Watcher.GaoJuan) +class TownCenterBatch extends BaseBatch{ + + DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") + + static main(args) { + runBatch(new TownCenterBatch(), args) + } + + @Override + void process(Object args) { + def city = "上海市" + def shapes = findHasShapeTown(city) + def centers = findAllCenters(city) + def retMap = checkCenter(shapes, centers) + retMap.each {key,values-> + if(values.size()>1){ + println values + } + } + + } + + def checkCenter(shapes, centers){ + def map = [:] + def index = 0 + for(def town : shapes){ + index ++ + def points = InsidePolygon.createPoint4(town.shape.coordinates) + //1.先判断当前镇的中心点是否在围栏中 + if(town.location){ + def flag = InsidePolygon.isInArea(town.location.lon, town.location.lat, points) + if(!flag){ + println town.city + " "+town.district+" "+town.street+" 不在本身围栏中" + }else{ + def list = map.get(town.city + "#"+town.district+"#"+town.street) + if(!list){ + list = [] + map.put(town.city + "#"+town.district+"#"+town.street, list) + } + list << town.city + "#"+town.district+"#"+town.street + } + } + //2.判断是否有其它点在围栏中 + for(def center : centers){ + if(center._id.toString() != town._id.toString()){ + def flag = InsidePolygon.isInArea(center.location.lon, center.location.lat, points) + if(flag){ + def list = map.get(town.city + "#"+town.district+"#"+town.street) + if(!list){ + list = [] + map.put(town.city + "#"+town.district+"#"+town.street, list) + } + list << center.city + "#"+center.district+"#"+center.street + println center.city + " "+center.district+" "+center.street+" 在 "+town.city + " "+town.district+" "+town.street+" 围栏中" + } + } + } + def list = map.get(town.city + "#"+town.district+"#"+town.street) + if(list){ + println index+" "+town.city + "#"+town.district+"#"+town.street + " "+list.size() + } + } + return map + } + + def findHasShapeTown(city){ + def q = new BasicDBObject() + q.put("type", "乡镇街道") + q.put("shape", new BasicDBObject('$exists', true)) + if(city){ + q.put("city", city) + } + + def keys = new BasicDBObject() + keys.put("location", 1) + keys.put("city", 1) + keys.put("district", 1) + keys.put("street", 1) + keys.put("shape", 1) + return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).toList() + } + + def findAllCenters(city){ + def q = new BasicDBObject() + q.put("type", "乡镇街道") + q.put("location", new BasicDBObject('$exists', true)) + if(city){ + q.put("city", city) + } + def keys = new BasicDBObject() + keys.put("location", 1) + keys.put("city", 1) + keys.put("district", 1) + keys.put("street", 1) + return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).toList() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy index 0f61101ea..4ecd49ed0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.GeocodeUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils @@ -77,7 +78,7 @@ class TownMiddleProcess { //对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表 // updateCommunityDiffStreet() // updateTownPoline()//更新乡镇街道围栏 - updateTownPolineBatch()//批量更新乡镇街道围栏 +// updateTownPolineBatch()//批量更新乡镇街道围栏 // checkTown() //区域民政部名称更新到乡镇街道中 // updateTownshipByRegion() @@ -93,6 +94,8 @@ class TownMiddleProcess { // addTownCenter()//百度接口添加中心点 // checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中 + + addSpecialCityTownAlias() } @@ -900,41 +903,92 @@ class TownMiddleProcess { } def updateTownPolineBatch(){ + def i = 0 ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx",{ row-> if(row['R'] && row['R']=="新增围栏"){ - def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D']) - println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q) - def d = TOWNPOLYLINE_COLL.findOne(q) - if(d){ - def resultMap = [:] - resultMap.type = "polygon" - def arrs = d.polyline.split("\\|") - def rets = [] - for(def s : arrs){ - def points = s.split(";") - def pointList = [] - for(def pointStr : points){ - def pointArr = pointStr.split(",") - def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble()) - def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble()) - pointList << [lon, lat] + i++ + if(i>=134){ + def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D']) + println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q) + def d = TOWNPOLYLINE_COLL.findOne(q) + if(d){ + def resultMap = [:] + def bd_resultMap = [:] + + resultMap.type = "polygon" + bd_resultMap.type = "polygon" + + def arrs = d.polyline.split("\\|") + def rets = [] + def bd_rets = [] + + println row['N'] +" 共有:"+arrs.size()+" 个围栏" + def index = 1 + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + def bd_pointList = [] + + for(def pointStr : points){ + def pointArr = pointStr.split(",") + def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble()) + def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble()) + pointList << [lon, lat] + + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(lon, lat, 3) + def bd_lng = baidu.x + def bd_lat = baidu.y + bd_pointList << [bd_lng, bd_lat] + } + + if(index<=1){ + resultMap.coordinates = pointList + bd_resultMap.coordinates = bd_pointList + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + udata.put("operator", "高娟") + udata.put("m_date", new Date()) + udata.put("bd_shape",bd_resultMap) + + def updateQuery = new BasicDBObject("name",row['N']) + DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata)) + }else{ + def dbo = DEVELOPMENT_ZONE_43_COLL.findOne(new BasicDBObject("name",row['N'])) + def obj = new BasicDBObject() + dbo.each {k,v-> + if(k!='_id'){ + obj.put(k,v) + } + } + + resultMap.coordinates = pointList + bd_resultMap.coordinates = bd_pointList + obj.put("shape", resultMap) + obj.put("operator", "高娟") + obj.put("m_date", new Date()) + obj.put("bd_shape",bd_resultMap) + obj.put("create_time", new Date()) + obj.put("block", dbo.name+"-地块"+index) + DEVELOPMENT_ZONE_43_COLL.save(obj) + } + +// rets << pointList +// bd_rets << bd_pointList + index++ } - rets << pointList + + +// if(rets.size()>1) { +// resultMap.type = "MultiPolygon" +// bd_resultMap.type = "MultiPolygon" +// } +// println d.city+" "+d.region+" "+d.town+" "+rets.size() + row['S'] = "已完成" } - resultMap.coordinates = rets - if(rets.size()>1) { - resultMap.type = "MultiPolygon" - } - println d.city+" "+d.region+" "+d.town+" "+rets.size() - - def udata = new BasicDBObject() - udata.put("shape", resultMap) - - - def updateQuery = new BasicDBObject("name",row['N']) - DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata)) - row['S'] = "已完成" } + } }) } @@ -1249,6 +1303,57 @@ class TownMiddleProcess { for(def d : dbcursor){ } + } + def addSpecialCityTownAlias(){ + def fullCityMap = CommonDatautils.queryFullNameCity() + + def citys = ["东莞","中山","嘉峪关","昆玉","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"] + def regionMap = CommonDatautils.getAllCityFullRegions() + for(def city : citys){ + def townMap = regionMap.get(city) + def streets = findAllStreetByCity(city) + for(def s : streets){ + def set = townMap.keySet() + def alias = null + for(def name : set){ + def townObj = townMap.get(name) + if(townObj && townObj.pattern && townObj.pattern.contains(s.street)){ + alias = townObj.pattern + break; + } + } + if(alias){ + def newAlias = [] + for(def n : alias){ + if(n!=s.street){ + newAlias << n + } + } + if(alias){ + def udata = new BasicDBObject() + udata.put("alias", newAlias) + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id", s._id), new BasicDBObject('$set', udata),false, false) + } + } + } + } + } + + + def findAllStreetByCity(city){ + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + + def q = new BasicDBObject() + q.put("type", "乡镇街道") + q.put("city", fullCityName) + + def keys = new BasicDBObject() + keys.put("street", 1) + return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy new file mode 100644 index 000000000..95ecf6a9f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -0,0 +1,279 @@ +package com.fangjia.script.gj.data_handle + + +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.mongodb.BasicDBObject +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.update.UpdateRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +/** + * 区域标准化 + * 1.小区库乡镇街道标准化,包含小区,商铺,写字楼 + * 2.poi数据库中乡镇街道标准化 + */ +class TownStandardBatch extends BaseBatch{ + + def fullCityMap = CommonDatautils.queryFullNameCity() + def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() + def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() + + def poiEsUrl = "http://192.168.0.89:9200/didata_map_poi/_search" + + def elasticsearchUrl = "http://192.168.0.89:9200/_search/scroll" + def initialSearchUrl = "http://192.168.0.89:9200/didata_map_poi/_search?scroll=1m" + def scrollTime = "1m" // Scroll context time-to-live + def pageSize = 1000 // Number of hits to return per scroll + + static main(args) { + runBatch(new TownStandardBatch(), args) + } + + + @Override + void process(Object args) { + TownStandard() + } + + def TownStandard(){ +// townStandardByCat("office") + townStandardByPoi() + } + def townStandardByCat(def cat){ + //"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德", + def citys = ["重庆","龙岩","南平","漳州","福州","厦门","泉州","莆田","三明","宁德"]//CommonDatautils.findAllCity() + for(def city : citys){ + + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + + def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName) + def shortRegionNameMap = CommonDatautils.getRegionShortName(fullCityName) + def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"], mzbRegionMap) + if(townMap){ + def regionSet = townMap.keySet() + if(regionSet.size()>1){ + haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap) + }else{ + println city +" 无区域" + } + } + } + } + + def townStandardByPoi(){ + def citys = citys = ["重庆","龙岩","南平","漳州","福州","厦门","泉州","莆田","三明","宁德"]//CommonDatautils.findAllCity() + for(def city : citys){ + + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + if(fullCityName){ + def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"]) + def rMap = regionFullNameMap.get(fullCityName) + if(rMap){ + def set = rMap.keySet() + def rets = findEsPoiByCity(fullCityName) + println city +" "+rets.size() + if(set.size()>1){ + poiHaveRegionProcess(fullCityName, townMap, rets) + }else{ +// haveStreetProcess(city, set, cat) + } + } + }else{ + println "没找到相应的全称:"+city + } + } + } + def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){ + def results = [] + if(fullCityName){ + def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township"]) + for(def d : districts){ + if(d.township){ + def towns = null + def updateTown = null + for(def m : townMap){ + def fullRegionName = shortRegionNameMap.get(d.region) + if(m.key == fullRegionName){ + towns = m.value + break + } + } + + def opt = true + if(towns){ + for(def town : towns){ + def townAlias = [] + if(town.alias){ + townAlias = town.alias + } + townAlias << town.street + if(townAlias.contains(d.township)){ + if(d.township!=town.street){ + updateTown = town.street + }else{ + opt = false + } + break + } + } + } + if(opt){ + if(updateTown){ + def udata = new BasicDBObject() + udata.put("township", updateTown) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown +// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + }else{ + def udata = new BasicDBObject() + udata.put("township", "") + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值" +// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + } + } + + } + } + } + } + + def poiHaveRegionProcess(fullCityName, townMap, rets){ + def results = [] + if(fullCityName){ + for(def d : rets){ + if(d.township){ + def towns = null + def updateTown = null + for(def m : townMap){ + def townDatas = townMap.get(d.district) + if(townDatas){ + towns = townDatas + break + } + } + + def opt = true + if(towns){ + for(def town : towns){ + def townAlias = [] + if(town.alias){ + townAlias = town.alias + } + townAlias << town.street + if(townAlias.contains(d.township)){ + if(d.township!=town.street){ + updateTown = town.street + }else{ + opt = false + } + break + } + } + } + if(opt){ + if(updateTown){ + def m = [:] + m.put(d.id, ["township":updateTown]) + results << m +// if(results.size()%100==0){ +// updateByEs(results) +// results.clear() +// } + println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown + }else{ + def m = [:] + m.put(d.id, ["township":""]) + results << m +// if(results.size()%100==0){ +// updateByEs(results) +// results.clear() +// } + println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值" + } + } + + } + } + } + + if(results.size()>0){ + updateByEs(results) + results.clear() + } + println fullCityName +" 处理完成" + } + + + + + def updateByEs(rows){ + def client = ElasticUtils89.getClient("192.168.0.89") + def bulkRequest = new BulkRequest() + + rows.each { row -> + def id = row.keySet().iterator().next() + def doc = row.get(id) + + def updateRequest = new UpdateRequest("didata_map_poi", id) + updateRequest.doc(doc, XContentType.JSON) + bulkRequest.add(updateRequest) + } + + try { + // 执行批量更新请求 + def bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT) + + // 检查响应 + if (bulkResponse.hasFailures()) { + println("批量更新中有失败的操作:") + bulkResponse.buildFailureMessage().each { println(it) } + } else { + println("批量更新成功!") + } + } catch (Exception e) { + e.printStackTrace() + } finally { + // 关闭客户端 + try { + client.close() + } catch (Exception e) { + e.printStackTrace() + } + } + } + + def findEsPoiByCity(city){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("status", 0)) + def rets = [] + + String[] includes = ["city", "district", "name", "township", "status", "id"] + def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder, includes) + for(def d: datas){ + def o = new BasicDBObject() + o.put("city", d.city) + o.put("district", d.district) + o.put("name", d.name) + o.put("township", d.township) + o.put("status", d.status) + o.put("id", d.id) + rets << o + } + return rets + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy index 2a6c1d7a3..d99e249c9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy @@ -42,7 +42,7 @@ class TownshipStrandUpdate { // task.updateTown() // task.stat() -// task.updateDiffRegionTown() + task.updateDiffRegionTown() // task.cleanTown() // task.checkTownFromMongo() // task.updateTownMapping() @@ -423,29 +423,29 @@ class TownshipStrandUpdate { } def updateDiffRegionTown(){ - def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx" - ExcelUtils.multiThreadingModify(fileName,{d-> - def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop") + def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx" + ExcelUtils.modify(fileName,{d-> + def lists = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district") d['AA'] = lists.size() //只处理库中只有一个的 - if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){ - def newRegion = RegionData.formatRegion(d['B'], d['M']) + if(lists && lists.size()==1 && d['R']){ + def newRegion = RegionData.formatRegion(d['A'], d['R']) if(newRegion){ - if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中 +// if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中 def dbo = lists[0] - def arrs = d['X'].split(" ") +// def arrs = d['X'].split(" ") def udata = new BasicDBObject() udata.put("region", newRegion) udata.put("m_date", new Date()) - udata.put("township", arrs[1]) +// udata.put("township", arrs[1]) DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata)) - CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟") + CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "district", "update-region","高娟") d['AB'] = "已处理" - } +// } } } - }) + },1) } def cleanTown(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index 76fa40fb2..3b3aed61a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -47,7 +47,7 @@ class DistrictFieldAddBatch extends BaseBatch{ //"alias":"alias", //ownership_type:权属类型 为商品房、经适房、动迁房等 //house_property 产权性质:国有产权 "house_property":"网站没有", - def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer", + def fieldsMap = ["lng":"location","estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer", "virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households", "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure", "build_type":"build_type", "land_property":"ownership_type","live_date":"live_date", @@ -115,7 +115,7 @@ class DistrictFieldAddBatch extends BaseBatch{ println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%" def precentMap = fieldUpdate(totalCount, updateRets) println c +" "+precentMap.toString() - //updateDB(c, updateRets) + updateDB(c, updateRets) println "------------------------------------------------------------------------------" def pre1 = 0 def pre2 = 0 @@ -212,6 +212,11 @@ class DistrictFieldAddBatch extends BaseBatch{ def rets = [] rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"] for(def matchDist : matchRets){ + +// if(!["智富中心","龙山新寓","天怡瀚园","远盛凤林府","京口路小区","梦溪路小区","浩创优尚","溪山丽苑","旭园"].contains(matchDist.name)){ +// continue +// } + def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields) if(!dist){ println c+" "+matchDist.match_id+" 不存在" @@ -306,6 +311,15 @@ class DistrictFieldAddBatch extends BaseBatch{ if(newVal){ newValues.put(seaweedfield, newVal) } + if(seaweedfield == "lng"){ + def locationMap = FiledStrand.processLngAndLat(esVal) + if(locationMap){ + newValues.put("lat", locationMap.lat) + newValues.put("lng", locationMap.lng) + newValues.put("lat2", locationMap.lat2) + newValues.put("lng2", locationMap.lng2) + } + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/RemoveDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/RemoveDistrict.groovy new file mode 100644 index 000000000..2f49dc227 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/RemoveDistrict.groovy @@ -0,0 +1,56 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject + +/** + * 临时移除 相同名称即在正常状态下有,在删除状态下也有的小区 + */ +class RemoveDistrict { + + public static void main(String[] args) { + def task = new RemoveDistrict() + task.run() + } + + def run(){ + def citys = CommonUtils.findAllCity() + def cat = "shop" + def keys = [] + citys.each {c-> + def districts = CommonDatautils.findAllDistricts(c, cat, ["city", "region", "name", "status"]) + districts.each {it-> + def key = it.city +"##"+it.region+"##"+it.name + if(!keys.contains(key)){ + keys << key + }else{ + println it.toString()+" 删除" + DbUtils.seaweed.remove(new BasicDBObject("_id",it._id)) + } +// def list = CommonDatautils.findAllStatusDistrict(it.city, it.region, it.name, cat, ["city", "region", "name", "status"]) +// if(list && list.size()>1){ +// println it.toString() + " "+list.size() +// for(def d : list){ +//// if(d.status.toString()=="0"){ +//// println d.toString()+" 保留" +//// }else if(d.status.toString()=="1"){ +//// println d.toString()+" 删除" +//// DbUtils.seaweed.remove(new BasicDBObject("_id",d._id)) +//// } +// +// if(d.status.toString()=="1"){ +// println d.toString()+" 保留" +// }else if(d.status.toString()!="1"){ +// println d.toString()+" 删除" +// DbUtils.seaweed.remove(new BasicDBObject("_id",d._id)) +// } +// } +// +// println "................................................................................................" +// } + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy new file mode 100644 index 000000000..9ba1fe9ca --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy @@ -0,0 +1,146 @@ +package com.fangjia.script.gj.images + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 更新小区图片 + */ +@BatchOwner(Watcher.GaoJuan) +class AddDistImagesBatch extends BaseBatch{ + final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + public static final DBCollection seaweed_fs_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_images", "mongo.db.fangjia.com") + + static main(args) { + runBatch(new AddDistImagesBatch(), args); + } + + @Override + public void process(Object args) { +// def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") + def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1) + def citys = CommonUtils.findAllCity() + citys.each {city-> + def communitys = findAllEsData2(city, mdate) + println "抓取城市: "+city +" 第三方小区有图片数:"+communitys.size() + //第三方匹配小区 + matchCommunitys(communitys) + } + } + + + + def findAllEsData2(city, mdate){ + QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true) + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "安居客小程序")) + .must(date_range).must(date_range) + + def rets = [] + def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder); + if(datas) { + for (int k = 0; k < datas.size(); k++) { + def d = datas[k]; + if(d.house_type_images_2 || d.outdoor_images) { + def data = new BasicDBObject() + data.put("city", d.city) + data.put("region", d.region) + data.put("name", d.name) + data.put("id", d.id) + data.put("out_images", d.outdoor_images) + data.put("house_type_images", d.house_type_images_2) + rets << data + } + } + } + return rets; + } + + def findAllEsData(city, mdate){ + def names = new HashSet() + def rets = [] + ElasticUtils.executeSqlQuery("select * from community where source='安居客小程序' and city='${city}' and m_date>='${mdate}' order by m_date", { d-> + if(d.house_type_images_2 || d.outdoor_images){ + def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type + if(!names.contains(s)){ + names << s + def data = new BasicDBObject() + data.put("city", d.city) + data.put("region", d.region) + data.put("name", d.name) + data.put("out_images", d.outdoor_images) + data.put("house_type_images", d.house_type_images_2) + rets << data + } + } + }) + return rets; + } + + + def matchCommunitys(communitys){ + def addCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(communitys.size()) + def rets = new CopyOnWriteArrayList() + def total = communitys.size() + for(def i=0;i map.avg_price = r.gujia_bds_clc_avprc @@ -202,6 +212,25 @@ class AddPriceTest { } + def findNewFjwAvgPrice(def commid, def cityid){ + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1" + def map = [:] + pgsqlInstance235.eachRow(sql, { r -> + map.avg_price = r.gujia_bds_clc_avprc + map.type = r.gujia_bds_clmtd + map.month = r.gujiabds_clc_yrmo_val + map.credit_value = r.gujia_bds_cfdc_dgr + if(r.gujia_bds_clmtd == 1){ + map.calc_type = "人工估值" + }else if(r.gujia_bds_clmtd == 2){ + map.calc_type = "样本计算" + }else if(r.gujia_bds_clmtd == 3){ + map.calc_type = "神经网络" + } + }) + return map + } + def addThirdPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> if (!d['X']) { @@ -241,12 +270,16 @@ class AddPriceTest { def addGXDPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (!d['M']) { - def dist = CommonDatautils.findDistrictById(d['B']) + if (!d['B']) { + def dist = CommonDatautils.findDistrictById(d['A']) if(dist){ def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) // def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) - d['M'] = gxdPrice + if(gxdPrice){ + d['B'] = gxdPrice.price + d['C'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd") + } + } } }) @@ -260,11 +293,11 @@ class AddPriceTest { q.put("district_name", name) q.put("source", "GXD") def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1) - def price = null + def obj = null for(def d : dbcursor){ - price = d.price + obj = d } - return price + return obj } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy index 2d2f76f24..a401bf480 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy @@ -26,6 +26,7 @@ import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.search.Scroll import org.elasticsearch.search.SearchHit import org.elasticsearch.search.builder.SearchSourceBuilder +import org.elasticsearch.search.fetch.subphase.FetchSourceContext import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection @@ -35,7 +36,6 @@ import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import java.util.function.BiConsumer - /** * 功能说明:Elasticsearch工具类 */ @@ -228,14 +228,21 @@ class ElasticUtils89 { } - static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){ + static def executeSimpleQuery(String host, String indexName, QueryBuilder qb, includes){ final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L)) RestHighLevelClient client = getClient(host) SearchRequest searchRequest = new SearchRequest(indexName); searchRequest.scroll(scroll); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + if(includes){ + String[] excludes = [] // 没有要排除的字段 + FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includes, excludes) + searchSourceBuilder.fetchSource(fetchSourceContext) + } + searchSourceBuilder.query(qb); +// searchSourceBuilder.size(1000) searchRequest.source(searchSourceBuilder); SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); String scrollId = searchResponse.getScrollId(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy index e0324502c..4efd79e7f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy @@ -12,15 +12,13 @@ import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import com.rabbitmq.client.AMQP - /** * 厦门涉台园区数据同步 */ @BatchOwner(Watcher.GaoJuan) class RelatedTaiWanBatch extends BaseBatch{ - def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMM")); + def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")); DBCollection yuanqu_qiye_coll = MongoUtils.getCollectionByName("crawl", "qzd_shetai_company_detail", "192.168.0.206", 27020) def relateTaiwn_yuanqu_datas = ["福州台商投资区","杏林台商投资区","集美台商投资区","海沧台商投资区","漳州台商投资区","泉州台商投资区","漳浦台湾农民创业园","漳平台湾农民创业园","仙游台湾农民创业园","清流台湾农民创业园","福清台湾农民创业园","惠安台湾农民创业园","海峡两岸集成电路产业合作试验区(厦门、泉州产业园)","海峡两岸生技和医疗健康产业合作区","海峡两岸农业合作试验区","海峡两岸(三明)现代林业合作实验区","漳州古雷石化基地","永泰闽台农业融合发展产业园","上杭闽台农业融合发展产业园","同安闽台农业融合发展产业园","南靖闽台农业融合发展产业园","漳浦闽台农业融合发展产业园","永春闽台农业融合发展产业园","永安闽台农业融合发展产业园","邵武闽台农业融合发展产业园","平潭闽台农业融合发展产业园","南靖闽台精密机械产业园"] @@ -36,9 +34,17 @@ class RelatedTaiWanBatch extends BaseBatch{ @Override public void process(Object args) { -// createDatas() //企业库获取涉台园区企业 - updateRangeByPoint() //更改企业所在自贸区 - export() //导出数据 + + def day = new Date().getDate() + if(day == 10){ + //createDatas() //企业库获取涉台园区企业 + updateRangeByPoint() //更改企业所在自贸区 + export() //导出数据 + }else{ + exportEmptyFile() + } + + } @@ -158,4 +164,21 @@ class RelatedTaiWanBatch extends BaseBatch{ FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|"); XmBankFtpUtils.ftpUpload(fileName2, curDate) } + + def exportEmptyFile(){ + def fileName = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat" + } + FileUtils.batchCreateFileByCustomSuffix([], fileName, null, "utf-8", false, "|@|","|+|"); + XmBankFtpUtils.ftpUpload(fileName, curDate) + + def fileName2 = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat.rdy" + if(CommonDatautils.isWindow()){ + fileName2 = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat.rdy" + } + def rdy_datas = ["0"] + FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|"); + XmBankFtpUtils.ftpUpload(fileName2, curDate) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy index 5b5e79b62..e78bc7018 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -150,8 +150,8 @@ class SeaweedDataSynch extends BaseBatch{ final CountDownLatch latch = new CountDownLatch(dists.size()) for(def i=0;i=203101){ + dingjiYear = 2030 + }else if(date.toInteger()>=202601){ + dingjiYear = 2025 + }else if(date.toInteger()>=202101){ + dingjiYear = 2020 + }else if(date.toInteger()>=201601){ + dingjiYear = 2015 + } fixedThreadPool.execute(new Runnable() { void run() { try{ @@ -220,7 +260,8 @@ class ImportIndexStats extends BaseBatch{ } }); } - latch.await(20, TimeUnit.SECONDS); + latch.await(1, TimeUnit.MINUTES); +// latch.await(1, TimeUnit.MINUTES); } def calculating(city, year, month, date){ @@ -647,6 +688,7 @@ class ImportIndexStats extends BaseBatch{ DBObject q = new BasicDBObject(); q.put("yearmonth", date); q.put("cat", "district") + q.put("type", "城市") DBObject keys = new BasicDBObject(); keys.put("city", 1) @@ -666,6 +708,7 @@ class ImportIndexStats extends BaseBatch{ q.put("yearmonth", date); q.put("city", city); q.put("cat", "district"); + q.put("type", "城市") DBObject keys = new BasicDBObject(); keys.put("city", 1) @@ -684,6 +727,7 @@ class ImportIndexStats extends BaseBatch{ dbo.put("source",result.source); dbo.put("remark",result.remark); dbo.put("cat",result.cat); + dbo.put("type",result.type); dbo.put("c_date",new Date()); if(result.markup){ dbo.put("chain", result.markup.toDouble()); @@ -785,69 +829,76 @@ class ImportIndexStats extends BaseBatch{ } def createTjjData(date){ - def queryStartDate = DateUtils.parseFromTightByPattern(date+"01", "yyyyMMdd") - def queryEndDate = DateUtils.parseFromTightByPattern(date+"05", "yyyyMMdd") - def sellIndexMap = findTjjIndex(indices_price_house_sellColl, queryStartDate, queryEndDate) - def sellIndexAreaMap = findTjjAreaIndex(indices_price_house_sell_typeColl, queryStartDate, queryEndDate) - def newIndexMap = findTjjIndex(indices_price_house_newColl, queryStartDate, queryEndDate) - def newAreaIndexAreaMap = findTjjAreaIndex(indices_price_house_new_typeColl, queryStartDate, queryEndDate) - def rets = [] - sellIndexMap.each{city, d-> - def dbo = new BasicDBObject() - dbo.put("city",d.city) - dbo.put("yearmonth",date.toInteger()) - dbo.put("source" ,1) - dbo.put("remark" , "统计局数据导入") - dbo.put("cat" , "district") - dbo.put("c_date" , new Date()) - dbo.put("m_date" , new Date()) - dbo.put("type" , "城市") - dbo.put("chain" ,d.mom) - dbo.put("yeartoyear" , d.yoy) - dbo.put("fixedbase" , d.fixedbase) + def count = STATS_INDEX_COLL.count(new BasicDBObject("type","城市").append("source", 1).append("yearmonth", date.toInteger())) + if(count == 0){ + def queryStartDate = DateUtils.parseFromTightByPattern(date+"01", "yyyyMMdd") + def queryEndDate = DateUtils.parseFromTightByPattern(date+"05", "yyyyMMdd") + def sellIndexMap = findTjjIndex(indices_price_house_sellColl, queryStartDate, queryEndDate) + def sellIndexAreaMap = findTjjAreaIndex(indices_price_house_sell_typeColl, queryStartDate, queryEndDate) + + def newIndexMap = findTjjIndex(indices_price_house_newColl, queryStartDate, queryEndDate) + def newAreaIndexAreaMap = findTjjAreaIndex(indices_price_house_new_typeColl, queryStartDate, queryEndDate) + def rets = [] + sellIndexMap.each{city, d-> + def dbo = new BasicDBObject() + dbo.put("city",d.city) + dbo.put("yearmonth",date.toInteger()) + dbo.put("source" ,1) + dbo.put("remark" , "统计局数据导入") + dbo.put("cat" , "district") + dbo.put("c_date" , new Date()) + dbo.put("m_date" , new Date()) + dbo.put("type" , "城市") + dbo.put("chain" ,d.mom) + dbo.put("yeartoyear" , d.yoy) + dbo.put("fixedbase" , d.fixedbase) - def areaMap = sellIndexAreaMap.get(d.city) - def areaObj1 = areaMap.get("90") - dbo.put("below_chain" , areaObj1.mom) - dbo.put("below_yeartoyear" , areaObj1.yoy) - dbo.put("below_fixedbase" , areaObj1.fixedbase) + def areaMap = sellIndexAreaMap.get(d.city) + def areaObj1 = areaMap.get("90") + dbo.put("below_chain" , areaObj1.mom) + dbo.put("below_yeartoyear" , areaObj1.yoy) + dbo.put("below_fixedbase" , areaObj1.fixedbase) - def areaObj2 = areaMap.get("90-144") - dbo.put("among_chain" , areaObj2.mom) - dbo.put("among_yeartoyear" , areaObj2.yoy) - dbo.put("among_fixedbase" , areaObj2.fixedbase) + def areaObj2 = areaMap.get("90-144") + dbo.put("among_chain" , areaObj2.mom) + dbo.put("among_yeartoyear" , areaObj2.yoy) + dbo.put("among_fixedbase" , areaObj2.fixedbase) - def areaObj3 = areaMap.get("144") - dbo.put("above_chain" , areaObj3.mom) - dbo.put("above_yeartoyear" , areaObj3.yoy) - dbo.put("above_fixedbase" , areaObj3.fixedbase) + def areaObj3 = areaMap.get("144") + dbo.put("above_chain" , areaObj3.mom) + dbo.put("above_yeartoyear" , areaObj3.yoy) + dbo.put("above_fixedbase" , areaObj3.fixedbase) - def newObj = newIndexMap.get(d.city) - dbo.put("new_chain" , newObj.mom) - dbo.put("new_yeartoyear" , newObj.yoy) - dbo.put("new_fixedbase" , newObj.fixedbase) + def newObj = newIndexMap.get(d.city) + dbo.put("new_chain" , newObj.mom) + dbo.put("new_yeartoyear" , newObj.yoy) + dbo.put("new_fixedbase" , newObj.fixedbase) - def newAreaMap = newAreaIndexAreaMap.get(d.city) - def areaNewObj1 = newAreaMap.get("90") - dbo.put("new_below_chain" , areaNewObj1.mom) - dbo.put("new_below_yeartoyear" , areaNewObj1.yoy) - dbo.put("new_below_fixedbase" , areaNewObj1.fixedbase) + def newAreaMap = newAreaIndexAreaMap.get(d.city) + def areaNewObj1 = newAreaMap.get("90") + dbo.put("new_below_chain" , areaNewObj1.mom) + dbo.put("new_below_yeartoyear" , areaNewObj1.yoy) + dbo.put("new_below_fixedbase" , areaNewObj1.fixedbase) - def areaNewObj2 = newAreaMap.get("90-144") - dbo.put("new_among_chain" , areaNewObj2.mom) - dbo.put("new_among_yeartoyear" , areaNewObj2.yoy) - dbo.put("new_among_fixedbase" , areaNewObj2.fixedbase) + def areaNewObj2 = newAreaMap.get("90-144") + dbo.put("new_among_chain" , areaNewObj2.mom) + dbo.put("new_among_yeartoyear" , areaNewObj2.yoy) + dbo.put("new_among_fixedbase" , areaNewObj2.fixedbase) - def areaNewObj3 = newAreaMap.get("144") - dbo.put("new_above_chain" , areaNewObj3.mom) - dbo.put("new_above_yeartoyear" , areaNewObj3.yoy) - dbo.put("new_above_fixedbase" , areaNewObj3.fixedbase) + def areaNewObj3 = newAreaMap.get("144") + dbo.put("new_above_chain" , areaNewObj3.mom) + dbo.put("new_above_yeartoyear" , areaNewObj3.yoy) + dbo.put("new_above_fixedbase" , areaNewObj3.fixedbase) + + rets << dbo + } + if(rets){ + STATS_INDEX_COLL.insert(rets) + } - rets << dbo } - STATS_INDEX_COLL.insert(rets) } def findTjjIndex(coll, queryStartDate, queryEndDate){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 50882747b..da882ff58 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -414,7 +414,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = Runtime.getRuntime().availableProcessors() * 4 + def nThread = Runtime.getRuntime().availableProcessors() * 6 println nThread ExecutorService executorService = Executors.newFixedThreadPool(nThread) final AtomicInteger counter = new AtomicInteger(0)