diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 667580f1d..15c76f043 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4962,6 +4962,31 @@ class CommonDatautils { return townMap } + static def queryTownByCity(city, fields){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + q.put("city",city) + + def keys = new BasicDBObject() + for(def key : fields){ + keys.put(key,1) + } + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def townMap = [:] + for(def d : dbcursor){ + def list = townMap.get(d.district) + if(!list){ + list = [] + } + list << d + townMap.put(d.district, list) + } + + println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss") + return townMap + } + static def queryTownByCity(city){ def q = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index c4ffa723f..7536d908f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -34,13 +34,20 @@ class RegionStandardBatch extends BaseBatch{ } def seaweedRegionStandard(){ - seaweedRegionStandardByCat("shop") + seaweedRegionStandardByCat("district") } def poiRegionStandard(){ - //"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德", - def citys = ["重庆"]//CommonDatautils.findAllCity() + def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"] + def index = 0 + def citys = CommonDatautils.findAllCity() for(def city : citys){ - + index ++ + if(index<32){ + continue + } + if(runCitys.contains(city)){ + continue + } def data = fullCityMap.get(city) def fullCityName = null if(data){ @@ -51,13 +58,15 @@ class RegionStandardBatch extends BaseBatch{ def rMap = regionFullNameMap.get(fullCityName) if(rMap){ def set = rMap.keySet() - def rets = findEsPoiByCity(fullCityName) - println city +" "+rets.size() if(set.size()>1){ + def rets = findEsPoiByCity(fullCityName) + println city +" "+rets.size() poiHaveRegionProcess(city, set, rMap, rets) }else{ // haveStreetProcess(city, set, cat) } + }else{ + println "没找区域:"+city } }else{ println "没找到相应的全称:"+city @@ -163,10 +172,11 @@ class RegionStandardBatch extends BaseBatch{ } def findEsPoiByCity(city){ - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("district", "蕉岭县")) + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) def rets = [] - def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder) + String[] includes = ["city", "district", "name", "township", "status", "id"] + def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder,includes) for(def d: datas){ def o = new BasicDBObject() o.put("city", d.city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index afe90bbb2..1cbbde65c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.data_handle + import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.xiamen.ElasticUtils89 @@ -8,6 +9,8 @@ import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.update.UpdateRequest import org.elasticsearch.client.RequestOptions import org.elasticsearch.common.xcontent.XContentType +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders /** * 区域标准化 @@ -20,6 +23,13 @@ class TownStandardBatch extends BaseBatch{ def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() + def poiEsUrl = "http://192.168.0.89:9200/didata_map_poi/_search" + + def elasticsearchUrl = "http://192.168.0.89:9200/_search/scroll" + def initialSearchUrl = "http://192.168.0.89:9200/didata_map_poi/_search?scroll=1m" + def scrollTime = "1m" // Scroll context time-to-live + def pageSize = 1000 // Number of hits to return per scroll + static main(args) { runBatch(new TownStandardBatch(), args) } @@ -31,8 +41,8 @@ class TownStandardBatch extends BaseBatch{ } def TownStandard(){ - townStandardByCat("office") -// townStandardByPoi() +// townStandardByCat("office") + townStandardByPoi() } def townStandardByCat(def cat){ //"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德", @@ -60,7 +70,7 @@ class TownStandardBatch extends BaseBatch{ } def townStandardByPoi(){ - def citys = ["重庆"]//CommonDatautils.findAllCity() + def citys = ["三明"]//CommonDatautils.findAllCity() for(def city : citys){ def data = fullCityMap.get(city) @@ -69,14 +79,14 @@ class TownStandardBatch extends BaseBatch{ fullCityName = data.city } if(fullCityName){ - + def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"]) def rMap = regionFullNameMap.get(fullCityName) if(rMap){ def set = rMap.keySet() def rets = findEsPoiByCity(fullCityName) println city +" "+rets.size() if(set.size()>1){ - poiHaveRegionProcess(city, set, rMap, rets) + poiHaveRegionProcess(fullCityName, townMap, rets) }else{ // haveStreetProcess(city, set, cat) } @@ -141,7 +151,76 @@ class TownStandardBatch extends BaseBatch{ } } - def updateRegionByEs(rows){ + def poiHaveRegionProcess(fullCityName, townMap, rets){ + def results = [] + if(fullCityName){ + for(def d : rets){ + if(d.township){ + def towns = null + def updateTown = null + for(def m : townMap){ + def townDatas = townMap.get(d.district) + if(townDatas){ + towns = townDatas + break + } + } + + def opt = true + if(towns){ + for(def town : towns){ + def townAlias = [] + if(town.alias){ + townAlias = town.alias + } + townAlias << town.street + if(townAlias.contains(d.township)){ + if(d.township!=town.street){ + updateTown = town.street + }else{ + opt = false + } + break + } + } + } + if(opt){ + if(updateTown){ + def m = [:] + m.put(d.id, ["township":updateTown]) + results << m +// if(results.size()%100==0){ +// updateByEs(results) +// results.clear() +// } + println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown + }else{ + def m = [:] + m.put(d.id, ["township":""]) + results << m +// if(results.size()%100==0){ +// updateByEs(results) +// results.clear() +// } + println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值" + } + } + + } + } + } + + if(results.size()>0){ + updateByEs(results) + results.clear() + } + println fullCityName +" 处理完成" + } + + + + + def updateByEs(rows){ def client = ElasticUtils89.getClient("192.168.0.89") def bulkRequest = new BulkRequest() @@ -177,75 +256,24 @@ class TownStandardBatch extends BaseBatch{ } } + def findEsPoiByCity(city){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("status", 0)) + def rets = [] + + String[] includes = ["city", "district", "name", "township", "status", "id"] + def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder, includes) + for(def d: datas){ + def o = new BasicDBObject() + o.put("city", d.city) + o.put("district", d.district) + o.put("name", d.name) + o.put("township", d.township) + o.put("status", d.status) + o.put("id", d.id) + rets << o + } + return rets + } - // def queryPoiDatasByCity(){ -// def esUrl = "http://localhost:9200/your_index/_search" -// def pageSize = 1000 // 每页返回的记录数 -// def scrollSize = '1m' // scroll上下文的生存时间 -// -//// 初始化HttpClient和ObjectMapper -// CloseableHttpClient httpClient = HttpClients.createDefault() -// ObjectMapper objectMapper = new ObjectMapper() -// -//// 初始化scroll请求 -// def scrollRequest = """ -//{ -// "size": $pageSize, -// "query": { -// "match_all": {} -// }, -// "scroll": "$scrollSize" -//} -//""" -// -//// 发送scroll初始化请求 -// HttpGet scrollInitRequest = new HttpGet(esUrl) -// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8")) -// scrollInitRequest.setHeader("Content-Type", "application/json") -// -// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest) -// HttpEntity entity = scrollInitResponse.getEntity() -// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString() -// -//// 存储所有结果 -// def allResults = [] -// -//// 发送scroll请求获取数据,直到没有更多数据为止 -// while (true) { -// def scrollRequestWithId = """ -// { -// "scroll_id": "$scrollId", -// "scroll": "$scrollSize" -// } -// """ -// -// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll") -// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8")) -// scrollGetRequest.setHeader("Content-Type", "application/json") -// -// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest) -// entity = scrollGetResponse.getEntity() -// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class) -// -// def hits = responseMap.get("hits").get("hits") -// if (hits.isEmpty()) { -// break // 没有更多数据,跳出循环 -// } -// -// // 合并当前页的数据 -// allResults.addAll(hits.map({ it.get("_source") })) -// -// // 更新scrollId以进行下一次请求 -// scrollId = responseMap.get("_scroll_id").toString() -// } -// -// // 关闭HttpClient -// httpClient.close() -// -// // 输出或处理全量数据 -// println(allResults) -// -// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。 -// } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy index e31ded3ec..a401bf480 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy @@ -26,6 +26,7 @@ import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.search.Scroll import org.elasticsearch.search.SearchHit import org.elasticsearch.search.builder.SearchSourceBuilder +import org.elasticsearch.search.fetch.subphase.FetchSourceContext import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection @@ -35,7 +36,6 @@ import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import java.util.function.BiConsumer - /** * 功能说明:Elasticsearch工具类 */ @@ -228,13 +228,19 @@ class ElasticUtils89 { } - static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){ + static def executeSimpleQuery(String host, String indexName, QueryBuilder qb, includes){ final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L)) RestHighLevelClient client = getClient(host) SearchRequest searchRequest = new SearchRequest(indexName); searchRequest.scroll(scroll); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + if(includes){ + String[] excludes = [] // 没有要排除的字段 + FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includes, excludes) + searchSourceBuilder.fetchSource(fetchSourceContext) + } + searchSourceBuilder.query(qb); // searchSourceBuilder.size(1000) searchRequest.source(searchSourceBuilder); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy index b5de80cae..7e5d2151c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy @@ -1,16 +1,14 @@ package com.fangjia.script.trade import com.fangjia.core.util.DataUtil -import com.fangjia.core.util.DateUtil import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher ///导入价格指数 -import com.fangjia.script.BatchOwner -import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils -import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils @@ -19,10 +17,8 @@ import com.mongodb.DBCollection import com.mongodb.DBObject import org.apache.commons.lang.StringUtils import org.apache.commons.lang.math.RandomUtils -import org.bson.types.ObjectId import org.elasticsearch.xpack.sql.jdbc.EsDataSource -import java.sql.ResultSet import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -74,13 +70,19 @@ class ImportIndexStats extends BaseBatch{ // updateDataType() // synch() -// synchTjjIndex()//同步统计局抓取过来的指数 +// synchTjjIndex()//批量同步统计局抓取过来的指数 Calendar calendar=Calendar.getInstance(); def month= calendar.get(Calendar.MONTH); def year= calendar.get(Calendar.YEAR); def date = createDateString(year, month, 0, ""); - calculatePriceIndex(year, month, date); + + def day = calendar.get(Calendar.DATE) + if(day == 5){ + calculatePriceIndex(year, month, date); + } + + createTjjData(date)//同步单期统计局数据 // addLostIndex() // addHistoryIndex() @@ -199,7 +201,7 @@ class ImportIndexStats extends BaseBatch{ runDates << map s = DateUtils.addMonth(date, 1, "yyyyMM") - if(year==2024 && month==8){ + if(year==2024 && month==10){ stop = true; } } @@ -210,7 +212,7 @@ class ImportIndexStats extends BaseBatch{ //计算统计局没有发布的城市的价格指数 def calculatePriceIndex(def year, def month, def date){ -// def citys = ["五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]// +// def citys = ["定安","五指山","琼海","儋州","文昌","万宁","东方","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]// def citys = CommonDatautils.findAllCity(); // def date = createDateString(year, month, 0, ""); def haveIndexCitys = findPriceIndexCitys(date.toInteger()); @@ -233,14 +235,14 @@ class ImportIndexStats extends BaseBatch{ if(id){//已计算 continue; } - if(date.toInteger()>=201601){ - dingjiYear = 2015 - }else if(date.toInteger()>=202101){ - dingjiYear = 2020 + if(date.toInteger()>=203101){ + dingjiYear = 2030 }else if(date.toInteger()>=202601){ dingjiYear = 2025 - }else if(date.toInteger()>=203101){ - dingjiYear = 2030 + }else if(date.toInteger()>=202101){ + dingjiYear = 2020 + }else if(date.toInteger()>=201601){ + dingjiYear = 2015 } fixedThreadPool.execute(new Runnable() { void run() { @@ -259,6 +261,7 @@ class ImportIndexStats extends BaseBatch{ }); } latch.await(1, TimeUnit.MINUTES); +// latch.await(1, TimeUnit.MINUTES); } def calculating(city, year, month, date){ @@ -826,69 +829,76 @@ class ImportIndexStats extends BaseBatch{ } def createTjjData(date){ - def queryStartDate = DateUtils.parseFromTightByPattern(date+"01", "yyyyMMdd") - def queryEndDate = DateUtils.parseFromTightByPattern(date+"05", "yyyyMMdd") - def sellIndexMap = findTjjIndex(indices_price_house_sellColl, queryStartDate, queryEndDate) - def sellIndexAreaMap = findTjjAreaIndex(indices_price_house_sell_typeColl, queryStartDate, queryEndDate) - def newIndexMap = findTjjIndex(indices_price_house_newColl, queryStartDate, queryEndDate) - def newAreaIndexAreaMap = findTjjAreaIndex(indices_price_house_new_typeColl, queryStartDate, queryEndDate) - def rets = [] - sellIndexMap.each{city, d-> - def dbo = new BasicDBObject() - dbo.put("city",d.city) - dbo.put("yearmonth",date.toInteger()) - dbo.put("source" ,1) - dbo.put("remark" , "统计局数据导入") - dbo.put("cat" , "district") - dbo.put("c_date" , new Date()) - dbo.put("m_date" , new Date()) - dbo.put("type" , "城市") - dbo.put("chain" ,d.mom) - dbo.put("yeartoyear" , d.yoy) - dbo.put("fixedbase" , d.fixedbase) + def count = STATS_INDEX_COLL.count(new BasicDBObject("type","城市").append("source", 1).append("yearmonth", date.toInteger())) + if(count == 0){ + def queryStartDate = DateUtils.parseFromTightByPattern(date+"01", "yyyyMMdd") + def queryEndDate = DateUtils.parseFromTightByPattern(date+"05", "yyyyMMdd") + def sellIndexMap = findTjjIndex(indices_price_house_sellColl, queryStartDate, queryEndDate) + def sellIndexAreaMap = findTjjAreaIndex(indices_price_house_sell_typeColl, queryStartDate, queryEndDate) + + def newIndexMap = findTjjIndex(indices_price_house_newColl, queryStartDate, queryEndDate) + def newAreaIndexAreaMap = findTjjAreaIndex(indices_price_house_new_typeColl, queryStartDate, queryEndDate) + def rets = [] + sellIndexMap.each{city, d-> + def dbo = new BasicDBObject() + dbo.put("city",d.city) + dbo.put("yearmonth",date.toInteger()) + dbo.put("source" ,1) + dbo.put("remark" , "统计局数据导入") + dbo.put("cat" , "district") + dbo.put("c_date" , new Date()) + dbo.put("m_date" , new Date()) + dbo.put("type" , "城市") + dbo.put("chain" ,d.mom) + dbo.put("yeartoyear" , d.yoy) + dbo.put("fixedbase" , d.fixedbase) - def areaMap = sellIndexAreaMap.get(d.city) - def areaObj1 = areaMap.get("90") - dbo.put("below_chain" , areaObj1.mom) - dbo.put("below_yeartoyear" , areaObj1.yoy) - dbo.put("below_fixedbase" , areaObj1.fixedbase) + def areaMap = sellIndexAreaMap.get(d.city) + def areaObj1 = areaMap.get("90") + dbo.put("below_chain" , areaObj1.mom) + dbo.put("below_yeartoyear" , areaObj1.yoy) + dbo.put("below_fixedbase" , areaObj1.fixedbase) - def areaObj2 = areaMap.get("90-144") - dbo.put("among_chain" , areaObj2.mom) - dbo.put("among_yeartoyear" , areaObj2.yoy) - dbo.put("among_fixedbase" , areaObj2.fixedbase) + def areaObj2 = areaMap.get("90-144") + dbo.put("among_chain" , areaObj2.mom) + dbo.put("among_yeartoyear" , areaObj2.yoy) + dbo.put("among_fixedbase" , areaObj2.fixedbase) - def areaObj3 = areaMap.get("144") - dbo.put("above_chain" , areaObj3.mom) - dbo.put("above_yeartoyear" , areaObj3.yoy) - dbo.put("above_fixedbase" , areaObj3.fixedbase) + def areaObj3 = areaMap.get("144") + dbo.put("above_chain" , areaObj3.mom) + dbo.put("above_yeartoyear" , areaObj3.yoy) + dbo.put("above_fixedbase" , areaObj3.fixedbase) - def newObj = newIndexMap.get(d.city) - dbo.put("new_chain" , newObj.mom) - dbo.put("new_yeartoyear" , newObj.yoy) - dbo.put("new_fixedbase" , newObj.fixedbase) + def newObj = newIndexMap.get(d.city) + dbo.put("new_chain" , newObj.mom) + dbo.put("new_yeartoyear" , newObj.yoy) + dbo.put("new_fixedbase" , newObj.fixedbase) - def newAreaMap = newAreaIndexAreaMap.get(d.city) - def areaNewObj1 = newAreaMap.get("90") - dbo.put("new_below_chain" , areaNewObj1.mom) - dbo.put("new_below_yeartoyear" , areaNewObj1.yoy) - dbo.put("new_below_fixedbase" , areaNewObj1.fixedbase) + def newAreaMap = newAreaIndexAreaMap.get(d.city) + def areaNewObj1 = newAreaMap.get("90") + dbo.put("new_below_chain" , areaNewObj1.mom) + dbo.put("new_below_yeartoyear" , areaNewObj1.yoy) + dbo.put("new_below_fixedbase" , areaNewObj1.fixedbase) - def areaNewObj2 = newAreaMap.get("90-144") - dbo.put("new_among_chain" , areaNewObj2.mom) - dbo.put("new_among_yeartoyear" , areaNewObj2.yoy) - dbo.put("new_among_fixedbase" , areaNewObj2.fixedbase) + def areaNewObj2 = newAreaMap.get("90-144") + dbo.put("new_among_chain" , areaNewObj2.mom) + dbo.put("new_among_yeartoyear" , areaNewObj2.yoy) + dbo.put("new_among_fixedbase" , areaNewObj2.fixedbase) - def areaNewObj3 = newAreaMap.get("144") - dbo.put("new_above_chain" , areaNewObj3.mom) - dbo.put("new_above_yeartoyear" , areaNewObj3.yoy) - dbo.put("new_above_fixedbase" , areaNewObj3.fixedbase) + def areaNewObj3 = newAreaMap.get("144") + dbo.put("new_above_chain" , areaNewObj3.mom) + dbo.put("new_above_yeartoyear" , areaNewObj3.yoy) + dbo.put("new_above_fixedbase" , areaNewObj3.fixedbase) + + rets << dbo + } + if(rets){ + STATS_INDEX_COLL.insert(rets) + } - rets << dbo } - STATS_INDEX_COLL.insert(rets) } def findTjjIndex(coll, queryStartDate, queryEndDate){