Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj

This commit is contained in:
2024-11-05 09:15:48 +08:00
5 changed files with 232 additions and 153 deletions
@@ -4962,6 +4962,31 @@ class CommonDatautils {
return townMap
}
static def queryTownByCity(city, fields){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",city)
def keys = new BasicDBObject()
for(def key : fields){
keys.put(key,1)
}
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:]
for(def d : dbcursor){
def list = townMap.get(d.district)
if(!list){
list = []
}
list << d
townMap.put(d.district, list)
}
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
return townMap
}
static def queryTownByCity(city){
def q = new BasicDBObject()
@@ -34,13 +34,20 @@ class RegionStandardBatch extends BaseBatch{
}
def seaweedRegionStandard(){
seaweedRegionStandardByCat("shop")
seaweedRegionStandardByCat("district")
}
def poiRegionStandard(){
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
def citys = ["重庆"]//CommonDatautils.findAllCity()
def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]
def index = 0
def citys = CommonDatautils.findAllCity()
for(def city : citys){
index ++
if(index<32){
continue
}
if(runCitys.contains(city)){
continue
}
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
@@ -51,13 +58,15 @@ class RegionStandardBatch extends BaseBatch{
def rMap = regionFullNameMap.get(fullCityName)
if(rMap){
def set = rMap.keySet()
def rets = findEsPoiByCity(fullCityName)
println city +" "+rets.size()
if(set.size()>1){
def rets = findEsPoiByCity(fullCityName)
println city +" "+rets.size()
poiHaveRegionProcess(city, set, rMap, rets)
}else{
// haveStreetProcess(city, set, cat)
}
}else{
println "没找区域:"+city
}
}else{
println "没找到相应的全称:"+city
@@ -163,10 +172,11 @@ class RegionStandardBatch extends BaseBatch{
}
def findEsPoiByCity(city){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("district", "蕉岭县"))
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
def rets = []
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder)
String[] includes = ["city", "district", "name", "township", "status", "id"]
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder,includes)
for(def d: datas){
def o = new BasicDBObject()
o.put("city", d.city)
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
@@ -8,6 +9,8 @@ import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
/**
* 区域标准化
@@ -20,6 +23,13 @@ class TownStandardBatch extends BaseBatch{
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
def poiEsUrl = "http://192.168.0.89:9200/didata_map_poi/_search"
def elasticsearchUrl = "http://192.168.0.89:9200/_search/scroll"
def initialSearchUrl = "http://192.168.0.89:9200/didata_map_poi/_search?scroll=1m"
def scrollTime = "1m" // Scroll context time-to-live
def pageSize = 1000 // Number of hits to return per scroll
static main(args) {
runBatch(new TownStandardBatch(), args)
}
@@ -31,8 +41,8 @@ class TownStandardBatch extends BaseBatch{
}
def TownStandard(){
townStandardByCat("office")
// townStandardByPoi()
// townStandardByCat("office")
townStandardByPoi()
}
def townStandardByCat(def cat){
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
@@ -60,7 +70,7 @@ class TownStandardBatch extends BaseBatch{
}
def townStandardByPoi(){
def citys = ["重庆"]//CommonDatautils.findAllCity()
def citys = ["三明"]//CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
@@ -69,14 +79,14 @@ class TownStandardBatch extends BaseBatch{
fullCityName = data.city
}
if(fullCityName){
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"])
def rMap = regionFullNameMap.get(fullCityName)
if(rMap){
def set = rMap.keySet()
def rets = findEsPoiByCity(fullCityName)
println city +" "+rets.size()
if(set.size()>1){
poiHaveRegionProcess(city, set, rMap, rets)
poiHaveRegionProcess(fullCityName, townMap, rets)
}else{
// haveStreetProcess(city, set, cat)
}
@@ -141,7 +151,76 @@ class TownStandardBatch extends BaseBatch{
}
}
def updateRegionByEs(rows){
def poiHaveRegionProcess(fullCityName, townMap, rets){
def results = []
if(fullCityName){
for(def d : rets){
if(d.township){
def towns = null
def updateTown = null
for(def m : townMap){
def townDatas = townMap.get(d.district)
if(townDatas){
towns = townDatas
break
}
}
def opt = true
if(towns){
for(def town : towns){
def townAlias = []
if(town.alias){
townAlias = town.alias
}
townAlias << town.street
if(townAlias.contains(d.township)){
if(d.township!=town.street){
updateTown = town.street
}else{
opt = false
}
break
}
}
}
if(opt){
if(updateTown){
def m = [:]
m.put(d.id, ["township":updateTown])
results << m
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
// }
println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
}else{
def m = [:]
m.put(d.id, ["township":""])
results << m
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
// }
println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值"
}
}
}
}
}
if(results.size()>0){
updateByEs(results)
results.clear()
}
println fullCityName +" 处理完成"
}
def updateByEs(rows){
def client = ElasticUtils89.getClient("192.168.0.89")
def bulkRequest = new BulkRequest()
@@ -177,75 +256,24 @@ class TownStandardBatch extends BaseBatch{
}
}
def findEsPoiByCity(city){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("status", 0))
def rets = []
String[] includes = ["city", "district", "name", "township", "status", "id"]
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder, includes)
for(def d: datas){
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", d.district)
o.put("name", d.name)
o.put("township", d.township)
o.put("status", d.status)
o.put("id", d.id)
rets << o
}
return rets
}
// def queryPoiDatasByCity(){
// def esUrl = "http://localhost:9200/your_index/_search"
// def pageSize = 1000 // 每页返回的记录数
// def scrollSize = '1m' // scroll上下文的生存时间
//
//// 初始化HttpClient和ObjectMapper
// CloseableHttpClient httpClient = HttpClients.createDefault()
// ObjectMapper objectMapper = new ObjectMapper()
//
//// 初始化scroll请求
// def scrollRequest = """
//{
// "size": $pageSize,
// "query": {
// "match_all": {}
// },
// "scroll": "$scrollSize"
//}
//"""
//
//// 发送scroll初始化请求
// HttpGet scrollInitRequest = new HttpGet(esUrl)
// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8"))
// scrollInitRequest.setHeader("Content-Type", "application/json")
//
// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest)
// HttpEntity entity = scrollInitResponse.getEntity()
// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString()
//
//// 存储所有结果
// def allResults = []
//
//// 发送scroll请求获取数据,直到没有更多数据为止
// while (true) {
// def scrollRequestWithId = """
// {
// "scroll_id": "$scrollId",
// "scroll": "$scrollSize"
// }
// """
//
// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll")
// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8"))
// scrollGetRequest.setHeader("Content-Type", "application/json")
//
// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest)
// entity = scrollGetResponse.getEntity()
// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class)
//
// def hits = responseMap.get("hits").get("hits")
// if (hits.isEmpty()) {
// break // 没有更多数据,跳出循环
// }
//
// // 合并当前页的数据
// allResults.addAll(hits.map({ it.get("_source") }))
//
// // 更新scrollId以进行下一次请求
// scrollId = responseMap.get("_scroll_id").toString()
// }
//
// // 关闭HttpClient
// httpClient.close()
//
// // 输出或处理全量数据
// println(allResults)
//
// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。
// }
}
@@ -26,6 +26,7 @@ import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.elasticsearch.search.fetch.subphase.FetchSourceContext
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
@@ -35,7 +36,6 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.function.BiConsumer
/**
* 功能说明:Elasticsearch工具类
*/
@@ -228,13 +228,19 @@ class ElasticUtils89 {
}
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb, includes){
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
RestHighLevelClient client = getClient(host)
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
if(includes){
String[] excludes = [] // 没有要排除的字段
FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includes, excludes)
searchSourceBuilder.fetchSource(fetchSourceContext)
}
searchSourceBuilder.query(qb);
// searchSourceBuilder.size(1000)
searchRequest.source(searchSourceBuilder);
@@ -1,16 +1,14 @@
package com.fangjia.script.trade
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
///导入价格指数
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
@@ -19,10 +17,8 @@ import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.math.RandomUtils
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
@@ -74,13 +70,19 @@ class ImportIndexStats extends BaseBatch{
// updateDataType()
// synch()
// synchTjjIndex()//同步统计局抓取过来的指数
// synchTjjIndex()//批量同步统计局抓取过来的指数
Calendar calendar=Calendar.getInstance();
def month= calendar.get(Calendar.MONTH);
def year= calendar.get(Calendar.YEAR);
def date = createDateString(year, month, 0, "");
calculatePriceIndex(year, month, date);
def day = calendar.get(Calendar.DATE)
if(day == 5){
calculatePriceIndex(year, month, date);
}
createTjjData(date)//同步单期统计局数据
// addLostIndex()
// addHistoryIndex()
@@ -199,7 +201,7 @@ class ImportIndexStats extends BaseBatch{
runDates << map
s = DateUtils.addMonth(date, 1, "yyyyMM")
if(year==2024 && month==8){
if(year==2024 && month==10){
stop = true;
}
}
@@ -210,7 +212,7 @@ class ImportIndexStats extends BaseBatch{
//计算统计局没有发布的城市的价格指数
def calculatePriceIndex(def year, def month, def date){
// def citys = ["五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]//
// def citys = ["定安","五指山","琼海","儋州","文昌","万宁","东方","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]//
def citys = CommonDatautils.findAllCity();
// def date = createDateString(year, month, 0, "");
def haveIndexCitys = findPriceIndexCitys(date.toInteger());
@@ -233,14 +235,14 @@ class ImportIndexStats extends BaseBatch{
if(id){//已计算
continue;
}
if(date.toInteger()>=201601){
dingjiYear = 2015
}else if(date.toInteger()>=202101){
dingjiYear = 2020
if(date.toInteger()>=203101){
dingjiYear = 2030
}else if(date.toInteger()>=202601){
dingjiYear = 2025
}else if(date.toInteger()>=203101){
dingjiYear = 2030
}else if(date.toInteger()>=202101){
dingjiYear = 2020
}else if(date.toInteger()>=201601){
dingjiYear = 2015
}
fixedThreadPool.execute(new Runnable() {
void run() {
@@ -259,6 +261,7 @@ class ImportIndexStats extends BaseBatch{
});
}
latch.await(1, TimeUnit.MINUTES);
// latch.await(1, TimeUnit.MINUTES);
}
def calculating(city, year, month, date){
@@ -826,69 +829,76 @@ class ImportIndexStats extends BaseBatch{
}
def createTjjData(date){
def queryStartDate = DateUtils.parseFromTightByPattern(date+"01", "yyyyMMdd")
def queryEndDate = DateUtils.parseFromTightByPattern(date+"05", "yyyyMMdd")
def sellIndexMap = findTjjIndex(indices_price_house_sellColl, queryStartDate, queryEndDate)
def sellIndexAreaMap = findTjjAreaIndex(indices_price_house_sell_typeColl, queryStartDate, queryEndDate)
def newIndexMap = findTjjIndex(indices_price_house_newColl, queryStartDate, queryEndDate)
def newAreaIndexAreaMap = findTjjAreaIndex(indices_price_house_new_typeColl, queryStartDate, queryEndDate)
def rets = []
sellIndexMap.each{city, d->
def dbo = new BasicDBObject()
dbo.put("city",d.city)
dbo.put("yearmonth",date.toInteger())
dbo.put("source" ,1)
dbo.put("remark" , "统计局数据导入")
dbo.put("cat" , "district")
dbo.put("c_date" , new Date())
dbo.put("m_date" , new Date())
dbo.put("type" , "城市")
dbo.put("chain" ,d.mom)
dbo.put("yeartoyear" , d.yoy)
dbo.put("fixedbase" , d.fixedbase)
def count = STATS_INDEX_COLL.count(new BasicDBObject("type","城市").append("source", 1).append("yearmonth", date.toInteger()))
if(count == 0){
def queryStartDate = DateUtils.parseFromTightByPattern(date+"01", "yyyyMMdd")
def queryEndDate = DateUtils.parseFromTightByPattern(date+"05", "yyyyMMdd")
def sellIndexMap = findTjjIndex(indices_price_house_sellColl, queryStartDate, queryEndDate)
def sellIndexAreaMap = findTjjAreaIndex(indices_price_house_sell_typeColl, queryStartDate, queryEndDate)
def newIndexMap = findTjjIndex(indices_price_house_newColl, queryStartDate, queryEndDate)
def newAreaIndexAreaMap = findTjjAreaIndex(indices_price_house_new_typeColl, queryStartDate, queryEndDate)
def rets = []
sellIndexMap.each{city, d->
def dbo = new BasicDBObject()
dbo.put("city",d.city)
dbo.put("yearmonth",date.toInteger())
dbo.put("source" ,1)
dbo.put("remark" , "统计局数据导入")
dbo.put("cat" , "district")
dbo.put("c_date" , new Date())
dbo.put("m_date" , new Date())
dbo.put("type" , "城市")
dbo.put("chain" ,d.mom)
dbo.put("yeartoyear" , d.yoy)
dbo.put("fixedbase" , d.fixedbase)
def areaMap = sellIndexAreaMap.get(d.city)
def areaObj1 = areaMap.get("90")
dbo.put("below_chain" , areaObj1.mom)
dbo.put("below_yeartoyear" , areaObj1.yoy)
dbo.put("below_fixedbase" , areaObj1.fixedbase)
def areaMap = sellIndexAreaMap.get(d.city)
def areaObj1 = areaMap.get("90")
dbo.put("below_chain" , areaObj1.mom)
dbo.put("below_yeartoyear" , areaObj1.yoy)
dbo.put("below_fixedbase" , areaObj1.fixedbase)
def areaObj2 = areaMap.get("90-144")
dbo.put("among_chain" , areaObj2.mom)
dbo.put("among_yeartoyear" , areaObj2.yoy)
dbo.put("among_fixedbase" , areaObj2.fixedbase)
def areaObj2 = areaMap.get("90-144")
dbo.put("among_chain" , areaObj2.mom)
dbo.put("among_yeartoyear" , areaObj2.yoy)
dbo.put("among_fixedbase" , areaObj2.fixedbase)
def areaObj3 = areaMap.get("144")
dbo.put("above_chain" , areaObj3.mom)
dbo.put("above_yeartoyear" , areaObj3.yoy)
dbo.put("above_fixedbase" , areaObj3.fixedbase)
def areaObj3 = areaMap.get("144")
dbo.put("above_chain" , areaObj3.mom)
dbo.put("above_yeartoyear" , areaObj3.yoy)
dbo.put("above_fixedbase" , areaObj3.fixedbase)
def newObj = newIndexMap.get(d.city)
dbo.put("new_chain" , newObj.mom)
dbo.put("new_yeartoyear" , newObj.yoy)
dbo.put("new_fixedbase" , newObj.fixedbase)
def newObj = newIndexMap.get(d.city)
dbo.put("new_chain" , newObj.mom)
dbo.put("new_yeartoyear" , newObj.yoy)
dbo.put("new_fixedbase" , newObj.fixedbase)
def newAreaMap = newAreaIndexAreaMap.get(d.city)
def areaNewObj1 = newAreaMap.get("90")
dbo.put("new_below_chain" , areaNewObj1.mom)
dbo.put("new_below_yeartoyear" , areaNewObj1.yoy)
dbo.put("new_below_fixedbase" , areaNewObj1.fixedbase)
def newAreaMap = newAreaIndexAreaMap.get(d.city)
def areaNewObj1 = newAreaMap.get("90")
dbo.put("new_below_chain" , areaNewObj1.mom)
dbo.put("new_below_yeartoyear" , areaNewObj1.yoy)
dbo.put("new_below_fixedbase" , areaNewObj1.fixedbase)
def areaNewObj2 = newAreaMap.get("90-144")
dbo.put("new_among_chain" , areaNewObj2.mom)
dbo.put("new_among_yeartoyear" , areaNewObj2.yoy)
dbo.put("new_among_fixedbase" , areaNewObj2.fixedbase)
def areaNewObj2 = newAreaMap.get("90-144")
dbo.put("new_among_chain" , areaNewObj2.mom)
dbo.put("new_among_yeartoyear" , areaNewObj2.yoy)
dbo.put("new_among_fixedbase" , areaNewObj2.fixedbase)
def areaNewObj3 = newAreaMap.get("144")
dbo.put("new_above_chain" , areaNewObj3.mom)
dbo.put("new_above_yeartoyear" , areaNewObj3.yoy)
dbo.put("new_above_fixedbase" , areaNewObj3.fixedbase)
def areaNewObj3 = newAreaMap.get("144")
dbo.put("new_above_chain" , areaNewObj3.mom)
dbo.put("new_above_yeartoyear" , areaNewObj3.yoy)
dbo.put("new_above_fixedbase" , areaNewObj3.fixedbase)
rets << dbo
}
if(rets){
STATS_INDEX_COLL.insert(rets)
}
rets << dbo
}
STATS_INDEX_COLL.insert(rets)
}
def findTjjIndex(coll, queryStartDate, queryEndDate){