Merge remote-tracking branch 'origin/master'

This commit is contained in:
2025-07-11 14:13:24 +08:00
8 changed files with 311 additions and 96 deletions
+19
View File
@@ -555,6 +555,25 @@
<version>0.0.12</version> <version>0.0.12</version>
</dependency> </dependency>
<dependency>
<groupId>org.codehaus.groovy.modules.http-builder</groupId>
<artifactId>http-builder</artifactId>
<version>0.7.1</version>
<exclusions>
<!-- 排除冲突的 xmlbeans -->
<exclusion>
<groupId>xmlbeans</groupId>
<artifactId>xmlbeans</artifactId>
</exclusion>
<!-- 如果还有其他冲突的依赖(如 xercesImpl),也一并排除 -->
<exclusion>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- <dependency>--> <!-- <dependency>-->
<!-- <groupId>com.fangjia</groupId>--> <!-- <groupId>com.fangjia</groupId>-->
<!-- <artifactId>fangjia-open</artifactId>--> <!-- <artifactId>fangjia-open</artifactId>-->
@@ -443,7 +443,7 @@ class CcbCommonUtils {
} }
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 // q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
// q.put("name","新泾三村") // q.put("name","龙源星城")
// q.put("name_cat", 0); // q.put("name_cat", 0);
// q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"])) // q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"]))
@@ -105,6 +105,8 @@ class CommonDatautils {
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com") public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com") public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com")
public static final DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static DSCollection getHouseDataStoreConnection(){ public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) { if(house_dataStoreConnection != null) {
@@ -811,6 +813,8 @@ class CommonDatautils {
// q.put("status",new BasicDBObject('$ne', 0)) // q.put("status",new BasicDBObject('$ne', 0))
q.put("cat", cat); q.put("cat", cat);
q.put("city", city) q.put("city", city)
// q.put("loop_line_info",new BasicDBObject('$exists',false))
// q.put("name","经纬天地园二期")
DBObject keys = new BasicDBObject(); DBObject keys = new BasicDBObject();
if(fields){ if(fields){
@@ -4142,9 +4146,13 @@ class CommonDatautils {
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true) QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true)
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
.must(QueryBuilders.termQuery("region", region))
.must(QueryBuilders.termQuery("cat", cat)).must(date_range) .must(QueryBuilders.termQuery("cat", cat)).must(date_range)
if(region){
queryBuilder.must(QueryBuilders.termQuery("region", region))
}
if(name || alias){ if(name || alias){
if(!alias){ if(!alias){
alias = [] alias = []
@@ -4250,6 +4258,19 @@ class CommonDatautils {
return rets; return rets;
} }
static def findAllNewPoiDatas(city, region, name, small_type){
def rets = []
def index= 0
String sql = "select cityname,adname,name,address,location,type,id,parent from amap_poi_2024 where cityname='${city}'"
ElasticUtils.executeSqlQuery(sql, { map->
// if(map.address && map.address.contains("五角场")){
index++
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y, "parent_id":map.parent]
// }
})
return rets;
}
@@ -5644,4 +5665,101 @@ class CommonDatautils {
}) })
return rets; return rets;
} }
static def findGaoDeDatas(city, region, name){
String sql = "select name,type from amap_poi_2024 where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') limit 100"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name == name){ //&& d.shape
obj = d
}
})
return obj;
}
static def findGaoDeDatas2(city, region, name){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_2024/_search"
def body = """
{
"_source": {
"includes": ["cat", "address", "id","name", "township","location"]
},
"query": {
"bool": {
"must": [
{
"term": {
"city" : "${city}"
}
},
{
"term": {
"district" : "${region}"
}
},
{
"match": {
"name": {"query": "${name}","operator": "and"}
}
}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = new JsonSlurper().parseText(res)
// if(name=="河北河间经济开发区"){
// println name
// }
def esObj = null
if(response && response.hits){
def hits = response.hits.hits
if(hits){
def d = hits[0]
// if(d._source.cat.contains("住宅")){
def source = d._source
def newName = source.name
if(newName == name){
esObj = source
}else if (newName.contains("开发区") && name.contains("开发区") && (newName.contains(name) || name.contains(newName))){
esObj = source
}else if(newName.contains("园区") && name.contains("园区") && (newName.contains(name) || name.contains(newName))){
esObj = source
}else if(newName.contains("示范区") && name.contains("示范区") && (newName.contains(name) || name.contains(newName))){
esObj = source
}else{
if(name.contains("开发区")){
name = name.replace("国家高新技术产业开发区","").replace("高新技术产业开发区","").replace("经济技术开发区","").replace("经济开发区", "")
newName = newName.replace("国家高新技术产业开发区","").replace("高新技术产业开发区","").replace("经济技术开发区","").replace("经济开发区", "")
}
if(newName == name){
esObj = source
}
}
// }
}
}
return esObj
}
static def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
def dbcursor = BAIDU_POI_COLL.find(q)
def address = null
for(def d : dbcursor){
address = d.addr
}
return address;
}
} }
@@ -189,7 +189,7 @@ class AddDevShapeData {
def run(){ def run(){
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\临时-没付费\\' def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\623西安'
def dir = new File(directoryPath) def dir = new File(directoryPath)
if (dir.isDirectory()) { if (dir.isDirectory()) {
dir.eachFileRecurse(FileType.FILES) { file -> dir.eachFileRecurse(FileType.FILES) { file ->
@@ -203,7 +203,7 @@ class AddDevShapeData {
def addDevInfo(file){ def addDevInfo(file){
ExcelUtils.multiThreadingModify(file, {d-> ExcelUtils.multiThreadingModify(file, {d->
// if(!d['I']){ if(!d['I']){
def name = d['E'] def name = d['E']
def q = new BasicDBObject() def q = new BasicDBObject()
q.put('_id',new ObjectId(d['A'])) q.put('_id',new ObjectId(d['A']))
@@ -218,24 +218,26 @@ class AddDevShapeData {
if(obj.shape){ if(obj.shape){
d['T'] = "有" d['T'] = "有"
} }
// if(obj.m_date){ if(obj.m_date){
// d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd") d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
// } }
// if(devAreaMap){ if(devAreaMap){
// devAreaMap.each {k,v-> devAreaMap.each {k,v->
// if(obj.name && k && obj.name.contains(k)){ if(obj.name && k && obj.name.contains(k)){
// d['M'] = v.area_info d['M'] = v.area_info
// d['N'] = v.count d['N'] = v.count
// d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd") d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
// } }
// } }
//
// }else{ }else{
// println name+" 空值" println name+" 空值"
// } }
}else{
d['S'] = "删除"
} }
// } }
}) })
} }
@@ -269,13 +271,15 @@ class AddDevShapeData {
DEVELOPMENT_ZONE_43_COLL.save(obj) DEVELOPMENT_ZONE_43_COLL.save(obj)
}else{ }else{
udata.put("shape", resultMap) udata.put("shape", resultMap)
// udata.put("m_date", new Date()) udata.put("m_date", new Date())
udata.put("bd_shape",bd_resultMap) udata.put("bd_shape",bd_resultMap)
udata.put("operate_result" , "已处理") udata.put("operate_result" , "已处理")
udata.put("operator", "外包") udata.put("operator", "外包")
DEVELOPMENT_ZONE_43_COLL.update(q, new BasicDBObject('$set', udata), false, false) DEVELOPMENT_ZONE_43_COLL.update(q, new BasicDBObject('$set', udata), false, false)
} }
} }
}else{
println "【开发区】"+retMap.name +" 库中未查到"
} }
} }
} }
@@ -296,7 +300,7 @@ class AddDevShapeData {
areaMap.count = coordinates.size() areaMap.count = coordinates.size()
def areas = [] def areas = []
// if(name.contains("江苏常熟综合保税区")){ if(name.contains("陕西蓝田工业园")){
for(def pointList : coordinates){ for(def pointList : coordinates){
index ++ index ++
def gaode_coordinates = [] def gaode_coordinates = []
@@ -339,7 +343,7 @@ class AddDevShapeData {
} }
areaMap.area_info = areas.join("\n") areaMap.area_info = areas.join("\n")
// } }
}else{ }else{
println file +" 有多个" println file +" 有多个"
@@ -31,16 +31,16 @@ class AutoAddDistrictBatch extends BaseBatch{
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping() def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
static main(args) { static main(args) {
// runBatch(new AutoAddDistrictBatch(), args) runBatch(new AutoAddDistrictBatch(), args)
def task = new AutoAddDistrictBatch(); // def task = new AutoAddDistrictBatch();
task.checkMap() // task.checkMap()
} }
@Override @Override
void process(Object args) { void process(Object args) {
//查找第三方匹配没有匹配上的小区 //查找第三方匹配没有匹配上的小区
def citys = CommonDatautils.findAllCity() def citys = ["三明"]//CommonDatautils.findAllCity()
def list = [] def list = []
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"] list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"]
for(def c : citys){ for(def c : citys){
@@ -52,7 +52,7 @@ class AutoAddDistrictBatch extends BaseBatch{
def fileName = "/home/goojia/gj/add_dist_info.xlsx" def fileName = "/home/goojia/gj/add_dist_info.xlsx"
if (CommonDatautils.isWindow()) { if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\add_dist_info_1.xlsx" fileName = "D:\\数据\\数据建设\\小区\\add_dist_info_1.xlsx"
} }
ExcelUtils.write(fileName, list, "待入库数据") ExcelUtils.write(fileName, list, "待入库数据")
@@ -102,12 +102,14 @@ class AutoAddDistrictBatch extends BaseBatch{
def gaode_obj = findGaodeCount(arrs[0], arrs[1], arrs[2]) def gaode_obj = findGaodeCount(arrs[0], arrs[1], arrs[2])
def gaode_count = 0 def gaode_count = 0
def gaode_type = null def gaode_type = null
def gaode_name = ""
if(gaode_obj){ if(gaode_obj){
gaode_count = 1 gaode_count = 1
gaode_type = gaode_obj.type gaode_type = gaode_obj.type
gaode_name = gaode_obj.name
} }
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type] rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type, gaode_name]
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace() e.printStackTrace()
} finally { } finally {
@@ -165,7 +167,7 @@ class AutoAddDistrictBatch extends BaseBatch{
String sql = "select name,type from amap_poi_2024 where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')" String sql = "select name,type from amap_poi_2024 where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null def obj = null
ElasticUtils.executeSqlQuery(sql,{ d-> ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name == name){ //&& d.shape if(d.name == name || d.name.contains(name) || name.contains(d.name)){ //&& d.shape
obj = d obj = d
} }
}) })
@@ -1,11 +1,13 @@
package com.fangjia.script.gj.data_handle package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.GeoUtils import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
@@ -44,6 +46,7 @@ class DevProcess {
// task.devNameSplitRoad() // task.devNameSplitRoad()
// task.addFlag() // task.addFlag()
// task.matchDev() // task.matchDev()
// task.findDev()
// task.calAddrDistince() // task.calAddrDistince()
// task.updateCityAndDist() // task.updateCityAndDist()
@@ -57,7 +60,7 @@ class DevProcess {
// task.createDevMapping() // task.createDevMapping()
// task.deleteAddr() // task.deleteAddr()
// task.deleteAddr2() // task.deleteAddr2()
task.synchDevTo100() // task.synchDevTo100()
// task.statShape() // task.statShape()
// task.addNewDev2() // task.addNewDev2()
@@ -85,9 +88,19 @@ class DevProcess {
// task.updateDevShape()//修改开发区结构 // task.updateDevShape()//修改开发区结构
task.aiDevTown()//ai获取结果
} }
def aiDevTown(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省开发区-ai结果.xlsx", { d ->
def res = FangjiaClientUtils.aiResult("河北省哪些乡镇街道完全隶属于"+d['E']+",有则以逗号隔开方式输出,无则输出无,其它信息不输出")
def message = res.data.replaceAll("\n", "")
d['F'] = res.code
d['G'] = message
println d['E'] +" "+res.code + " "+message
})
}
def parseCityAndDist(){ def parseCityAndDist(){
@@ -440,35 +453,80 @@ class DevProcess {
} }
def matchDev(){ def matchDev(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\高新区.xls",{d-> ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\省级开发区名单\\2018年版.xlsx",{d->
if(!d['E'] && d['D']){ if(!d['G'] && d['C']){
def q = new BasicDBObject() def q = new BasicDBObject()
// q.put("name", d['D']) q.put("name", d['C'])
q.put("city",d['C']) def way = "相同"
// q.put('alias', new BasicDBObject('$in',[d['D']]))
def name = d['D'] //.replace('经济技术开发区','经济开发区')
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
def obj = development_zone_CONN.findOne(q) def obj = development_zone_CONN.findOne(q)
if(obj){ if(!obj){
d['G'] = "相似" q.put('alias', new BasicDBObject('$in',[d['C']]))
// d['K'] = obj.name obj = development_zone_CONN.findOne(q)
// d['C'] = obj._id.toString() way = "别名"
d['E'] = obj.name
// if(obj.name == d['D']){
// d['G'] = "相同"
// }else{
// d['G'] = "包含"
// }
if(obj.shape){
d['F'] = "是"
}
// d['E'] = "别名"
// d['F'] = obj.name
} }
if(!obj){
def name = d['C'] //.replace('经济技术开发区','经济开发区')
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
obj = development_zone_CONN.findOne(q)
way = "模糊"
}
def allDevNames = []
if(!obj){
def query = new BasicDBObject()
def keys = new BasicDBObject()
keys.put("name", 1)
keys.put("alias", 1)
allDevNames = development_zone_CONN.find(query, keys).toList()
}
if(obj){
d['H'] = obj.name
d['G'] = way
}else{
if(allDevNames){
def maxScore = -100
def smilarObj = null
for(def dev : allDevNames){
def devNames = []
if(dev.alias){
devNames = dev.alias
}
devNames << dev.name
for(def aliasName : devNames){
def nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(d['C'], aliasName);
if(nameScore>maxScore){
maxScore = nameScore
smilarObj = dev
}
}
}
if(smilarObj){
d['H'] = smilarObj.name
d['G'] = "相似得分"
d['I'] = maxScore
}
}
}
} }
},0); },0);
} }
def findDev(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区外包\\开发区-0423(161个-浙江).xlsx", { d ->
def q = new BasicDBObject()
q.put("name", d['B'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['C'] = obj.city
d['D'] = obj.district
}
})
}
def calAddrDistince(){ def calAddrDistince(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d-> ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d->
if(d['M'] && d['N'] && d['Y'] && d['Z']){ if(d['M'] && d['N'] && d['Y'] && d['Z']){
@@ -5,6 +5,8 @@ import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.yinjihaun.util.HttpClient import com.fangjia.script.yinjihaun.util.HttpClient
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import groovyx.net.http.ContentType
import groovyx.net.http.RESTClient
import net.sf.json.JSONArray import net.sf.json.JSONArray
import net.sf.json.JSONObject import net.sf.json.JSONObject
@@ -423,25 +425,17 @@ class FangjiaClientUtils {
} }
static def aiResult(content){ static def aiResult(content){
def headers = [:] def client = new RESTClient('http://didata-cloud-server.fangjia.com/')
headers.put("accept", "application/json") def response = client.post(
headers.put("Content-Type", "application/json") path: '/chat/deepseek',
contentType: ContentType.JSON,
def url = "http://didata-cloud-server.fangjia.com/chat/deepseek" headers: [accept: 'application/json'],
try { body: [content: content]
Map<String, String> params = new HashMap<String, String>(); )
params.put("content", content); //
// println "Response status: ${response.responseData}"
String result = HttpClient.post(url, params, headers, "utf-8", true) // println "Response data: ${response.data}"
println result return response.data
if (result) {
def json = JSONObject.fromObject(result)
return json
}
} catch (Exception e) {
println e.getMessage()
}
return null
} }
} }
@@ -8,25 +8,41 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/** /**
* 浙商本地化部署同步脚本-逻辑修改 * 浙商本地化部署同步脚本-逻辑修改
* 浙商改为接口,所以不用以前的数据
*/ */
@BatchOwner(Watcher.GaoJuan) @BatchOwner(Watcher.GaoJuan)
class DistrictLevelBatch extends BaseBatch{ class DistrictLevelBatch extends BaseBatch{
public static final def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; // public static final def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName2("fangjia", "district_analy", ZS_MONGODB_URL) // public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName2("fangjia", "district_analy", ZS_MONGODB_URL)
public static final zsDistLevelMap = [:] public static final zsDistLevelMap = [:]
public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) { static main(args) {
runBatch(new DistrictLevelBatch(), args); runBatch(new DistrictLevelBatch(), args);
} }
@Override @Override
public void process(Object args) { public void process(Object args) {
def keys = new BasicDBObject("comm_id", 1)
keys.put("city", 1)
keys.put("region", 1)
keys.put("district_name", 1)
keys.put("status", 1)
keys.put("avg_price", 1)
keys.put("value_rate", 1)
def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject(),keys)
def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject())
for(def distLevel : zsDistLevelCursor){ for(def distLevel : zsDistLevelCursor){
zsDistLevelMap.put(distLevel._id.toString(), distLevel) zsDistLevelMap.put(distLevel.comm_id.toString(), distLevel)
} }
def citys = CommonDatautils.findAllCity(); def citys = CommonDatautils.findAllCity();
@@ -44,6 +60,7 @@ class DistrictLevelBatch extends BaseBatch{
def addPriceData(dists, priceMap){ def addPriceData(dists, priceMap){
def newLists = [] def newLists = []
for(def d : dists){ for(def d : dists){
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name); def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
if(price){ if(price){
d.avg_price = price; d.avg_price = price;
@@ -100,9 +117,11 @@ class DistrictLevelBatch extends BaseBatch{
def saveOrUpdateDB(city, datas){ def saveOrUpdateDB(city, datas){
def zsDistLevels = [] def zsDistLevels = []
def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject("city",city)) def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject("city",city), new BasicDBObject("comm_id", 1))
for(def distLevel : zsDistLevelCursor){ for(def distLevel : zsDistLevelCursor){
zsDistLevels << distLevel._id.toString() if(distLevel.comm_id){
zsDistLevels << distLevel.comm_id.toString()
}
} }
def newDataMap = [:] def newDataMap = [:]
@@ -113,17 +132,17 @@ class DistrictLevelBatch extends BaseBatch{
def udata = new BasicDBObject() def udata = new BasicDBObject()
udata.put("city", d.city) udata.put("city", d.city)
udata.put("region", d.region) udata.put("region", d.region)
udata.put("name", d.name) udata.put("district_name", d.name)
udata.put("level", d.level) udata.put("value_rate", d.level)
udata.put("avg_price", d.avg_price) // udata.put("avg_price", d.avg_price)
udata.put("status",0) udata.put("status",0)
udata.put("update_date",new Date().getTime()) udata.put("update_date",new Date().getTime())
if(zsDistLevel){ if(zsDistLevel){
if(zsDistLevel.city!=d.city||zsDistLevel.region!=d.region||zsDistLevel.name!=d.name||zsDistLevel.level!=d.level||zsDistLevel.avg_price!=d.avg_price||zsDistLevel.status!=0){ if(zsDistLevel.city!=d.city||zsDistLevel.region!=d.region||zsDistLevel.district_name!=d.name||zsDistLevel.value_rate!=d.level||zsDistLevel.avg_price!=d.avg_price||zsDistLevel.status!=0){
ZS_LEVEL_CONN.update(new BasicDBObject("_id",zsDistLevel._id), new BasicDBObject('$set',udata)) ZS_LEVEL_CONN.update(new BasicDBObject("_id",zsDistLevel._id), new BasicDBObject('$set',udata))
} }
}else{ }else{
udata.put("_id", d._id.toString()) udata.put("comm_id", d._id.toString())
addDatas << new BasicDBObject(udata) addDatas << new BasicDBObject(udata)
if(addDatas.size()%10000==0){ if(addDatas.size()%10000==0){
ZS_LEVEL_CONN.insert(addDatas); ZS_LEVEL_CONN.insert(addDatas);
@@ -134,16 +153,17 @@ class DistrictLevelBatch extends BaseBatch{
if(addDatas.size()>0){ if(addDatas.size()>0){
ZS_LEVEL_CONN.insert(addDatas); ZS_LEVEL_CONN.insert(addDatas);
} }
if(zsDistLevels.size()>0){ //先不删除,有些没价格
for(def zs_id : zsDistLevels){ // if(zsDistLevels.size()>0){
def newData = newDataMap.get(zs_id.toString()); // for(def zs_id : zsDistLevels){
if(!newData){ // def newData = newDataMap.get(zs_id.toString());
def udata = new BasicDBObject() // if(!newData){
udata.put("status",1) // def udata = new BasicDBObject()
udata.put("update_date",new Date().getTime()) // udata.put("status",1)
ZS_LEVEL_CONN.update(new BasicDBObject("_id",zs_id), new BasicDBObject('$set',udata)) // udata.put("update_date",new Date().getTime())
} // ZS_LEVEL_CONN.update(new BasicDBObject("comm_id",zs_id), new BasicDBObject('$set',udata))
} // }
} // }
// }
} }
} }