Merge branch 'script_gj' into 'master'
Script gj See merge request !1148
This commit is contained in:
@@ -555,6 +555,25 @@
|
||||
<version>0.0.12</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.codehaus.groovy.modules.http-builder</groupId>
|
||||
<artifactId>http-builder</artifactId>
|
||||
<version>0.7.1</version>
|
||||
<exclusions>
|
||||
<!-- 排除冲突的 xmlbeans -->
|
||||
<exclusion>
|
||||
<groupId>xmlbeans</groupId>
|
||||
<artifactId>xmlbeans</artifactId>
|
||||
</exclusion>
|
||||
<!-- 如果还有其他冲突的依赖(如 xercesImpl),也一并排除 -->
|
||||
<exclusion>
|
||||
<groupId>xerces</groupId>
|
||||
<artifactId>xercesImpl</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<!-- <dependency>-->
|
||||
<!-- <groupId>com.fangjia</groupId>-->
|
||||
<!-- <artifactId>fangjia-open</artifactId>-->
|
||||
|
||||
+1
-1
@@ -443,7 +443,7 @@ class CcbCommonUtils {
|
||||
}
|
||||
|
||||
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
|
||||
// q.put("name","新泾三村")
|
||||
// q.put("name","龙源星城")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"]))
|
||||
|
||||
|
||||
+119
-1
@@ -105,6 +105,8 @@ class CommonDatautils {
|
||||
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com")
|
||||
public static final DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
|
||||
|
||||
public static DSCollection getHouseDataStoreConnection(){
|
||||
if(house_dataStoreConnection != null) {
|
||||
@@ -811,6 +813,8 @@ class CommonDatautils {
|
||||
// q.put("status",new BasicDBObject('$ne', 0))
|
||||
q.put("cat", cat);
|
||||
q.put("city", city)
|
||||
// q.put("loop_line_info",new BasicDBObject('$exists',false))
|
||||
// q.put("name","经纬天地园二期")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -4142,9 +4146,13 @@ class CommonDatautils {
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true)
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
.must(QueryBuilders.termQuery("region", region))
|
||||
.must(QueryBuilders.termQuery("cat", cat)).must(date_range)
|
||||
|
||||
if(region){
|
||||
queryBuilder.must(QueryBuilders.termQuery("region", region))
|
||||
}
|
||||
|
||||
|
||||
if(name || alias){
|
||||
if(!alias){
|
||||
alias = []
|
||||
@@ -4250,6 +4258,19 @@ class CommonDatautils {
|
||||
return rets;
|
||||
}
|
||||
|
||||
static def findAllNewPoiDatas(city, region, name, small_type){
|
||||
def rets = []
|
||||
def index= 0
|
||||
String sql = "select cityname,adname,name,address,location,type,id,parent from amap_poi_2024 where cityname='${city}'"
|
||||
ElasticUtils.executeSqlQuery(sql, { map->
|
||||
// if(map.address && map.address.contains("五角场")){
|
||||
index++
|
||||
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y, "parent_id":map.parent]
|
||||
// }
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -5644,4 +5665,101 @@ class CommonDatautils {
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
static def findGaoDeDatas(city, region, name){
|
||||
String sql = "select name,type from amap_poi_2024 where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') limit 100"
|
||||
def obj = null
|
||||
ElasticUtils.executeSqlQuery(sql,{ d->
|
||||
if(d.name == name){ //&& d.shape
|
||||
obj = d
|
||||
}
|
||||
})
|
||||
return obj;
|
||||
}
|
||||
|
||||
|
||||
static def findGaoDeDatas2(city, region, name){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_2024/_search"
|
||||
def body = """
|
||||
{
|
||||
"_source": {
|
||||
"includes": ["cat", "address", "id","name", "township","location"]
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"term": {
|
||||
"city" : "${city}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"term": {
|
||||
"district" : "${region}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"name": {"query": "${name}","operator": "and"}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = new JsonSlurper().parseText(res)
|
||||
// if(name=="河北河间经济开发区"){
|
||||
// println name
|
||||
// }
|
||||
def esObj = null
|
||||
if(response && response.hits){
|
||||
def hits = response.hits.hits
|
||||
if(hits){
|
||||
def d = hits[0]
|
||||
// if(d._source.cat.contains("住宅")){
|
||||
def source = d._source
|
||||
def newName = source.name
|
||||
if(newName == name){
|
||||
esObj = source
|
||||
}else if (newName.contains("开发区") && name.contains("开发区") && (newName.contains(name) || name.contains(newName))){
|
||||
esObj = source
|
||||
}else if(newName.contains("园区") && name.contains("园区") && (newName.contains(name) || name.contains(newName))){
|
||||
esObj = source
|
||||
}else if(newName.contains("示范区") && name.contains("示范区") && (newName.contains(name) || name.contains(newName))){
|
||||
esObj = source
|
||||
}else{
|
||||
if(name.contains("开发区")){
|
||||
name = name.replace("国家高新技术产业开发区","").replace("高新技术产业开发区","").replace("经济技术开发区","").replace("经济开发区", "")
|
||||
newName = newName.replace("国家高新技术产业开发区","").replace("高新技术产业开发区","").replace("经济技术开发区","").replace("经济开发区", "")
|
||||
}
|
||||
if(newName == name){
|
||||
esObj = source
|
||||
}
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
return esObj
|
||||
}
|
||||
|
||||
static def findBaiduDatas(city, district, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("name", name)
|
||||
q.put('std_tag', '房地产;住宅区')
|
||||
|
||||
def dbcursor = BAIDU_POI_COLL.find(q)
|
||||
def address = null
|
||||
for(def d : dbcursor){
|
||||
address = d.addr
|
||||
}
|
||||
return address;
|
||||
}
|
||||
}
|
||||
|
||||
+25
-21
@@ -189,7 +189,7 @@ class AddDevShapeData {
|
||||
|
||||
|
||||
def run(){
|
||||
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\临时-没付费\\'
|
||||
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\623西安'
|
||||
def dir = new File(directoryPath)
|
||||
if (dir.isDirectory()) {
|
||||
dir.eachFileRecurse(FileType.FILES) { file ->
|
||||
@@ -203,7 +203,7 @@ class AddDevShapeData {
|
||||
|
||||
def addDevInfo(file){
|
||||
ExcelUtils.multiThreadingModify(file, {d->
|
||||
// if(!d['I']){
|
||||
if(!d['I']){
|
||||
def name = d['E']
|
||||
def q = new BasicDBObject()
|
||||
q.put('_id',new ObjectId(d['A']))
|
||||
@@ -218,24 +218,26 @@ class AddDevShapeData {
|
||||
if(obj.shape){
|
||||
d['T'] = "有"
|
||||
}
|
||||
// if(obj.m_date){
|
||||
// d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
|
||||
// }
|
||||
if(obj.m_date){
|
||||
d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
|
||||
}
|
||||
|
||||
// if(devAreaMap){
|
||||
// devAreaMap.each {k,v->
|
||||
// if(obj.name && k && obj.name.contains(k)){
|
||||
// d['M'] = v.area_info
|
||||
// d['N'] = v.count
|
||||
// d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// }else{
|
||||
// println name+" 空值"
|
||||
// }
|
||||
if(devAreaMap){
|
||||
devAreaMap.each {k,v->
|
||||
if(obj.name && k && obj.name.contains(k)){
|
||||
d['M'] = v.area_info
|
||||
d['N'] = v.count
|
||||
d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
|
||||
}
|
||||
}
|
||||
|
||||
}else{
|
||||
println name+" 空值"
|
||||
}
|
||||
}else{
|
||||
d['S'] = "删除"
|
||||
}
|
||||
// }
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
@@ -269,13 +271,15 @@ class AddDevShapeData {
|
||||
DEVELOPMENT_ZONE_43_COLL.save(obj)
|
||||
}else{
|
||||
udata.put("shape", resultMap)
|
||||
// udata.put("m_date", new Date())
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("bd_shape",bd_resultMap)
|
||||
udata.put("operate_result" , "已处理")
|
||||
udata.put("operator", "外包")
|
||||
DEVELOPMENT_ZONE_43_COLL.update(q, new BasicDBObject('$set', udata), false, false)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println "【开发区】"+retMap.name +" 库中未查到"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -296,7 +300,7 @@ class AddDevShapeData {
|
||||
|
||||
areaMap.count = coordinates.size()
|
||||
def areas = []
|
||||
// if(name.contains("江苏常熟综合保税区")){
|
||||
if(name.contains("陕西蓝田工业园")){
|
||||
for(def pointList : coordinates){
|
||||
index ++
|
||||
def gaode_coordinates = []
|
||||
@@ -339,7 +343,7 @@ class AddDevShapeData {
|
||||
|
||||
}
|
||||
areaMap.area_info = areas.join("\n")
|
||||
// }
|
||||
}
|
||||
|
||||
}else{
|
||||
println file +" 有多个"
|
||||
|
||||
+9
-7
@@ -31,16 +31,16 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
static main(args) {
|
||||
// runBatch(new AutoAddDistrictBatch(), args)
|
||||
runBatch(new AutoAddDistrictBatch(), args)
|
||||
|
||||
def task = new AutoAddDistrictBatch();
|
||||
task.checkMap()
|
||||
// def task = new AutoAddDistrictBatch();
|
||||
// task.checkMap()
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
//查找第三方匹配没有匹配上的小区
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def citys = ["三明"]//CommonDatautils.findAllCity()
|
||||
def list = []
|
||||
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"]
|
||||
for(def c : citys){
|
||||
@@ -52,7 +52,7 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
|
||||
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\add_dist_info_1.xlsx"
|
||||
fileName = "D:\\数据\\数据建设\\小区\\add_dist_info_1.xlsx"
|
||||
}
|
||||
ExcelUtils.write(fileName, list, "待入库数据")
|
||||
|
||||
@@ -102,12 +102,14 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
def gaode_obj = findGaodeCount(arrs[0], arrs[1], arrs[2])
|
||||
def gaode_count = 0
|
||||
def gaode_type = null
|
||||
def gaode_name = ""
|
||||
if(gaode_obj){
|
||||
gaode_count = 1
|
||||
gaode_type = gaode_obj.type
|
||||
gaode_name = gaode_obj.name
|
||||
}
|
||||
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
|
||||
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type]
|
||||
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type, gaode_name]
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
@@ -165,7 +167,7 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
String sql = "select name,type from amap_poi_2024 where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
|
||||
def obj = null
|
||||
ElasticUtils.executeSqlQuery(sql,{ d->
|
||||
if(d.name == name){ //&& d.shape
|
||||
if(d.name == name || d.name.contains(name) || name.contains(d.name)){ //&& d.shape
|
||||
obj = d
|
||||
}
|
||||
})
|
||||
|
||||
+82
-24
@@ -1,11 +1,13 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
@@ -44,6 +46,7 @@ class DevProcess {
|
||||
// task.devNameSplitRoad()
|
||||
// task.addFlag()
|
||||
// task.matchDev()
|
||||
// task.findDev()
|
||||
|
||||
// task.calAddrDistince()
|
||||
// task.updateCityAndDist()
|
||||
@@ -57,7 +60,7 @@ class DevProcess {
|
||||
// task.createDevMapping()
|
||||
// task.deleteAddr()
|
||||
// task.deleteAddr2()
|
||||
task.synchDevTo100()
|
||||
// task.synchDevTo100()
|
||||
// task.statShape()
|
||||
// task.addNewDev2()
|
||||
|
||||
@@ -85,9 +88,19 @@ class DevProcess {
|
||||
|
||||
// task.updateDevShape()//修改开发区结构
|
||||
|
||||
task.aiDevTown()//ai获取结果
|
||||
|
||||
}
|
||||
|
||||
|
||||
def aiDevTown(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省开发区-ai结果.xlsx", { d ->
|
||||
def res = FangjiaClientUtils.aiResult("河北省哪些乡镇街道完全隶属于"+d['E']+",有则以逗号隔开方式输出,无则输出无,其它信息不输出")
|
||||
def message = res.data.replaceAll("\n", "")
|
||||
d['F'] = res.code
|
||||
d['G'] = message
|
||||
println d['E'] +" "+res.code + " "+message
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def parseCityAndDist(){
|
||||
@@ -440,35 +453,80 @@ class DevProcess {
|
||||
}
|
||||
|
||||
def matchDev(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\高新区.xls",{d->
|
||||
if(!d['E'] && d['D']){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\省级开发区名单\\2018年版.xlsx",{d->
|
||||
if(!d['G'] && d['C']){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("name", d['D'])
|
||||
q.put("city",d['C'])
|
||||
// q.put('alias', new BasicDBObject('$in',[d['D']]))
|
||||
def name = d['D'] //.replace('经济技术开发区','经济开发区')
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put("name", d['C'])
|
||||
def way = "相同"
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['G'] = "相似"
|
||||
// d['K'] = obj.name
|
||||
// d['C'] = obj._id.toString()
|
||||
d['E'] = obj.name
|
||||
// if(obj.name == d['D']){
|
||||
// d['G'] = "相同"
|
||||
// }else{
|
||||
// d['G'] = "包含"
|
||||
// }
|
||||
if(obj.shape){
|
||||
d['F'] = "是"
|
||||
}
|
||||
// d['E'] = "别名"
|
||||
// d['F'] = obj.name
|
||||
if(!obj){
|
||||
q.put('alias', new BasicDBObject('$in',[d['C']]))
|
||||
obj = development_zone_CONN.findOne(q)
|
||||
way = "别名"
|
||||
}
|
||||
if(!obj){
|
||||
def name = d['C'] //.replace('经济技术开发区','经济开发区')
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
obj = development_zone_CONN.findOne(q)
|
||||
way = "模糊"
|
||||
}
|
||||
def allDevNames = []
|
||||
if(!obj){
|
||||
def query = new BasicDBObject()
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("name", 1)
|
||||
keys.put("alias", 1)
|
||||
allDevNames = development_zone_CONN.find(query, keys).toList()
|
||||
}
|
||||
if(obj){
|
||||
d['H'] = obj.name
|
||||
d['G'] = way
|
||||
}else{
|
||||
if(allDevNames){
|
||||
def maxScore = -100
|
||||
def smilarObj = null
|
||||
for(def dev : allDevNames){
|
||||
def devNames = []
|
||||
if(dev.alias){
|
||||
devNames = dev.alias
|
||||
}
|
||||
devNames << dev.name
|
||||
for(def aliasName : devNames){
|
||||
def nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(d['C'], aliasName);
|
||||
if(nameScore>maxScore){
|
||||
maxScore = nameScore
|
||||
smilarObj = dev
|
||||
}
|
||||
}
|
||||
}
|
||||
if(smilarObj){
|
||||
d['H'] = smilarObj.name
|
||||
d['G'] = "相似得分"
|
||||
d['I'] = maxScore
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
def findDev(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区外包\\开发区-0423(161个-浙江).xlsx", { d ->
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['B'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['C'] = obj.city
|
||||
d['D'] = obj.district
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def calAddrDistince(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d->
|
||||
if(d['M'] && d['N'] && d['Y'] && d['Z']){
|
||||
|
||||
+13
-19
@@ -5,6 +5,8 @@ import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.yinjihaun.util.HttpClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import groovyx.net.http.ContentType
|
||||
import groovyx.net.http.RESTClient
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
@@ -423,25 +425,17 @@ class FangjiaClientUtils {
|
||||
}
|
||||
|
||||
static def aiResult(content){
|
||||
def headers = [:]
|
||||
headers.put("accept", "application/json")
|
||||
headers.put("Content-Type", "application/json")
|
||||
|
||||
def url = "http://didata-cloud-server.fangjia.com/chat/deepseek"
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("content", content);
|
||||
|
||||
String result = HttpClient.post(url, params, headers, "utf-8", true)
|
||||
println result
|
||||
if (result) {
|
||||
def json = JSONObject.fromObject(result)
|
||||
return json
|
||||
}
|
||||
} catch (Exception e) {
|
||||
println e.getMessage()
|
||||
}
|
||||
return null
|
||||
def client = new RESTClient('http://didata-cloud-server.fangjia.com/')
|
||||
def response = client.post(
|
||||
path: '/chat/deepseek',
|
||||
contentType: ContentType.JSON,
|
||||
headers: [accept: 'application/json'],
|
||||
body: [content: content]
|
||||
)
|
||||
//
|
||||
// println "Response status: ${response.responseData}"
|
||||
// println "Response data: ${response.data}"
|
||||
return response.data
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+43
-23
@@ -8,25 +8,41 @@ import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 浙商本地化部署同步脚本-逻辑修改
|
||||
* 浙商改为接口,所以不用以前的数据
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class DistrictLevelBatch extends BaseBatch{
|
||||
|
||||
public static final def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName2("fangjia", "district_analy", ZS_MONGODB_URL)
|
||||
// public static final def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
// public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName2("fangjia", "district_analy", ZS_MONGODB_URL)
|
||||
public static final zsDistLevelMap = [:]
|
||||
|
||||
public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
static main(args) {
|
||||
runBatch(new DistrictLevelBatch(), args);
|
||||
}
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def keys = new BasicDBObject("comm_id", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
keys.put("district_name", 1)
|
||||
keys.put("status", 1)
|
||||
keys.put("avg_price", 1)
|
||||
keys.put("value_rate", 1)
|
||||
def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject(),keys)
|
||||
|
||||
def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject())
|
||||
for(def distLevel : zsDistLevelCursor){
|
||||
zsDistLevelMap.put(distLevel._id.toString(), distLevel)
|
||||
zsDistLevelMap.put(distLevel.comm_id.toString(), distLevel)
|
||||
}
|
||||
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
@@ -44,6 +60,7 @@ class DistrictLevelBatch extends BaseBatch{
|
||||
def addPriceData(dists, priceMap){
|
||||
def newLists = []
|
||||
for(def d : dists){
|
||||
|
||||
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
|
||||
if(price){
|
||||
d.avg_price = price;
|
||||
@@ -100,9 +117,11 @@ class DistrictLevelBatch extends BaseBatch{
|
||||
|
||||
def saveOrUpdateDB(city, datas){
|
||||
def zsDistLevels = []
|
||||
def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject("city",city))
|
||||
def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject("city",city), new BasicDBObject("comm_id", 1))
|
||||
for(def distLevel : zsDistLevelCursor){
|
||||
zsDistLevels << distLevel._id.toString()
|
||||
if(distLevel.comm_id){
|
||||
zsDistLevels << distLevel.comm_id.toString()
|
||||
}
|
||||
}
|
||||
|
||||
def newDataMap = [:]
|
||||
@@ -113,17 +132,17 @@ class DistrictLevelBatch extends BaseBatch{
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d.city)
|
||||
udata.put("region", d.region)
|
||||
udata.put("name", d.name)
|
||||
udata.put("level", d.level)
|
||||
udata.put("avg_price", d.avg_price)
|
||||
udata.put("district_name", d.name)
|
||||
udata.put("value_rate", d.level)
|
||||
// udata.put("avg_price", d.avg_price)
|
||||
udata.put("status",0)
|
||||
udata.put("update_date",new Date().getTime())
|
||||
if(zsDistLevel){
|
||||
if(zsDistLevel.city!=d.city||zsDistLevel.region!=d.region||zsDistLevel.name!=d.name||zsDistLevel.level!=d.level||zsDistLevel.avg_price!=d.avg_price||zsDistLevel.status!=0){
|
||||
if(zsDistLevel.city!=d.city||zsDistLevel.region!=d.region||zsDistLevel.district_name!=d.name||zsDistLevel.value_rate!=d.level||zsDistLevel.avg_price!=d.avg_price||zsDistLevel.status!=0){
|
||||
ZS_LEVEL_CONN.update(new BasicDBObject("_id",zsDistLevel._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}else{
|
||||
udata.put("_id", d._id.toString())
|
||||
udata.put("comm_id", d._id.toString())
|
||||
addDatas << new BasicDBObject(udata)
|
||||
if(addDatas.size()%10000==0){
|
||||
ZS_LEVEL_CONN.insert(addDatas);
|
||||
@@ -134,16 +153,17 @@ class DistrictLevelBatch extends BaseBatch{
|
||||
if(addDatas.size()>0){
|
||||
ZS_LEVEL_CONN.insert(addDatas);
|
||||
}
|
||||
if(zsDistLevels.size()>0){
|
||||
for(def zs_id : zsDistLevels){
|
||||
def newData = newDataMap.get(zs_id.toString());
|
||||
if(!newData){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status",1)
|
||||
udata.put("update_date",new Date().getTime())
|
||||
ZS_LEVEL_CONN.update(new BasicDBObject("_id",zs_id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
//先不删除,有些没价格
|
||||
// if(zsDistLevels.size()>0){
|
||||
// for(def zs_id : zsDistLevels){
|
||||
// def newData = newDataMap.get(zs_id.toString());
|
||||
// if(!newData){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("status",1)
|
||||
// udata.put("update_date",new Date().getTime())
|
||||
// ZS_LEVEL_CONN.update(new BasicDBObject("comm_id",zs_id), new BasicDBObject('$set',udata))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user