diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml
index ee869ba86..f398b19eb 100644
--- a/projects/fangjia-script/pom.xml
+++ b/projects/fangjia-script/pom.xml
@@ -555,6 +555,25 @@
0.0.12
+
+
+ org.codehaus.groovy.modules.http-builder
+ http-builder
+ 0.7.1
+
+
+
+ xmlbeans
+ xmlbeans
+
+
+
+ xerces
+ xercesImpl
+
+
+
+
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy
index be5714bfd..4dd815364 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy
@@ -443,7 +443,7 @@ class CcbCommonUtils {
}
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//Òò5ÔÂ1ÈÕÐÂÔö¹ý¶à£¬ËùÒÔÔÝÇÒÏÈÈ¥µô ´ËÅúÊý¾Ý¸ÄΪname_cat=13
-// q.put("name","ÐÂãþÈý´å")
+// q.put("name","ÁúÔ´ÐdzÇ")
// q.put("name_cat", 0);
// q.put("name", new BasicDBObject('$in', ["ÖÐÐÅÌ©¸»ÓÖÒ»³ÇÒ»ÆÚ","ÖÐÐÅÌ©¸»ÓÖÒ»³Ç"]))
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
index 17a82f2fc..a595aa2fe 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
@@ -105,6 +105,8 @@ class CommonDatautils {
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com")
+ public static final DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
+
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
@@ -811,6 +813,8 @@ class CommonDatautils {
// q.put("status",new BasicDBObject('$ne', 0))
q.put("cat", cat);
q.put("city", city)
+// q.put("loop_line_info",new BasicDBObject('$exists',false))
+// q.put("name","¾Î³ÌìµØÔ°¶þÆÚ")
DBObject keys = new BasicDBObject();
if(fields){
@@ -4142,9 +4146,13 @@ class CommonDatautils {
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true)
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
- .must(QueryBuilders.termQuery("region", region))
.must(QueryBuilders.termQuery("cat", cat)).must(date_range)
+ if(region){
+ queryBuilder.must(QueryBuilders.termQuery("region", region))
+ }
+
+
if(name || alias){
if(!alias){
alias = []
@@ -4250,6 +4258,19 @@ class CommonDatautils {
return rets;
}
+ static def findAllNewPoiDatas(city, region, name, small_type){
+ def rets = []
+ def index= 0
+ String sql = "select cityname,adname,name,address,location,type,id,parent from amap_poi_2024 where cityname='${city}'"
+ ElasticUtils.executeSqlQuery(sql, { map->
+// if(map.address && map.address.contains("Îå½Ç³¡")){
+ index++
+ rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y, "parent_id":map.parent]
+// }
+ })
+ return rets;
+ }
+
@@ -5644,4 +5665,101 @@ class CommonDatautils {
})
return rets;
}
+
+
+ static def findGaoDeDatas(city, region, name){
+ String sql = "select name,type from amap_poi_2024 where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') limit 100"
+ def obj = null
+ ElasticUtils.executeSqlQuery(sql,{ d->
+ if(d.name == name){ //&& d.shape
+ obj = d
+ }
+ })
+ return obj;
+ }
+
+
+ static def findGaoDeDatas2(city, region, name){
+ def header = [:]
+ header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
+ String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_2024/_search"
+ def body = """
+ {
+ "_source": {
+ "includes": ["cat", "address", "id","name", "township","location"]
+ },
+ "query": {
+ "bool": {
+ "must": [
+ {
+ "term": {
+ "city" : "${city}"
+ }
+ },
+ {
+ "term": {
+ "district" : "${region}"
+ }
+ },
+ {
+ "match": {
+ "name": {"query": "${name}","operator": "and"}
+ }
+ }
+ ]
+ }
+ }
+ }
+ """
+ HttpRequest request = new HttpRequest()
+ String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
+ def response = new JsonSlurper().parseText(res)
+// if(name=="ºÓ±±ºÓ¼ä¾¼Ã¿ª·¢Çø"){
+// println name
+// }
+ def esObj = null
+ if(response && response.hits){
+ def hits = response.hits.hits
+ if(hits){
+ def d = hits[0]
+// if(d._source.cat.contains("סլ")){
+ def source = d._source
+ def newName = source.name
+ if(newName == name){
+ esObj = source
+ }else if (newName.contains("¿ª·¢Çø") && name.contains("¿ª·¢Çø") && (newName.contains(name) || name.contains(newName))){
+ esObj = source
+ }else if(newName.contains("Ô°Çø") && name.contains("Ô°Çø") && (newName.contains(name) || name.contains(newName))){
+ esObj = source
+ }else if(newName.contains("ʾ·¶Çø") && name.contains("ʾ·¶Çø") && (newName.contains(name) || name.contains(newName))){
+ esObj = source
+ }else{
+ if(name.contains("¿ª·¢Çø")){
+ name = name.replace("¹ú¼Ò¸ßм¼Êõ²úÒµ¿ª·¢Çø","").replace("¸ßм¼Êõ²úÒµ¿ª·¢Çø","").replace("¾¼Ã¼¼Êõ¿ª·¢Çø","").replace("¾¼Ã¿ª·¢Çø", "")
+ newName = newName.replace("¹ú¼Ò¸ßм¼Êõ²úÒµ¿ª·¢Çø","").replace("¸ßм¼Êõ²úÒµ¿ª·¢Çø","").replace("¾¼Ã¼¼Êõ¿ª·¢Çø","").replace("¾¼Ã¿ª·¢Çø", "")
+ }
+ if(newName == name){
+ esObj = source
+ }
+ }
+// }
+ }
+ }
+ return esObj
+ }
+
+ static def findBaiduDatas(city, district, name){
+ def q = new BasicDBObject()
+ q.put("city", city)
+ q.put("district", district)
+ q.put("name", name)
+ q.put('std_tag', '·¿µØ²ú;×¡Õ¬Çø')
+
+ def dbcursor = BAIDU_POI_COLL.find(q)
+ def address = null
+ for(def d : dbcursor){
+ address = d.addr
+ }
+ return address;
+ }
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy
index a2d1c56b8..a605dcc02 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy
@@ -189,7 +189,7 @@ class AddDevShapeData {
def run(){
- def directoryPath = 'D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢ÇøÍâ°ü\\ÁÙʱ-û¸¶·Ñ\\'
+ def directoryPath = 'D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢ÇøÍâ°ü\\623Î÷°²'
def dir = new File(directoryPath)
if (dir.isDirectory()) {
dir.eachFileRecurse(FileType.FILES) { file ->
@@ -203,7 +203,7 @@ class AddDevShapeData {
def addDevInfo(file){
ExcelUtils.multiThreadingModify(file, {d->
-// if(!d['I']){
+ if(!d['I']){
def name = d['E']
def q = new BasicDBObject()
q.put('_id',new ObjectId(d['A']))
@@ -218,24 +218,26 @@ class AddDevShapeData {
if(obj.shape){
d['T'] = "ÓÐ"
}
-// if(obj.m_date){
-// d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
-// }
+ if(obj.m_date){
+ d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
+ }
-// if(devAreaMap){
-// devAreaMap.each {k,v->
-// if(obj.name && k && obj.name.contains(k)){
-// d['M'] = v.area_info
-// d['N'] = v.count
-// d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
-// }
-// }
-//
-// }else{
-// println name+" ¿ÕÖµ"
-// }
+ if(devAreaMap){
+ devAreaMap.each {k,v->
+ if(obj.name && k && obj.name.contains(k)){
+ d['M'] = v.area_info
+ d['N'] = v.count
+ d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
+ }
+ }
+
+ }else{
+ println name+" ¿ÕÖµ"
+ }
+ }else{
+ d['S'] = "ɾ³ý"
}
-// }
+ }
})
}
@@ -269,13 +271,15 @@ class AddDevShapeData {
DEVELOPMENT_ZONE_43_COLL.save(obj)
}else{
udata.put("shape", resultMap)
-// udata.put("m_date", new Date())
+ udata.put("m_date", new Date())
udata.put("bd_shape",bd_resultMap)
udata.put("operate_result" , "ÒÑ´¦Àí")
udata.put("operator", "Íâ°ü")
DEVELOPMENT_ZONE_43_COLL.update(q, new BasicDBObject('$set', udata), false, false)
}
}
+ }else{
+ println "¡¾¿ª·¢Çø¡¿"+retMap.name +" ¿âÖÐδ²éµ½"
}
}
}
@@ -296,7 +300,7 @@ class AddDevShapeData {
areaMap.count = coordinates.size()
def areas = []
-// if(name.contains("½ËÕ³£Êì×ۺϱ£Ë°Çø")){
+ if(name.contains("ÉÂÎ÷À¶Ì﹤ҵ԰")){
for(def pointList : coordinates){
index ++
def gaode_coordinates = []
@@ -339,7 +343,7 @@ class AddDevShapeData {
}
areaMap.area_info = areas.join("\n")
-// }
+ }
}else{
println file +" Óжà¸ö"
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy
index 9d4bc3bb9..1cdff119f 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy
@@ -31,16 +31,16 @@ class AutoAddDistrictBatch extends BaseBatch{
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
static main(args) {
-// runBatch(new AutoAddDistrictBatch(), args)
+ runBatch(new AutoAddDistrictBatch(), args)
- def task = new AutoAddDistrictBatch();
- task.checkMap()
+// def task = new AutoAddDistrictBatch();
+// task.checkMap()
}
@Override
void process(Object args) {
//²éÕÒµÚÈý·½Æ¥ÅäûÓÐÆ¥ÅäÉϵÄÐ¡Çø
- def citys = CommonDatautils.findAllCity()
+ def citys = ["ÈýÃ÷"]//CommonDatautils.findAllCity()
def list = []
list << ["³ÇÊÐ", "ÇøÓò", "Ãû³Æ", "³É½»Á¿", "·¿Ô´Á¿", "°Ù¶ÈµØÍ¼", "°Ù¶ÈÀàÐÍ","¸ßµÂµØÍ¼", "¸ßµÂÀàÐÍ"]
for(def c : citys){
@@ -52,7 +52,7 @@ class AutoAddDistrictBatch extends BaseBatch{
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
if (CommonDatautils.isWindow()) {
- fileName = "D:\\Êý¾Ý\\add_dist_info_1.xlsx"
+ fileName = "D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\Ð¡Çø\\add_dist_info_1.xlsx"
}
ExcelUtils.write(fileName, list, "´ýÈë¿âÊý¾Ý")
@@ -102,12 +102,14 @@ class AutoAddDistrictBatch extends BaseBatch{
def gaode_obj = findGaodeCount(arrs[0], arrs[1], arrs[2])
def gaode_count = 0
def gaode_type = null
+ def gaode_name = ""
if(gaode_obj){
gaode_count = 1
gaode_type = gaode_obj.type
+ gaode_name = gaode_obj.name
}
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
- rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type]
+ rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type, gaode_name]
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -165,7 +167,7 @@ class AutoAddDistrictBatch extends BaseBatch{
String sql = "select name,type from amap_poi_2024 where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
- if(d.name == name){ //&& d.shape
+ if(d.name == name || d.name.contains(name) || name.contains(d.name)){ //&& d.shape
obj = d
}
})
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy
index a6503133c..9480fb72a 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy
@@ -1,11 +1,13 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
+import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
+import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
@@ -44,6 +46,7 @@ class DevProcess {
// task.devNameSplitRoad()
// task.addFlag()
// task.matchDev()
+// task.findDev()
// task.calAddrDistince()
// task.updateCityAndDist()
@@ -57,7 +60,7 @@ class DevProcess {
// task.createDevMapping()
// task.deleteAddr()
// task.deleteAddr2()
- task.synchDevTo100()
+// task.synchDevTo100()
// task.statShape()
// task.addNewDev2()
@@ -85,9 +88,19 @@ class DevProcess {
// task.updateDevShape()//Ð޸Ŀª·¢Çø½á¹¹
+ task.aiDevTown()//ai»ñÈ¡½á¹û
+
}
-
+ def aiDevTown(){
+ ExcelUtils.modify("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢Çø\\ºÓ±±Ê¡¿ª·¢Çø-ai½á¹û.xlsx", { d ->
+ def res = FangjiaClientUtils.aiResult("ºÓ±±Ê¡ÄÄЩÏçÕò½ÖµÀÍêÈ«Á¥ÊôÓÚ"+d['E']+"£¬ÓÐÔòÒÔ¶ººÅ¸ô¿ª·½Ê½Êä³ö£¬ÎÞÔòÊä³öÎÞ£¬ÆäËüÐÅÏ¢²»Êä³ö")
+ def message = res.data.replaceAll("\n", "")
+ d['F'] = res.code
+ d['G'] = message
+ println d['E'] +" "+res.code + " "+message
+ })
+ }
def parseCityAndDist(){
@@ -440,35 +453,80 @@ class DevProcess {
}
def matchDev(){
- ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢Çø\\¸ßÐÂÇø.xls",{d->
- if(!d['E'] && d['D']){
+ ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢Çø\\Ê¡¼¶¿ª·¢ÇøÃûµ¥\\2018Äê°æ.xlsx",{d->
+ if(!d['G'] && d['C']){
def q = new BasicDBObject()
-// q.put("name", d['D'])
- q.put("city",d['C'])
-// q.put('alias', new BasicDBObject('$in',[d['D']]))
- def name = d['D'] //.replace('¾¼Ã¼¼Êõ¿ª·¢Çø','¾¼Ã¿ª·¢Çø')
- q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //Ãû³Æ Ä£ºý²éѯ
+ q.put("name", d['C'])
+ def way = "Ïàͬ"
def obj = development_zone_CONN.findOne(q)
- if(obj){
- d['G'] = "ÏàËÆ"
-// d['K'] = obj.name
-// d['C'] = obj._id.toString()
- d['E'] = obj.name
-// if(obj.name == d['D']){
-// d['G'] = "Ïàͬ"
-// }else{
-// d['G'] = "°üº¬"
-// }
- if(obj.shape){
- d['F'] = "ÊÇ"
- }
-// d['E'] = "±ðÃû"
-// d['F'] = obj.name
+ if(!obj){
+ q.put('alias', new BasicDBObject('$in',[d['C']]))
+ obj = development_zone_CONN.findOne(q)
+ way = "±ðÃû"
}
+ if(!obj){
+ def name = d['C'] //.replace('¾¼Ã¼¼Êõ¿ª·¢Çø','¾¼Ã¿ª·¢Çø')
+ q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //Ãû³Æ Ä£ºý²éѯ
+ obj = development_zone_CONN.findOne(q)
+ way = "Ä£ºý"
+ }
+ def allDevNames = []
+ if(!obj){
+ def query = new BasicDBObject()
+ def keys = new BasicDBObject()
+ keys.put("name", 1)
+ keys.put("alias", 1)
+ allDevNames = development_zone_CONN.find(query, keys).toList()
+ }
+ if(obj){
+ d['H'] = obj.name
+ d['G'] = way
+ }else{
+ if(allDevNames){
+ def maxScore = -100
+ def smilarObj = null
+ for(def dev : allDevNames){
+ def devNames = []
+ if(dev.alias){
+ devNames = dev.alias
+ }
+ devNames << dev.name
+ for(def aliasName : devNames){
+ def nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(d['C'], aliasName);
+ if(nameScore>maxScore){
+ maxScore = nameScore
+ smilarObj = dev
+ }
+ }
+ }
+ if(smilarObj){
+ d['H'] = smilarObj.name
+ d['G'] = "ÏàËÆµÃ·Ö"
+ d['I'] = maxScore
+ }
+ }
+
+ }
+
+
+
}
},0);
}
+
+ def findDev(){
+ ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢ÇøÍâ°ü\\¿ª·¢Çø-0423(161¸ö-Õã½).xlsx", { d ->
+ def q = new BasicDBObject()
+ q.put("name", d['B'])
+ def obj = development_zone_CONN.findOne(q)
+ if(obj){
+ d['C'] = obj.city
+ d['D'] = obj.district
+ }
+ })
+ }
+
def calAddrDistince(){
ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\ÆóÒµÔ°ÇøµØÖ·-ÐÂ.xlsx",{d->
if(d['M'] && d['N'] && d['Y'] && d['Z']){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy
index f7b23f7f6..ad2d5309a 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy
@@ -5,6 +5,8 @@ import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.yinjihaun.util.HttpClient
import com.mongodb.BasicDBObject
+import groovyx.net.http.ContentType
+import groovyx.net.http.RESTClient
import net.sf.json.JSONArray
import net.sf.json.JSONObject
@@ -423,25 +425,17 @@ class FangjiaClientUtils {
}
static def aiResult(content){
- def headers = [:]
- headers.put("accept", "application/json")
- headers.put("Content-Type", "application/json")
-
- def url = "http://didata-cloud-server.fangjia.com/chat/deepseek"
- try {
- Map params = new HashMap();
- params.put("content", content);
-
- String result = HttpClient.post(url, params, headers, "utf-8", true)
- println result
- if (result) {
- def json = JSONObject.fromObject(result)
- return json
- }
- } catch (Exception e) {
- println e.getMessage()
- }
- return null
+ def client = new RESTClient('http://didata-cloud-server.fangjia.com/')
+ def response = client.post(
+ path: '/chat/deepseek',
+ contentType: ContentType.JSON,
+ headers: [accept: 'application/json'],
+ body: [content: content]
+ )
+//
+// println "Response status: ${response.responseData}"
+// println "Response data: ${response.data}"
+ return response.data
}
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/DistrictLevelBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/DistrictLevelBatch.groovy
index e1cddfa83..2a4d4fe27 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/DistrictLevelBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/DistrictLevelBatch.groovy
@@ -8,25 +8,40 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
+
+import java.util.concurrent.CopyOnWriteArrayList
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicLong
+
/**
* ÕãÉ̱¾µØ»¯²¿Êðͬ²½½Å±¾-Âß¼ÐÞ¸Ä
*/
@BatchOwner(Watcher.GaoJuan)
class DistrictLevelBatch extends BaseBatch{
- public static final def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
- public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName2("fangjia", "district_analy", ZS_MONGODB_URL)
+// public static final def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
+// public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName2("fangjia", "district_analy", ZS_MONGODB_URL)
public static final zsDistLevelMap = [:]
-
+ public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
+ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
runBatch(new DistrictLevelBatch(), args);
}
@Override
public void process(Object args) {
+ def keys = new BasicDBObject("comm_id", 1)
+ keys.put("city", 1)
+ keys.put("region", 1)
+ keys.put("district_name", 1)
+ keys.put("status", 1)
+ keys.put("avg_price", 1)
+ keys.put("value_rate", 1)
+ def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject(),keys)
- def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject())
for(def distLevel : zsDistLevelCursor){
- zsDistLevelMap.put(distLevel._id.toString(), distLevel)
+ zsDistLevelMap.put(distLevel.comm_id.toString(), distLevel)
}
def citys = CommonDatautils.findAllCity();
@@ -44,6 +59,7 @@ class DistrictLevelBatch extends BaseBatch{
def addPriceData(dists, priceMap){
def newLists = []
for(def d : dists){
+
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
if(price){
d.avg_price = price;
@@ -100,9 +116,11 @@ class DistrictLevelBatch extends BaseBatch{
def saveOrUpdateDB(city, datas){
def zsDistLevels = []
- def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject("city",city))
+ def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject("city",city), new BasicDBObject("comm_id", 1))
for(def distLevel : zsDistLevelCursor){
- zsDistLevels << distLevel._id.toString()
+ if(distLevel.comm_id){
+ zsDistLevels << distLevel.comm_id.toString()
+ }
}
def newDataMap = [:]
@@ -113,17 +131,17 @@ class DistrictLevelBatch extends BaseBatch{
def udata = new BasicDBObject()
udata.put("city", d.city)
udata.put("region", d.region)
- udata.put("name", d.name)
- udata.put("level", d.level)
- udata.put("avg_price", d.avg_price)
+ udata.put("district_name", d.name)
+ udata.put("value_rate", d.level)
+// udata.put("avg_price", d.avg_price)
udata.put("status",0)
udata.put("update_date",new Date().getTime())
if(zsDistLevel){
- if(zsDistLevel.city!=d.city||zsDistLevel.region!=d.region||zsDistLevel.name!=d.name||zsDistLevel.level!=d.level||zsDistLevel.avg_price!=d.avg_price||zsDistLevel.status!=0){
+ if(zsDistLevel.city!=d.city||zsDistLevel.region!=d.region||zsDistLevel.district_name!=d.name||zsDistLevel.value_rate!=d.level||zsDistLevel.avg_price!=d.avg_price||zsDistLevel.status!=0){
ZS_LEVEL_CONN.update(new BasicDBObject("_id",zsDistLevel._id), new BasicDBObject('$set',udata))
}
}else{
- udata.put("_id", d._id.toString())
+ udata.put("comm_id", d._id.toString())
addDatas << new BasicDBObject(udata)
if(addDatas.size()%10000==0){
ZS_LEVEL_CONN.insert(addDatas);
@@ -134,16 +152,17 @@ class DistrictLevelBatch extends BaseBatch{
if(addDatas.size()>0){
ZS_LEVEL_CONN.insert(addDatas);
}
- if(zsDistLevels.size()>0){
- for(def zs_id : zsDistLevels){
- def newData = newDataMap.get(zs_id.toString());
- if(!newData){
- def udata = new BasicDBObject()
- udata.put("status",1)
- udata.put("update_date",new Date().getTime())
- ZS_LEVEL_CONN.update(new BasicDBObject("_id",zs_id), new BasicDBObject('$set',udata))
- }
- }
- }
+ //ÏȲ»É¾³ý£¬ÓÐЩû¼Û¸ñ
+// if(zsDistLevels.size()>0){
+// for(def zs_id : zsDistLevels){
+// def newData = newDataMap.get(zs_id.toString());
+// if(!newData){
+// def udata = new BasicDBObject()
+// udata.put("status",1)
+// udata.put("update_date",new Date().getTime())
+// ZS_LEVEL_CONN.update(new BasicDBObject("comm_id",zs_id), new BasicDBObject('$set',udata))
+// }
+// }
+// }
}
}