From 6a193baf42c70a205420a0cab4fb05c3fc3534c4 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Jul 2025 17:44:08 +0800 Subject: [PATCH] update --- projects/fangjia-script/pom.xml | 19 +++ .../ccbevaluation/CcbCommonUtils.groovy | 2 +- .../script/commData/CommonDatautils.groovy | 120 +++++++++++++++++- .../gj/data_handle/AddDevShapeData.groovy | 46 ++++--- .../data_handle/AutoAddDistrictBatch.groovy | 16 ++- .../script/gj/data_handle/DevProcess.groovy | 106 ++++++++++++---- .../gj/district/FangjiaClientUtils.groovy | 32 ++--- .../script/zsbank/DistrictLevelBatch.groovy | 65 ++++++---- 8 files changed, 310 insertions(+), 96 deletions(-) diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index ee869ba86..f398b19eb 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -555,6 +555,25 @@ 0.0.12 + + + org.codehaus.groovy.modules.http-builder + http-builder + 0.7.1 + + + + xmlbeans + xmlbeans + + + + xerces + xercesImpl + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index be5714bfd..4dd815364 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -443,7 +443,7 @@ class CcbCommonUtils { } // q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//Òò5ÔÂ1ÈÕÐÂÔö¹ý¶à£¬ËùÒÔÔÝÇÒÏÈÈ¥µô ´ËÅúÊý¾Ý¸ÄΪname_cat=13 -// q.put("name","ÐÂãþÈý´å") +// q.put("name","ÁúÔ´ÐdzÇ") // q.put("name_cat", 0); // q.put("name", new BasicDBObject('$in', ["ÖÐÐÅÌ©¸»ÓÖÒ»³ÇÒ»ÆÚ","ÖÐÐÅÌ©¸»ÓÖÒ»³Ç"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 17a82f2fc..a595aa2fe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -105,6 +105,8 @@ class CommonDatautils { public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com") public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com") + public static final DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) + public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { @@ -811,6 +813,8 @@ class CommonDatautils { // q.put("status",new BasicDBObject('$ne', 0)) q.put("cat", cat); q.put("city", city) +// q.put("loop_line_info",new BasicDBObject('$exists',false)) +// q.put("name","¾­Î³ÌìµØÔ°¶þÆÚ") DBObject keys = new BasicDBObject(); if(fields){ @@ -4142,9 +4146,13 @@ class CommonDatautils { QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true) QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) - .must(QueryBuilders.termQuery("region", region)) .must(QueryBuilders.termQuery("cat", cat)).must(date_range) + if(region){ + queryBuilder.must(QueryBuilders.termQuery("region", region)) + } + + if(name || alias){ if(!alias){ alias = [] @@ -4250,6 +4258,19 @@ class CommonDatautils { return rets; } + static def findAllNewPoiDatas(city, region, name, small_type){ + def rets = [] + def index= 0 + String sql = "select cityname,adname,name,address,location,type,id,parent from amap_poi_2024 where cityname='${city}'" + ElasticUtils.executeSqlQuery(sql, { map-> +// if(map.address && map.address.contains("Îå½Ç³¡")){ + index++ + rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y, "parent_id":map.parent] +// } + }) + return rets; + } + @@ -5644,4 +5665,101 @@ class CommonDatautils { }) return rets; } + + + static def findGaoDeDatas(city, region, name){ + String sql = "select name,type from amap_poi_2024 where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') limit 100" + def obj = null + ElasticUtils.executeSqlQuery(sql,{ d-> + if(d.name == name){ //&& d.shape + obj = d + } + }) + return obj; + } + + + static def findGaoDeDatas2(city, region, name){ + def header = [:] + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString() + String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_2024/_search" + def body = """ + { + "_source": { + "includes": ["cat", "address", "id","name", "township","location"] + }, + "query": { + "bool": { + "must": [ + { + "term": { + "city" : "${city}" + } + }, + { + "term": { + "district" : "${region}" + } + }, + { + "match": { + "name": {"query": "${name}","operator": "and"} + } + } + ] + } + } + } + """ + HttpRequest request = new HttpRequest() + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + def response = new JsonSlurper().parseText(res) +// if(name=="ºÓ±±ºÓ¼ä¾­¼Ã¿ª·¢Çø"){ +// println name +// } + def esObj = null + if(response && response.hits){ + def hits = response.hits.hits + if(hits){ + def d = hits[0] +// if(d._source.cat.contains("סլ")){ + def source = d._source + def newName = source.name + if(newName == name){ + esObj = source + }else if (newName.contains("¿ª·¢Çø") && name.contains("¿ª·¢Çø") && (newName.contains(name) || name.contains(newName))){ + esObj = source + }else if(newName.contains("Ô°Çø") && name.contains("Ô°Çø") && (newName.contains(name) || name.contains(newName))){ + esObj = source + }else if(newName.contains("ʾ·¶Çø") && name.contains("ʾ·¶Çø") && (newName.contains(name) || name.contains(newName))){ + esObj = source + }else{ + if(name.contains("¿ª·¢Çø")){ + name = name.replace("¹ú¼Ò¸ßм¼Êõ²úÒµ¿ª·¢Çø","").replace("¸ßм¼Êõ²úÒµ¿ª·¢Çø","").replace("¾­¼Ã¼¼Êõ¿ª·¢Çø","").replace("¾­¼Ã¿ª·¢Çø", "") + newName = newName.replace("¹ú¼Ò¸ßм¼Êõ²úÒµ¿ª·¢Çø","").replace("¸ßм¼Êõ²úÒµ¿ª·¢Çø","").replace("¾­¼Ã¼¼Êõ¿ª·¢Çø","").replace("¾­¼Ã¿ª·¢Çø", "") + } + if(newName == name){ + esObj = source + } + } +// } + } + } + return esObj + } + + static def findBaiduDatas(city, district, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("district", district) + q.put("name", name) + q.put('std_tag', '·¿µØ²ú;×¡Õ¬Çø') + + def dbcursor = BAIDU_POI_COLL.find(q) + def address = null + for(def d : dbcursor){ + address = d.addr + } + return address; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy index a2d1c56b8..a605dcc02 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy @@ -189,7 +189,7 @@ class AddDevShapeData { def run(){ - def directoryPath = 'D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢ÇøÍâ°ü\\ÁÙʱ-û¸¶·Ñ\\' + def directoryPath = 'D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢ÇøÍâ°ü\\623Î÷°²' def dir = new File(directoryPath) if (dir.isDirectory()) { dir.eachFileRecurse(FileType.FILES) { file -> @@ -203,7 +203,7 @@ class AddDevShapeData { def addDevInfo(file){ ExcelUtils.multiThreadingModify(file, {d-> -// if(!d['I']){ + if(!d['I']){ def name = d['E'] def q = new BasicDBObject() q.put('_id',new ObjectId(d['A'])) @@ -218,24 +218,26 @@ class AddDevShapeData { if(obj.shape){ d['T'] = "ÓÐ" } -// if(obj.m_date){ -// d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd") -// } + if(obj.m_date){ + d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd") + } -// if(devAreaMap){ -// devAreaMap.each {k,v-> -// if(obj.name && k && obj.name.contains(k)){ -// d['M'] = v.area_info -// d['N'] = v.count -// d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd") -// } -// } -// -// }else{ -// println name+" ¿ÕÖµ" -// } + if(devAreaMap){ + devAreaMap.each {k,v-> + if(obj.name && k && obj.name.contains(k)){ + d['M'] = v.area_info + d['N'] = v.count + d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd") + } + } + + }else{ + println name+" ¿ÕÖµ" + } + }else{ + d['S'] = "ɾ³ý" } -// } + } }) } @@ -269,13 +271,15 @@ class AddDevShapeData { DEVELOPMENT_ZONE_43_COLL.save(obj) }else{ udata.put("shape", resultMap) -// udata.put("m_date", new Date()) + udata.put("m_date", new Date()) udata.put("bd_shape",bd_resultMap) udata.put("operate_result" , "ÒÑ´¦Àí") udata.put("operator", "Íâ°ü") DEVELOPMENT_ZONE_43_COLL.update(q, new BasicDBObject('$set', udata), false, false) } } + }else{ + println "¡¾¿ª·¢Çø¡¿"+retMap.name +" ¿âÖÐδ²éµ½" } } } @@ -296,7 +300,7 @@ class AddDevShapeData { areaMap.count = coordinates.size() def areas = [] -// if(name.contains("½­ËÕ³£Êì×ۺϱ£Ë°Çø")){ + if(name.contains("ÉÂÎ÷À¶Ì﹤ҵ԰")){ for(def pointList : coordinates){ index ++ def gaode_coordinates = [] @@ -339,7 +343,7 @@ class AddDevShapeData { } areaMap.area_info = areas.join("\n") -// } + } }else{ println file +" Óжà¸ö" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy index 9d4bc3bb9..1cdff119f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy @@ -31,16 +31,16 @@ class AutoAddDistrictBatch extends BaseBatch{ def mcaDistMap = CommonDatautils.queryDistNameCityByMapping() static main(args) { -// runBatch(new AutoAddDistrictBatch(), args) + runBatch(new AutoAddDistrictBatch(), args) - def task = new AutoAddDistrictBatch(); - task.checkMap() +// def task = new AutoAddDistrictBatch(); +// task.checkMap() } @Override void process(Object args) { //²éÕÒµÚÈý·½Æ¥ÅäûÓÐÆ¥ÅäÉϵÄÐ¡Çø - def citys = CommonDatautils.findAllCity() + def citys = ["ÈýÃ÷"]//CommonDatautils.findAllCity() def list = [] list << ["³ÇÊÐ", "ÇøÓò", "Ãû³Æ", "³É½»Á¿", "·¿Ô´Á¿", "°Ù¶ÈµØÍ¼", "°Ù¶ÈÀàÐÍ","¸ßµÂµØÍ¼", "¸ßµÂÀàÐÍ"] for(def c : citys){ @@ -52,7 +52,7 @@ class AutoAddDistrictBatch extends BaseBatch{ def fileName = "/home/goojia/gj/add_dist_info.xlsx" if (CommonDatautils.isWindow()) { - fileName = "D:\\Êý¾Ý\\add_dist_info_1.xlsx" + fileName = "D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\Ð¡Çø\\add_dist_info_1.xlsx" } ExcelUtils.write(fileName, list, "´ýÈë¿âÊý¾Ý") @@ -102,12 +102,14 @@ class AutoAddDistrictBatch extends BaseBatch{ def gaode_obj = findGaodeCount(arrs[0], arrs[1], arrs[2]) def gaode_count = 0 def gaode_type = null + def gaode_name = "" if(gaode_obj){ gaode_count = 1 gaode_type = gaode_obj.type + gaode_name = gaode_obj.name } println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count - rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type] + rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type, gaode_name] } catch (Exception e) { e.printStackTrace() } finally { @@ -165,7 +167,7 @@ class AutoAddDistrictBatch extends BaseBatch{ String sql = "select name,type from amap_poi_2024 where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')" def obj = null ElasticUtils.executeSqlQuery(sql,{ d-> - if(d.name == name){ //&& d.shape + if(d.name == name || d.name.contains(name) || name.contains(d.name)){ //&& d.shape obj = d } }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index a6503133c..9480fb72a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -1,11 +1,13 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.RegionData +import com.fangjia.core.util.DataUtil import com.fangjia.core.util.GeoUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils @@ -44,6 +46,7 @@ class DevProcess { // task.devNameSplitRoad() // task.addFlag() // task.matchDev() +// task.findDev() // task.calAddrDistince() // task.updateCityAndDist() @@ -57,7 +60,7 @@ class DevProcess { // task.createDevMapping() // task.deleteAddr() // task.deleteAddr2() - task.synchDevTo100() +// task.synchDevTo100() // task.statShape() // task.addNewDev2() @@ -85,9 +88,19 @@ class DevProcess { // task.updateDevShape()//Ð޸Ŀª·¢Çø½á¹¹ + task.aiDevTown()//ai»ñÈ¡½á¹û + } - + def aiDevTown(){ + ExcelUtils.modify("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢Çø\\ºÓ±±Ê¡¿ª·¢Çø-ai½á¹û.xlsx", { d -> + def res = FangjiaClientUtils.aiResult("ºÓ±±Ê¡ÄÄЩÏçÕò½ÖµÀÍêÈ«Á¥ÊôÓÚ"+d['E']+"£¬ÓÐÔòÒÔ¶ººÅ¸ô¿ª·½Ê½Êä³ö£¬ÎÞÔòÊä³öÎÞ£¬ÆäËüÐÅÏ¢²»Êä³ö") + def message = res.data.replaceAll("\n", "") + d['F'] = res.code + d['G'] = message + println d['E'] +" "+res.code + " "+message + }) + } def parseCityAndDist(){ @@ -440,35 +453,80 @@ class DevProcess { } def matchDev(){ - ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢Çø\\¸ßÐÂÇø.xls",{d-> - if(!d['E'] && d['D']){ + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢Çø\\Ê¡¼¶¿ª·¢ÇøÃûµ¥\\2018Äê°æ.xlsx",{d-> + if(!d['G'] && d['C']){ def q = new BasicDBObject() -// q.put("name", d['D']) - q.put("city",d['C']) -// q.put('alias', new BasicDBObject('$in',[d['D']])) - def name = d['D'] //.replace('¾­¼Ã¼¼Êõ¿ª·¢Çø','¾­¼Ã¿ª·¢Çø') - q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //Ãû³Æ Ä£ºý²éѯ + q.put("name", d['C']) + def way = "Ïàͬ" def obj = development_zone_CONN.findOne(q) - if(obj){ - d['G'] = "ÏàËÆ" -// d['K'] = obj.name -// d['C'] = obj._id.toString() - d['E'] = obj.name -// if(obj.name == d['D']){ -// d['G'] = "Ïàͬ" -// }else{ -// d['G'] = "°üº¬" -// } - if(obj.shape){ - d['F'] = "ÊÇ" - } -// d['E'] = "±ðÃû" -// d['F'] = obj.name + if(!obj){ + q.put('alias', new BasicDBObject('$in',[d['C']])) + obj = development_zone_CONN.findOne(q) + way = "±ðÃû" } + if(!obj){ + def name = d['C'] //.replace('¾­¼Ã¼¼Êõ¿ª·¢Çø','¾­¼Ã¿ª·¢Çø') + q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //Ãû³Æ Ä£ºý²éѯ + obj = development_zone_CONN.findOne(q) + way = "Ä£ºý" + } + def allDevNames = [] + if(!obj){ + def query = new BasicDBObject() + def keys = new BasicDBObject() + keys.put("name", 1) + keys.put("alias", 1) + allDevNames = development_zone_CONN.find(query, keys).toList() + } + if(obj){ + d['H'] = obj.name + d['G'] = way + }else{ + if(allDevNames){ + def maxScore = -100 + def smilarObj = null + for(def dev : allDevNames){ + def devNames = [] + if(dev.alias){ + devNames = dev.alias + } + devNames << dev.name + for(def aliasName : devNames){ + def nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(d['C'], aliasName); + if(nameScore>maxScore){ + maxScore = nameScore + smilarObj = dev + } + } + } + if(smilarObj){ + d['H'] = smilarObj.name + d['G'] = "ÏàËÆµÃ·Ö" + d['I'] = maxScore + } + } + + } + + + } },0); } + + def findDev(){ + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢ÇøÍâ°ü\\¿ª·¢Çø-0423(161¸ö-Õã½­).xlsx", { d -> + def q = new BasicDBObject() + q.put("name", d['B']) + def obj = development_zone_CONN.findOne(q) + if(obj){ + d['C'] = obj.city + d['D'] = obj.district + } + }) + } + def calAddrDistince(){ ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\ÆóÒµÔ°ÇøµØÖ·-ÐÂ.xlsx",{d-> if(d['M'] && d['N'] && d['Y'] && d['Z']){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index f7b23f7f6..ad2d5309a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -5,6 +5,8 @@ import com.fangjia.script.temp.shenjian.util.HttpUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.yinjihaun.util.HttpClient import com.mongodb.BasicDBObject +import groovyx.net.http.ContentType +import groovyx.net.http.RESTClient import net.sf.json.JSONArray import net.sf.json.JSONObject @@ -423,25 +425,17 @@ class FangjiaClientUtils { } static def aiResult(content){ - def headers = [:] - headers.put("accept", "application/json") - headers.put("Content-Type", "application/json") - - def url = "http://didata-cloud-server.fangjia.com/chat/deepseek" - try { - Map params = new HashMap(); - params.put("content", content); - - String result = HttpClient.post(url, params, headers, "utf-8", true) - println result - if (result) { - def json = JSONObject.fromObject(result) - return json - } - } catch (Exception e) { - println e.getMessage() - } - return null + def client = new RESTClient('http://didata-cloud-server.fangjia.com/') + def response = client.post( + path: '/chat/deepseek', + contentType: ContentType.JSON, + headers: [accept: 'application/json'], + body: [content: content] + ) +// +// println "Response status: ${response.responseData}" +// println "Response data: ${response.data}" + return response.data } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/DistrictLevelBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/DistrictLevelBatch.groovy index e1cddfa83..2a4d4fe27 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/DistrictLevelBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/DistrictLevelBatch.groovy @@ -8,25 +8,40 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + /** * ÕãÉ̱¾µØ»¯²¿Êðͬ²½½Å±¾-Âß¼­ÐÞ¸Ä */ @BatchOwner(Watcher.GaoJuan) class DistrictLevelBatch extends BaseBatch{ - public static final def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; - public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName2("fangjia", "district_analy", ZS_MONGODB_URL) +// public static final def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; +// public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName2("fangjia", "district_analy", ZS_MONGODB_URL) public static final zsDistLevelMap = [:] - + public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com") + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) static main(args) { runBatch(new DistrictLevelBatch(), args); } @Override public void process(Object args) { + def keys = new BasicDBObject("comm_id", 1) + keys.put("city", 1) + keys.put("region", 1) + keys.put("district_name", 1) + keys.put("status", 1) + keys.put("avg_price", 1) + keys.put("value_rate", 1) + def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject(),keys) - def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject()) for(def distLevel : zsDistLevelCursor){ - zsDistLevelMap.put(distLevel._id.toString(), distLevel) + zsDistLevelMap.put(distLevel.comm_id.toString(), distLevel) } def citys = CommonDatautils.findAllCity(); @@ -44,6 +59,7 @@ class DistrictLevelBatch extends BaseBatch{ def addPriceData(dists, priceMap){ def newLists = [] for(def d : dists){ + def price = priceMap.get(d.city+"##"+d.region+"##"+d.name); if(price){ d.avg_price = price; @@ -100,9 +116,11 @@ class DistrictLevelBatch extends BaseBatch{ def saveOrUpdateDB(city, datas){ def zsDistLevels = [] - def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject("city",city)) + def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject("city",city), new BasicDBObject("comm_id", 1)) for(def distLevel : zsDistLevelCursor){ - zsDistLevels << distLevel._id.toString() + if(distLevel.comm_id){ + zsDistLevels << distLevel.comm_id.toString() + } } def newDataMap = [:] @@ -113,17 +131,17 @@ class DistrictLevelBatch extends BaseBatch{ def udata = new BasicDBObject() udata.put("city", d.city) udata.put("region", d.region) - udata.put("name", d.name) - udata.put("level", d.level) - udata.put("avg_price", d.avg_price) + udata.put("district_name", d.name) + udata.put("value_rate", d.level) +// udata.put("avg_price", d.avg_price) udata.put("status",0) udata.put("update_date",new Date().getTime()) if(zsDistLevel){ - if(zsDistLevel.city!=d.city||zsDistLevel.region!=d.region||zsDistLevel.name!=d.name||zsDistLevel.level!=d.level||zsDistLevel.avg_price!=d.avg_price||zsDistLevel.status!=0){ + if(zsDistLevel.city!=d.city||zsDistLevel.region!=d.region||zsDistLevel.district_name!=d.name||zsDistLevel.value_rate!=d.level||zsDistLevel.avg_price!=d.avg_price||zsDistLevel.status!=0){ ZS_LEVEL_CONN.update(new BasicDBObject("_id",zsDistLevel._id), new BasicDBObject('$set',udata)) } }else{ - udata.put("_id", d._id.toString()) + udata.put("comm_id", d._id.toString()) addDatas << new BasicDBObject(udata) if(addDatas.size()%10000==0){ ZS_LEVEL_CONN.insert(addDatas); @@ -134,16 +152,17 @@ class DistrictLevelBatch extends BaseBatch{ if(addDatas.size()>0){ ZS_LEVEL_CONN.insert(addDatas); } - if(zsDistLevels.size()>0){ - for(def zs_id : zsDistLevels){ - def newData = newDataMap.get(zs_id.toString()); - if(!newData){ - def udata = new BasicDBObject() - udata.put("status",1) - udata.put("update_date",new Date().getTime()) - ZS_LEVEL_CONN.update(new BasicDBObject("_id",zs_id), new BasicDBObject('$set',udata)) - } - } - } + //ÏȲ»É¾³ý£¬ÓÐЩû¼Û¸ñ +// if(zsDistLevels.size()>0){ +// for(def zs_id : zsDistLevels){ +// def newData = newDataMap.get(zs_id.toString()); +// if(!newData){ +// def udata = new BasicDBObject() +// udata.put("status",1) +// udata.put("update_date",new Date().getTime()) +// ZS_LEVEL_CONN.update(new BasicDBObject("comm_id",zs_id), new BasicDBObject('$set',udata)) +// } +// } +// } } }