Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
+1
-1
@@ -24,7 +24,7 @@ class AddThirdPriceSample {
|
||||
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
|
||||
if(dist){
|
||||
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407)
|
||||
def udata = new BasicDBObject("zhongyuan_esf_price", d['C'].toString().toDouble().toInteger())
|
||||
def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
|
||||
|
||||
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
|
||||
}else{
|
||||
|
||||
+6
-4
@@ -68,10 +68,10 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
//664a22000000000000000000 5-20
|
||||
//664b73800000000000000000 5-21
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def citys =CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
println "curDate:"+curDate
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
@@ -720,7 +720,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
if(!t.match_id){
|
||||
def region = RegionData.formatRegion(t.city, t.region)
|
||||
def rMap = null
|
||||
if(region) {
|
||||
if(region || (!region && ["澄迈"].contains(t.city))) {
|
||||
if(t.house_use && t.house_use.contains("商住")){
|
||||
if(!t.name.contains("商住")){
|
||||
t.name = t.name +"商住楼"
|
||||
@@ -744,7 +744,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}else if(rMap && rMap.address && address && rMap.region!=region && address.contains(rMap.address)){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}else {
|
||||
}else if (rMap && ["澄迈"].contains(rMap.city)){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}else{
|
||||
if (rMap) {
|
||||
println "区域不同"
|
||||
}
|
||||
|
||||
+12
-2
@@ -260,6 +260,9 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
if(third.get("fccs_esf_price")){
|
||||
price_map.fccs_avg_price = third.get("fccs_esf_price")
|
||||
}
|
||||
if(third.get("csjj_esf_price")){
|
||||
price_map.csjj_avg_price = third.get("csjj_esf_price")
|
||||
}
|
||||
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
allRows << dataMap
|
||||
}
|
||||
@@ -308,6 +311,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
web = "我爱我家"
|
||||
}else if(k=="fccs_avg_price"){
|
||||
web = "房产超市"
|
||||
}else if(k=="csjj_avg_price"){
|
||||
web = "城市居家"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
@@ -525,6 +530,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
q.put("city", city)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
|
||||
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
@@ -593,6 +599,10 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
|
||||
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("csjj_esf_price") && db.get("csjj_esf_price") && db.get("csjj_esf_price")>3000){
|
||||
dataMap.put("csjj_esf_price", db.get("csjj_esf_price"))
|
||||
}
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
@@ -600,8 +610,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def resultMap = [:]
|
||||
map.each {key, value->
|
||||
if(value && value.size()>1){
|
||||
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){
|
||||
if(value && value.size()>1){ //csjj_esf_price 国信达成交
|
||||
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price && !value.csjj_esf_price){
|
||||
def price = statMap.get(key)
|
||||
if(price){
|
||||
def priceMap = [:]
|
||||
|
||||
+1
-1
@@ -4746,7 +4746,7 @@ class CommonDatautils {
|
||||
townMap.put(key, list)
|
||||
}
|
||||
|
||||
println "乡镇街道 加载完成"
|
||||
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
return townMap
|
||||
}
|
||||
|
||||
|
||||
@@ -15,13 +15,16 @@ import org.apache.commons.lang.StringUtils
|
||||
class OpenApiUtils {
|
||||
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag, def suggestName){
|
||||
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||
if(preciseLevelflag){
|
||||
url = url +"&preciseLevel=strict"
|
||||
}
|
||||
if(suggestName){
|
||||
url = url +"&suggestName="+suggestName
|
||||
}
|
||||
if(suggestDistrict){
|
||||
url = url +"&suggestDistrict="+suggestDistrict
|
||||
}
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
def rMap = [:]
|
||||
if(content){
|
||||
|
||||
+95
-19
@@ -192,42 +192,112 @@ class DevProcess {
|
||||
}
|
||||
}
|
||||
|
||||
def findDev(city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", name)
|
||||
q.put("city", city)
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
def findDev( name){
|
||||
def obj = findDevByName(name)
|
||||
if(!obj){
|
||||
obj = findDevByAlias(name)
|
||||
if(obj){
|
||||
obj.match_way = "别名"
|
||||
}
|
||||
}else{
|
||||
obj.match_way = "相同"
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
def findDebBySimlar(def devs, def name, def city){
|
||||
name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
|
||||
def last = null
|
||||
for(def d : devs){
|
||||
def alias1 = null
|
||||
if(!d.alias){
|
||||
alias1 = []
|
||||
}else{
|
||||
alias1 = d.alias
|
||||
}
|
||||
if(!alias1.contains(d.name)){
|
||||
alias1 << d.name
|
||||
}
|
||||
if(alias1){
|
||||
try {
|
||||
for(def n : alias1){
|
||||
n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
|
||||
if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){
|
||||
last = d
|
||||
break
|
||||
}
|
||||
}
|
||||
}catch(exception){
|
||||
println alias1
|
||||
}
|
||||
|
||||
}
|
||||
if(last){
|
||||
break
|
||||
}
|
||||
}
|
||||
return last
|
||||
}
|
||||
|
||||
def findDevByName(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", name)
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevByAlias(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('alias', new BasicDBObject('$in',[name]))
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevBySimlar(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevByPark(prov, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put("type", "园区")
|
||||
return dev_mapping_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def devSplitRoad(){
|
||||
def i = 0;
|
||||
def i = 1;
|
||||
while(i<20){
|
||||
def fileName = "/home/goojia/gj/qzd-${i}.xlsx"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\数据建设\\开发区\\福建省.xlsx"
|
||||
fileName = "D:\\数据\\数据建设\\开发区\\新跑省份\\河北省\\test-${i}.xlsx"
|
||||
}
|
||||
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['E'] && !d['G']){
|
||||
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
|
||||
if(d['C'] && !d['F']){
|
||||
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
|
||||
if(jsonObject1) {
|
||||
if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") {
|
||||
def devName = jsonObject1.get("dev")
|
||||
def obj = findDev(d['B'], devName)
|
||||
def obj = findDev(devName)
|
||||
d['F'] = devName
|
||||
if(jsonObject1.containsKey("detail") && jsonObject1.detail.dev_confi!='null'){
|
||||
d['S'] = jsonObject1.detail.dev_confi
|
||||
}
|
||||
if (obj) {
|
||||
d['Q'] = obj._id.toString()
|
||||
d['R'] = obj.name
|
||||
d['O'] = obj._id.toString()
|
||||
d['P'] = obj.name
|
||||
d['U'] = obj.match_way
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(d['C'] && !d['S']) {
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
|
||||
if(d['E'] && !d['M']) {
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if (jsonObject.get("province") && jsonObject.get("province") != "null") {
|
||||
@@ -256,7 +326,7 @@ class DevProcess {
|
||||
d['N'] = "信息不完整"
|
||||
}
|
||||
|
||||
if (!d['N'] && !d['C'].contains(road)) {
|
||||
if (!d['N'] && d['C'] && !d['C'].contains(road)) {
|
||||
d['N'] = "原地址不含路"
|
||||
}
|
||||
|
||||
@@ -276,17 +346,23 @@ class DevProcess {
|
||||
|
||||
if (jsonObject.get("dev") && jsonObject.get("dev") != "null") {
|
||||
def devName = jsonObject.get("dev")
|
||||
def obj = findDev(d['B'], devName)
|
||||
def obj = findDev(devName)
|
||||
d['M'] = devName
|
||||
if (obj) {
|
||||
d['S'] = obj._id.toString()
|
||||
d['T'] = obj.name
|
||||
d['Q'] = obj._id.toString()
|
||||
d['R'] = obj.name
|
||||
d['V'] = obj.match_way
|
||||
}
|
||||
if(jsonObject.containsKey("detail") && jsonObject.detail.dev_confi!='null'){
|
||||
d['T'] = jsonObject.detail.dev_confi
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
i++
|
||||
|
||||
}
|
||||
i++
|
||||
println fileName+" 完成"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+32
-7
@@ -33,13 +33,13 @@ class MinglujiData {
|
||||
def prov = "Õã½Ê¡"
|
||||
def city = "¼ÎÐË"
|
||||
// exportDatas(city)
|
||||
// matchDev(prov, city)
|
||||
matchDev(prov, city)
|
||||
matchCommunity(city)
|
||||
|
||||
// tempMatchDev(prov)
|
||||
|
||||
|
||||
// saveMljData()
|
||||
|
||||
fiveLevelSplit(city)
|
||||
}
|
||||
def exportDatas(city){
|
||||
def datas = findMinglujiDatas(city)
|
||||
@@ -113,7 +113,7 @@ class MinglujiData {
|
||||
})
|
||||
}
|
||||
i++
|
||||
println fileName +" 完成"
|
||||
println fileName +" 完成(匹配开发区不使用接口)"
|
||||
}
|
||||
|
||||
}
|
||||
@@ -196,15 +196,40 @@ class MinglujiData {
|
||||
}
|
||||
}
|
||||
}, 0)
|
||||
println fileName +" 远行完成"
|
||||
println fileName +" 运行完成"
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def fiveLevelSplit(city){
|
||||
def i = 1;
|
||||
while(i<20) {
|
||||
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\名录集\\mlj_${i}.xlsx"
|
||||
}
|
||||
|
||||
File file = new File(fileName);
|
||||
if (file.exists()) {
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
if(d['J'] && !d['X']){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatFivelevel(d['B'], d['J'])
|
||||
if(jsonObject) {
|
||||
if (jsonObject.get("address") && jsonObject.get("address") != "null") {
|
||||
d['X'] = jsonObject.get("address")
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
def saveMljData(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库数据.xlsx",{d->
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库2.xlsx",{d->
|
||||
def obj = new BasicDBObject()
|
||||
obj.put("province", d['A'])
|
||||
obj.put("city", d['B'])
|
||||
@@ -213,7 +238,7 @@ class MinglujiData {
|
||||
obj.put("address", d['E'])
|
||||
obj.put("dev_name", d['F'])
|
||||
obj.put("committee", d['G'])
|
||||
obj.put("street", d['H'])
|
||||
obj.put("township", d['H'])
|
||||
company_100_CONN.save(obj)
|
||||
})
|
||||
}
|
||||
|
||||
+17
-7
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.addrepo.InvokeHttpRequest
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
@@ -335,7 +336,7 @@ class FangjiaClientUtils {
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
// def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.100:32080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
@@ -352,18 +353,27 @@ class FangjiaClientUtils {
|
||||
public static def formatNinelevel(city, address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true"
|
||||
def url = null
|
||||
address = address.replace("{","")
|
||||
int retryCount = 0
|
||||
while (retryCount < 3) {
|
||||
url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
if (content) {
|
||||
break
|
||||
}
|
||||
retryCount++
|
||||
}
|
||||
if (!content) {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
// println jsonObj
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
println address +" "+content+" "+e.getMessage()
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
+4
-3
@@ -90,6 +90,7 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
println "txt Éú³É " + rows.size()
|
||||
saveToPgSql(rows)
|
||||
|
||||
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
|
||||
@@ -356,7 +357,7 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
|
||||
def converData(def trade, def cityCodeDatas, def distCodeMap){
|
||||
def ret = []
|
||||
def data = distCodeMap.get(trade.region)
|
||||
def data = distCodeMap.get(trade.match_region)
|
||||
if(!data){
|
||||
println trade.city +" "+trade.region+" 没有找到对应的区域"
|
||||
return ret
|
||||
@@ -580,8 +581,8 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def getYouDaData(statMap){
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def q = new BasicDBObject()
|
||||
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ class QiliangTest {
|
||||
// task.run()
|
||||
|
||||
def data = [
|
||||
'address': '湖州市南太湖高新技术产业园区大港村陈家斗80号',
|
||||
'address': '浙江省湖州市长兴县煤山镇工业园区山鹰大道7号',
|
||||
'type': '1' //1:对公 0:对私
|
||||
]
|
||||
def s = task.getProxyResult(data);
|
||||
|
||||
+105
-44
@@ -10,6 +10,7 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -32,6 +33,7 @@ import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
@@ -126,7 +128,11 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
}
|
||||
|
||||
def sysnchVillage(){
|
||||
def buducitys = [ '临汾市', '临沂市', '临沧市', '临高县', '丹东市',
|
||||
def buducitys = [ "宁波市","丽水市","杭州市","温州市","嘉兴市","台州市","绍兴市","金华市","衢州市",'茂名市',
|
||||
'荆州市', '荆门市', '莆田市','苏州市','菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市',
|
||||
'衢州市', '襄阳市', '西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市',
|
||||
'辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市', '中山市', '临夏回族自治州',
|
||||
'临汾市', '临沂市', '临沧市', '临高县', '丹东市','邵阳市', '郑州市', '郴州市', '鄂尔多斯市', '鄂州市', '酒泉市',
|
||||
'丽水市', '丽江市', '乌兰察布市', '乌海市', '乌鲁木齐市', '乐东黎族自治县', '乐山市', '九江市',
|
||||
'云浮市', '五家渠市', '五指山市', '亳州市', '仙桃市', '伊春市', '伊犁哈萨克自治州', '佛山市',
|
||||
'佳木斯市', '保亭黎族苗族自治县', '保定市', '保山市', '信阳市', '儋州市', '克孜勒苏柯尔克孜自治州',
|
||||
@@ -136,7 +142,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
'吕梁市', '吴忠市', '周口市', '呼伦贝尔市', '呼和浩特市', '和田地区', '咸宁市', '咸阳市',
|
||||
'哈密地区', '哈尔滨市', '唐山市', '商丘市', '商洛市', '喀什地区', '嘉兴市', '嘉峪关市', '四平市',
|
||||
'固原市', '图木舒克市', '塔城地区', '大兴安岭地区', '大同市', '大庆市', '大理白族自治州', '大连市',
|
||||
'天水市', '天津市', '天门市', '太原市', '威海市', '娄底市', '孝感市', '宁德市', '宁波市',
|
||||
'天水市', '天津市', '天门市', '太原市', '威海市', '娄底市', '孝感市', '宁德市',
|
||||
'安庆市', '安康市', '安阳市', '安顺市', '定安县', '定西市', '宜宾市', '宜昌市', '宜春市',
|
||||
'宝鸡市', '宣城市', '宿州市', '宿迁市', '屯昌县', '山南地区', '岳阳市', '崇左市', '巴中市',
|
||||
'巴彦淖尔市', '巴音郭楞蒙古自治州', '常州市', '常德市', '平凉市', '平顶山市', '广元市', '广安市',
|
||||
@@ -149,17 +155,12 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
'梅州市', '梧州市', '楚雄彝族自治州', '榆林市', '武威市', '武汉市', '毕节地区', '永州市', '汉中市',
|
||||
'汕头市', '汕尾市', '江门市', '池州市', '沈阳市', '沧州市', '河池市', '河源市', '泉州市',
|
||||
'泰安市', '泰州市', '泸州市', '洛阳市', '济南市', '济宁市', '济源市', '海东地区', '海口市',
|
||||
'淄博市', '淮北市', '淮南市', '淮安市', '深圳市', '清远市', '温州市', '渭南市', '湖州市',
|
||||
'淄博市', '淮北市', '淮南市', '淮安市', '深圳市', '清远市', '温州市', '渭南市',
|
||||
'湘潭市', '湘西土家族苗族自治州', '湛江市', '滁州市', '滨州市', '漯河市', '漳州市', '潍坊市',
|
||||
'潜江市', '潮州市', '澄迈县', '濮阳市', '烟台市', '焦作市', '牡丹江市', '玉林市', '玉溪市',
|
||||
'珠海市', '琼中黎族苗族自治县', '琼海市', '甘南藏族自治州', '白城市', '白山市', '白沙黎族自治县',
|
||||
'白银市', '百色市', '益阳市', '盐城市', '盘锦市', '眉山市', '石嘴山市', '石家庄市', '石河子市',
|
||||
'神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绍兴市', '绥化市', '绵阳市', '聊城市',
|
||||
'肇庆市', '自贡市', '舟山市', '芜湖市', '苏州市', '茂名市', '荆州市', '荆门市', '莆田市',
|
||||
'菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市', '衢州市', '襄阳市',
|
||||
'西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市',
|
||||
'辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市',
|
||||
'邢台市', '那曲地区', '邯郸市', '邵阳市', '郑州市', '郴州市', '鄂尔多斯市', '鄂州市', '酒泉市',
|
||||
'神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绥化市', '邢台市', '那曲地区', '邯郸市',
|
||||
'重庆市', '金华市', '金昌市', '钦州市', '铁岭市', '铜仁地区', '铜川市', '铜陵市', '银川市',
|
||||
'锡林郭勒盟', '锦州市', '镇江市', '长春市', '长沙市', '长治市', '阜新市', '阜阳市', '防城港市',
|
||||
'阳江市', '阳泉市', '阿克苏地区', '阿勒泰地区', '阿拉善盟', '阿拉尔市', '阿里地区', '陇南市',
|
||||
@@ -167,50 +168,55 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
'鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市',
|
||||
'黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市']
|
||||
|
||||
//"丽水市","杭州市",
|
||||
def runbuducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市",'七台河市', '上海市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市',
|
||||
'东营市', '中卫市', '中山市', '临夏回族自治州']
|
||||
def runbuducitys = ["舟山市","湖州市"]
|
||||
for(def city : buducitys){
|
||||
if(runbuducitys.contains(city)){
|
||||
continue
|
||||
}
|
||||
def pois = findAllVillagePoiMap(city)
|
||||
println "百度loading finish: "+pois.size()+" "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
def rows = createVillageData(city, pois)
|
||||
|
||||
|
||||
save(city, rows)
|
||||
// saveToMG(city, rows)
|
||||
println city +" "+pois.size()+" add:"+rows.size()
|
||||
println city +" "+pois.size()+" add:"+rows.size()+" "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def createVillageData(city, pois){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def keyRets = new CopyOnWriteArrayList()
|
||||
def rets = new ConcurrentHashMap()
|
||||
//def keyRets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(pois.size())
|
||||
// def townMap = findTownRange(city)
|
||||
def townMap = CommonDatautils.queryTownByCity(city)
|
||||
def poiKeysFilter = findEsPoiByCity(city)
|
||||
def esMap = findEsPoiByCity(city)
|
||||
def nameKeys = esMap.keySet()
|
||||
println "Es 加载完成:"+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
for(def i=0;i<pois.size();i++){
|
||||
def p = pois[i]
|
||||
final def p = pois[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
// def obj = MAPPOI_CONN.findOne(new BasicDBObject("name", p.name).append("district", p.district).append("city", p.city))
|
||||
def key = p.city+"##"+p.district+"##"+p.name
|
||||
if(!poiKeysFilter.mightContain(key)){
|
||||
def key = p.city+"##"+p.district+"##"+p.name.replace("-","").replace("·","")
|
||||
if(!nameKeys.contains(key)){
|
||||
def prov = ProvinceData.getProvince(p.city.replace("市",""))
|
||||
|
||||
def ret = createVillageData(prov, p, townMap)
|
||||
if(!keyRets.contains(ret.city+"##"+ret.district+"##"+ret.name)){
|
||||
keyRets.addAll(ret)
|
||||
rets << ret
|
||||
if(ret){
|
||||
def simlarObj = null
|
||||
if(esMap){
|
||||
simlarObj = findSimlarPoi(ret, esMap)
|
||||
}
|
||||
if(!simlarObj){
|
||||
def newKey = ret.city+"##"+ret.district+"##"+ret.name.replace("-","").replace("·","")
|
||||
if(!rets.containsKey(newKey)){
|
||||
rets.put(newKey, ret)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
@@ -223,21 +229,73 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
return rets
|
||||
}
|
||||
|
||||
def findSimlarPoi(ret, esMap){
|
||||
def simlarObj = null
|
||||
def region = ""
|
||||
if(ret.district){
|
||||
region = ret.district
|
||||
}
|
||||
def regionMap = esMap.get(region)
|
||||
if(regionMap){
|
||||
def keys = regionMap.keySet()
|
||||
|
||||
for(def key : keys){
|
||||
def esValue = regionMap.get(key)
|
||||
if(esValue.township && ret.township){
|
||||
if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//||
|
||||
def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
|
||||
if(distance <100){
|
||||
simlarObj = esValue
|
||||
println esValue.name +" "+ret.name+" "+distance
|
||||
break;
|
||||
}
|
||||
}
|
||||
// else{
|
||||
// def esKey = esValue.name.replace("-","").replace("·","").replace("(", "").replace(")", "")
|
||||
// def bdKey = ret.name.replace("-","").replace("·","").replace("(", "").replace(")", "")
|
||||
// double score = CosineSimilarAlgorithm.getSimilarity(esKey,bdKey);
|
||||
// if(score && score>=0.95) {
|
||||
// def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
|
||||
// if(distance <100){
|
||||
// simlarObj = esValue
|
||||
// println esValue.name +" "+ret.name+" "+distance +" "+score
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println "无数据"
|
||||
}
|
||||
|
||||
return simlarObj
|
||||
}
|
||||
|
||||
|
||||
def findEsPoiByCity(city){
|
||||
// def sql = "select city,district,name,location from didata_map_poi_single where city='${city}' "
|
||||
// def rets = new HashSet()
|
||||
// ElasticUtils89.executeSqlQuery(sql, { d->
|
||||
//
|
||||
// })
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
// def rets = new HashSet()
|
||||
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
// BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
def map = new HashMap()
|
||||
ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi_single", queryBuilder, { String id, Map d->
|
||||
// rets << d.city +"##"+d.district+"##"+d.name
|
||||
filter.put(d.city +"##"+d.district+"##"+d.name)
|
||||
def district = ""
|
||||
if(d.district){
|
||||
district = d.district
|
||||
}
|
||||
def regionMap = map.get(district)
|
||||
if(!regionMap){
|
||||
regionMap = new HashMap()
|
||||
}
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", d.city)
|
||||
o.put("district", d.district)
|
||||
o.put("name", d.name)
|
||||
o.put("township", d.township)
|
||||
o.put("location", d.location)
|
||||
regionMap.put(d.name.replace("-","").replace("·",""), o)
|
||||
map.put(district, regionMap)
|
||||
})
|
||||
return filter
|
||||
return map
|
||||
}
|
||||
|
||||
def queryMapPoi(city){
|
||||
@@ -325,6 +383,11 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
// retMap.small_type = "村庄级地名"
|
||||
retMap.cat = map.std_tag
|
||||
|
||||
if(!map.std_tag){
|
||||
// println map.name +" std_tag: "+ map.std_tag +" 为空过滤"
|
||||
return null
|
||||
}
|
||||
|
||||
def nameArr = map.std_tag.split(";|:")
|
||||
if(nameArr.size()>=1){
|
||||
retMap.type = nameArr[0]
|
||||
@@ -429,7 +492,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
keys.put("gcj02_x", 1)
|
||||
keys.put("gcj02_y", 1)
|
||||
keys.put("std_tag", 1)
|
||||
def list = BAIDU_POI_COLL.find(q, keys).toList()
|
||||
def list = BAIDU_POI_COLL.find(q, keys).batchSize(2000).toList()
|
||||
return list;
|
||||
}
|
||||
|
||||
@@ -701,12 +764,11 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
int batchSize = 1000;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils89.getClient("192.168.0.89")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
|
||||
rows.each{key,map->
|
||||
// def map = rows.get(key)
|
||||
|
||||
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
}
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
@@ -717,7 +779,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
}
|
||||
|
||||
request = new BulkRequest();
|
||||
@@ -736,7 +798,6 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
}
|
||||
}
|
||||
|
||||
println "[${city}] 结束时间 "+new Date()
|
||||
}
|
||||
|
||||
|
||||
|
||||
+6
-18
@@ -1,28 +1,16 @@
|
||||
package com.fangjia.script.trade
|
||||
|
||||
|
||||
import java.util.regex.Matcher
|
||||
|
||||
package com.fangjia.script.sync
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.script.BaseBatch;
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.util.DateUtils;
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import com.fangjia.core.util.StringUtil;
|
||||
import com.fangjia.datastore.driver.DSCollection;
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection;
|
||||
import com.mongodb.DBObject
|
||||
import com.fangjia.script.util.MongoUtils;
|
||||
import com.fangjia.core.util.MongoUtils;
|
||||
import com.fangjia.script.Watcher;
|
||||
import com.mongodb.Bytes
|
||||
import org.apache.commons.io.FileUtils
|
||||
|
||||
/**
|
||||
* ½«ÏßÉÏopen uriÊý¾Ýͬ²½µ½192.168.0.60ÉÏ
|
||||
* @author gaojuan
|
||||
@@ -31,7 +19,7 @@ import org.apache.commons.io.FileUtils
|
||||
class CopyOpenUriLog extends BaseBatch{
|
||||
|
||||
|
||||
static DBCollection DESC_OPEN = MongoUtils.getCollectionByName("business", "open_uri_log", "seawater.mongo.db.fangjia.com", 27020)
|
||||
static DBCollection DESC_OPEN = MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
public static final DBCollection SOURCE_OPEN = MongoUtils.getCollectionByName("business", "open_uri_log", "log.mongo.db.fangjia.com", 52002)
|
||||
static main(String[] args){
|
||||
runBatch(new CopyOpenUriLog(), args)
|
||||
Reference in New Issue
Block a user