This commit is contained in:
2024-09-03 09:55:17 +08:00
45 changed files with 1838 additions and 289 deletions
@@ -24,7 +24,7 @@ class AddThirdPriceSample {
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
if(dist){
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407)
def udata = new BasicDBObject("zhongyuan_esf_price", d['C'].toString().toDouble().toInteger())
def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
}else{
@@ -68,10 +68,10 @@ class BigdataDealBatch extends BaseBatch{
//664a22000000000000000000 5-20
//664b73800000000000000000 5-21
def citys = CcbCommonUtils.getRunningCitys(args)
def citys =CcbCommonUtils.getRunningCitys(args)
println "curDate:"+curDate
def allCitys = CommonDatautils.findAllCity()
def allCitys = CommonDatautils.findAllCity()
for(def city : allCitys){
addDealPriceMatch(city);
}
@@ -720,7 +720,7 @@ class BigdataDealBatch extends BaseBatch{
if(!t.match_id){
def region = RegionData.formatRegion(t.city, t.region)
def rMap = null
if(region) {
if(region || (!region && ["澄迈"].contains(t.city))) {
if(t.house_use && t.house_use.contains("商住")){
if(!t.name.contains("商住")){
t.name = t.name +"商住楼"
@@ -744,7 +744,9 @@ class BigdataDealBatch extends BaseBatch{
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else if(rMap && rMap.address && address && rMap.region!=region && address.contains(rMap.address)){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else {
}else if (rMap && ["澄迈"].contains(rMap.city)){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else{
if (rMap) {
println "区域不同"
}
@@ -260,6 +260,9 @@ class ThirdPartyPriceBatch extends BaseBatch {
if(third.get("fccs_esf_price")){
price_map.fccs_avg_price = third.get("fccs_esf_price")
}
if(third.get("csjj_esf_price")){
price_map.csjj_avg_price = third.get("csjj_esf_price")
}
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
allRows << dataMap
}
@@ -308,6 +311,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
web = "我爱我家"
}else if(k=="fccs_avg_price"){
web = "房产超市"
}else if(k=="csjj_avg_price"){
web = "城市居家"
}
ret << sourceMap.get(web)
@@ -525,6 +530,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
@@ -593,6 +599,10 @@ class ThirdPartyPriceBatch extends BaseBatch {
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
}
if(!dataMap.get("csjj_esf_price") && db.get("csjj_esf_price") && db.get("csjj_esf_price")>3000){
dataMap.put("csjj_esf_price", db.get("csjj_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
@@ -600,8 +610,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
def resultMap = [:]
map.each {key, value->
if(value && value.size()>1){
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){
if(value && value.size()>1){ //csjj_esf_price 国信达成交
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price && !value.csjj_esf_price){
def price = statMap.get(key)
if(price){
def priceMap = [:]
@@ -4746,7 +4746,7 @@ class CommonDatautils {
townMap.put(key, list)
}
println "乡镇街道 加载完成"
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
return townMap
}
@@ -15,13 +15,16 @@ import org.apache.commons.lang.StringUtils
class OpenApiUtils {
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag, def suggestName){
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
if(preciseLevelflag){
url = url +"&preciseLevel=strict"
}
if(suggestName){
url = url +"&suggestName="+suggestName
}
if(suggestDistrict){
url = url +"&suggestDistrict="+suggestDistrict
}
String content = new HttpRequest().get(url, "utf-8");
def rMap = [:]
if(content){
@@ -192,42 +192,112 @@ class DevProcess {
}
}
def findDev(city, name){
def q = new BasicDBObject()
q.put("name", name)
q.put("city", city)
def obj = development_zone_CONN.findOne(q)
def findDev( name){
def obj = findDevByName(name)
if(!obj){
obj = findDevByAlias(name)
if(obj){
obj.match_way = "别名"
}
}else{
obj.match_way = "相同"
}
return obj
}
def findDebBySimlar(def devs, def name, def city){
name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
def last = null
for(def d : devs){
def alias1 = null
if(!d.alias){
alias1 = []
}else{
alias1 = d.alias
}
if(!alias1.contains(d.name)){
alias1 << d.name
}
if(alias1){
try {
for(def n : alias1){
n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){
last = d
break
}
}
}catch(exception){
println alias1
}
}
if(last){
break
}
}
return last
}
def findDevByName(name){
def q = new BasicDBObject()
q.put("name", name)
return development_zone_CONN.findOne(q)
}
def findDevByAlias(name){
def q = new BasicDBObject()
q.put('alias', new BasicDBObject('$in',[name]))
return development_zone_CONN.findOne(q)
}
def findDevBySimlar(name){
def q = new BasicDBObject()
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
return development_zone_CONN.findOne(q)
}
def findDevByPark(prov, city, name){
def q = new BasicDBObject()
q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put("type", "园区")
return dev_mapping_CONN.findOne(q)
}
def devSplitRoad(){
def i = 0;
def i = 1;
while(i<20){
def fileName = "/home/goojia/gj/qzd-${i}.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\数据建设\\开发区\\福建省.xlsx"
fileName = "D:\\数据\\数据建设\\开发区\\新跑省份\\河北省\\test-${i}.xlsx"
}
File file=new File(fileName);
if(file.exists()){
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['E'] && !d['G']){
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
if(d['C'] && !d['F']){
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
if(jsonObject1) {
if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") {
def devName = jsonObject1.get("dev")
def obj = findDev(d['B'], devName)
def obj = findDev(devName)
d['F'] = devName
if(jsonObject1.containsKey("detail") && jsonObject1.detail.dev_confi!='null'){
d['S'] = jsonObject1.detail.dev_confi
}
if (obj) {
d['Q'] = obj._id.toString()
d['R'] = obj.name
d['O'] = obj._id.toString()
d['P'] = obj.name
d['U'] = obj.match_way
}
}
}
}
if(d['C'] && !d['S']) {
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
if(d['E'] && !d['M']) {
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if (jsonObject.get("province") && jsonObject.get("province") != "null") {
@@ -256,7 +326,7 @@ class DevProcess {
d['N'] = "信息不完整"
}
if (!d['N'] && !d['C'].contains(road)) {
if (!d['N'] && d['C'] && !d['C'].contains(road)) {
d['N'] = "原地址不含路"
}
@@ -276,17 +346,23 @@ class DevProcess {
if (jsonObject.get("dev") && jsonObject.get("dev") != "null") {
def devName = jsonObject.get("dev")
def obj = findDev(d['B'], devName)
def obj = findDev(devName)
d['M'] = devName
if (obj) {
d['S'] = obj._id.toString()
d['T'] = obj.name
d['Q'] = obj._id.toString()
d['R'] = obj.name
d['V'] = obj.match_way
}
if(jsonObject.containsKey("detail") && jsonObject.detail.dev_confi!='null'){
d['T'] = jsonObject.detail.dev_confi
}
}
}
},0)
i++
}
i++
println fileName+" 完成"
}
}
@@ -33,13 +33,13 @@ class MinglujiData {
def prov = "Õã½­Ê¡"
def city = "¼ÎÐË"
// exportDatas(city)
// matchDev(prov, city)
matchDev(prov, city)
matchCommunity(city)
// tempMatchDev(prov)
// saveMljData()
fiveLevelSplit(city)
}
def exportDatas(city){
def datas = findMinglujiDatas(city)
@@ -113,7 +113,7 @@ class MinglujiData {
})
}
i++
println fileName +" 完成"
println fileName +" 完成(匹配开发区不使用接口)"
}
}
@@ -196,15 +196,40 @@ class MinglujiData {
}
}
}, 0)
println fileName +" 远行完成"
println fileName +" 运行完成"
}
i++
}
}
def fiveLevelSplit(city){
def i = 1;
while(i<20) {
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\名录集\\mlj_${i}.xlsx"
}
File file = new File(fileName);
if (file.exists()) {
ExcelUtils.multiThreadingModify(fileName, { d ->
if(d['J'] && !d['X']){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatFivelevel(d['B'], d['J'])
if(jsonObject) {
if (jsonObject.get("address") && jsonObject.get("address") != "null") {
d['X'] = jsonObject.get("address")
}
}
}
})
}
i++
}
}
def saveMljData(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库数据.xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库2.xlsx",{d->
def obj = new BasicDBObject()
obj.put("province", d['A'])
obj.put("city", d['B'])
@@ -213,7 +238,7 @@ class MinglujiData {
obj.put("address", d['E'])
obj.put("dev_name", d['F'])
obj.put("committee", d['G'])
obj.put("street", d['H'])
obj.put("township", d['H'])
company_100_CONN.save(obj)
})
}
@@ -1,6 +1,7 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.addrepo.InvokeHttpRequest
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
@@ -335,7 +336,7 @@ class FangjiaClientUtils {
try {
address = address.replace("{","")
// def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.100:32080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -352,18 +353,27 @@ class FangjiaClientUtils {
public static def formatNinelevel(city, address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true"
def url = null
address = address.replace("{","")
int retryCount = 0
while (retryCount < 3) {
url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
if (content) {
break
}
retryCount++
}
if (!content) {
return null
}
try {
jsonObj = JSONObject.fromObject(content)
// println jsonObj
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
println address +" "+content+" "+e.getMessage()
}
return jsonObj;
}
@@ -90,6 +90,7 @@ class FjwBigdataDealBatch extends BaseBatch{
}
println "txt Éú³É " + rows.size()
saveToPgSql(rows)
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
@@ -356,7 +357,7 @@ class FjwBigdataDealBatch extends BaseBatch{
def converData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
def data = distCodeMap.get(trade.region)
def data = distCodeMap.get(trade.match_region)
if(!data){
println trade.city +" "+trade.region+" 没有找到对应的区域"
return ret
@@ -580,8 +581,8 @@ class FjwBigdataDealBatch extends BaseBatch{
}
def getYouDaData(statMap){
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
@@ -27,7 +27,7 @@ class QiliangTest {
// task.run()
def data = [
'address': '湖州市南太湖高新技术产业园区大港村陈家斗80号',
'address': '浙江省湖州市长兴县煤山镇工业园区山鹰大道7号',
'type': '1' //1:对公 0:对私
]
def s = task.getProxyResult(data);
@@ -10,6 +10,7 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
@@ -32,6 +33,7 @@ import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -126,7 +128,11 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def sysnchVillage(){
def buducitys = [ '临汾市', '临沂市', '临沧市', '临高县', '丹东市',
def buducitys = [ "宁波市","丽水市","杭州市","温州市","嘉兴市","台州市","绍兴市","金华市","衢州市",'茂名市',
'荆州市', '荆门市', '莆田市','苏州市','菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市',
'衢州市', '襄阳市', '西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市',
'辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市', '中山市', '临夏回族自治州',
'临汾市', '临沂市', '临沧市', '临高县', '丹东市','邵阳市', '郑州市', '郴州市', '鄂尔多斯市', '鄂州市', '酒泉市',
'丽水市', '丽江市', '乌兰察布市', '乌海市', '乌鲁木齐市', '乐东黎族自治县', '乐山市', '九江市',
'云浮市', '五家渠市', '五指山市', '亳州市', '仙桃市', '伊春市', '伊犁哈萨克自治州', '佛山市',
'佳木斯市', '保亭黎族苗族自治县', '保定市', '保山市', '信阳市', '儋州市', '克孜勒苏柯尔克孜自治州',
@@ -136,7 +142,7 @@ class GaoDePoiDataProcess extends BaseBatch{
'吕梁市', '吴忠市', '周口市', '呼伦贝尔市', '呼和浩特市', '和田地区', '咸宁市', '咸阳市',
'哈密地区', '哈尔滨市', '唐山市', '商丘市', '商洛市', '喀什地区', '嘉兴市', '嘉峪关市', '四平市',
'固原市', '图木舒克市', '塔城地区', '大兴安岭地区', '大同市', '大庆市', '大理白族自治州', '大连市',
'天水市', '天津市', '天门市', '太原市', '威海市', '娄底市', '孝感市', '宁德市', '宁波市',
'天水市', '天津市', '天门市', '太原市', '威海市', '娄底市', '孝感市', '宁德市',
'安庆市', '安康市', '安阳市', '安顺市', '定安县', '定西市', '宜宾市', '宜昌市', '宜春市',
'宝鸡市', '宣城市', '宿州市', '宿迁市', '屯昌县', '山南地区', '岳阳市', '崇左市', '巴中市',
'巴彦淖尔市', '巴音郭楞蒙古自治州', '常州市', '常德市', '平凉市', '平顶山市', '广元市', '广安市',
@@ -149,17 +155,12 @@ class GaoDePoiDataProcess extends BaseBatch{
'梅州市', '梧州市', '楚雄彝族自治州', '榆林市', '武威市', '武汉市', '毕节地区', '永州市', '汉中市',
'汕头市', '汕尾市', '江门市', '池州市', '沈阳市', '沧州市', '河池市', '河源市', '泉州市',
'泰安市', '泰州市', '泸州市', '洛阳市', '济南市', '济宁市', '济源市', '海东地区', '海口市',
'淄博市', '淮北市', '淮南市', '淮安市', '深圳市', '清远市', '温州市', '渭南市', '湖州市',
'淄博市', '淮北市', '淮南市', '淮安市', '深圳市', '清远市', '温州市', '渭南市',
'湘潭市', '湘西土家族苗族自治州', '湛江市', '滁州市', '滨州市', '漯河市', '漳州市', '潍坊市',
'潜江市', '潮州市', '澄迈县', '濮阳市', '烟台市', '焦作市', '牡丹江市', '玉林市', '玉溪市',
'珠海市', '琼中黎族苗族自治县', '琼海市', '甘南藏族自治州', '白城市', '白山市', '白沙黎族自治县',
'白银市', '百色市', '益阳市', '盐城市', '盘锦市', '眉山市', '石嘴山市', '石家庄市', '石河子市',
'神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绍兴市', '绥化市', '绵阳市', '聊城市',
'肇庆市', '自贡市', '舟山市', '芜湖市', '苏州市', '茂名市', '荆州市', '荆门市', '莆田市',
'菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市', '衢州市', '襄阳市',
'西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市',
'辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市',
'邢台市', '那曲地区', '邯郸市', '邵阳市', '郑州市', '郴州市', '鄂尔多斯市', '鄂州市', '酒泉市',
'神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绥化市', '邢台市', '那曲地区', '邯郸市',
'重庆市', '金华市', '金昌市', '钦州市', '铁岭市', '铜仁地区', '铜川市', '铜陵市', '银川市',
'锡林郭勒盟', '锦州市', '镇江市', '长春市', '长沙市', '长治市', '阜新市', '阜阳市', '防城港市',
'阳江市', '阳泉市', '阿克苏地区', '阿勒泰地区', '阿拉善盟', '阿拉尔市', '阿里地区', '陇南市',
@@ -167,50 +168,55 @@ class GaoDePoiDataProcess extends BaseBatch{
'鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市',
'黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市']
//"丽水市","杭州市",
def runbuducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市",'七台河市', '上海市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市',
'东营市', '中卫市', '中山市', '临夏回族自治州']
def runbuducitys = ["舟山市","湖州市"]
for(def city : buducitys){
if(runbuducitys.contains(city)){
continue
}
def pois = findAllVillagePoiMap(city)
println "百度loading finish: "+pois.size()+" "+new Date().format("yyyy-MM-dd HH:mm:ss")
def rows = createVillageData(city, pois)
save(city, rows)
// saveToMG(city, rows)
println city +" "+pois.size()+" add:"+rows.size()
println city +" "+pois.size()+" add:"+rows.size()+" "+new Date().format("yyyy-MM-dd HH:mm:ss")
}
}
def createVillageData(city, pois){
def rets = new CopyOnWriteArrayList()
def keyRets = new CopyOnWriteArrayList()
def rets = new ConcurrentHashMap()
//def keyRets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(pois.size())
// def townMap = findTownRange(city)
def townMap = CommonDatautils.queryTownByCity(city)
def poiKeysFilter = findEsPoiByCity(city)
def esMap = findEsPoiByCity(city)
def nameKeys = esMap.keySet()
println "Es 加载完成:"+new Date().format("yyyy-MM-dd HH:mm:ss")
for(def i=0;i<pois.size();i++){
def p = pois[i]
final def p = pois[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
// def obj = MAPPOI_CONN.findOne(new BasicDBObject("name", p.name).append("district", p.district).append("city", p.city))
def key = p.city+"##"+p.district+"##"+p.name
if(!poiKeysFilter.mightContain(key)){
def key = p.city+"##"+p.district+"##"+p.name.replace("-","").replace("·","")
if(!nameKeys.contains(key)){
def prov = ProvinceData.getProvince(p.city.replace("市",""))
def ret = createVillageData(prov, p, townMap)
if(!keyRets.contains(ret.city+"##"+ret.district+"##"+ret.name)){
keyRets.addAll(ret)
rets << ret
if(ret){
def simlarObj = null
if(esMap){
simlarObj = findSimlarPoi(ret, esMap)
}
if(!simlarObj){
def newKey = ret.city+"##"+ret.district+"##"+ret.name.replace("-","").replace("·","")
if(!rets.containsKey(newKey)){
rets.put(newKey, ret)
}
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -223,21 +229,73 @@ class GaoDePoiDataProcess extends BaseBatch{
return rets
}
def findSimlarPoi(ret, esMap){
def simlarObj = null
def region = ""
if(ret.district){
region = ret.district
}
def regionMap = esMap.get(region)
if(regionMap){
def keys = regionMap.keySet()
for(def key : keys){
def esValue = regionMap.get(key)
if(esValue.township && ret.township){
if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//||
def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
if(distance <100){
simlarObj = esValue
println esValue.name +" "+ret.name+" "+distance
break;
}
}
// else{
// def esKey = esValue.name.replace("-","").replace("·","").replace("(", "").replace(")", "")
// def bdKey = ret.name.replace("-","").replace("·","").replace("(", "").replace(")", "")
// double score = CosineSimilarAlgorithm.getSimilarity(esKey,bdKey);
// if(score && score>=0.95) {
// def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
// if(distance <100){
// simlarObj = esValue
// println esValue.name +" "+ret.name+" "+distance +" "+score
// break;
// }
// }
// }
}
}
}else{
println "无数据"
}
return simlarObj
}
def findEsPoiByCity(city){
// def sql = "select city,district,name,location from didata_map_poi_single where city='${city}' "
// def rets = new HashSet()
// ElasticUtils89.executeSqlQuery(sql, { d->
//
// })
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
// def rets = new HashSet()
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
// BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
def map = new HashMap()
ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi_single", queryBuilder, { String id, Map d->
// rets << d.city +"##"+d.district+"##"+d.name
filter.put(d.city +"##"+d.district+"##"+d.name)
def district = ""
if(d.district){
district = d.district
}
def regionMap = map.get(district)
if(!regionMap){
regionMap = new HashMap()
}
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", d.district)
o.put("name", d.name)
o.put("township", d.township)
o.put("location", d.location)
regionMap.put(d.name.replace("-","").replace("·",""), o)
map.put(district, regionMap)
})
return filter
return map
}
def queryMapPoi(city){
@@ -325,6 +383,11 @@ class GaoDePoiDataProcess extends BaseBatch{
// retMap.small_type = "村庄级地名"
retMap.cat = map.std_tag
if(!map.std_tag){
// println map.name +" std_tag: "+ map.std_tag +" 为空过滤"
return null
}
def nameArr = map.std_tag.split(";|:")
if(nameArr.size()>=1){
retMap.type = nameArr[0]
@@ -429,7 +492,7 @@ class GaoDePoiDataProcess extends BaseBatch{
keys.put("gcj02_x", 1)
keys.put("gcj02_y", 1)
keys.put("std_tag", 1)
def list = BAIDU_POI_COLL.find(q, keys).toList()
def list = BAIDU_POI_COLL.find(q, keys).batchSize(2000).toList()
return list;
}
@@ -701,12 +764,11 @@ class GaoDePoiDataProcess extends BaseBatch{
int batchSize = 1000;
int processedCount = 0;
def client = ElasticUtils89.getClient("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
rows.each{key,map->
// def map = rows.get(key)
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
@@ -717,7 +779,7 @@ class GaoDePoiDataProcess extends BaseBatch{
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss")
}
request = new BulkRequest();
@@ -736,7 +798,6 @@ class GaoDePoiDataProcess extends BaseBatch{
}
}
println "[${city}] 结束时间 "+new Date()
}
@@ -1,28 +1,16 @@
package com.fangjia.script.trade
import java.util.regex.Matcher
package com.fangjia.script.sync
import com.fangjia.core.Constants
import com.fangjia.script.BaseBatch;
import com.fangjia.script.BatchOwner
import com.fangjia.script.util.DateUtils;
import java.util.regex.Pattern
import org.apache.commons.lang.StringUtils;
import com.fangjia.core.util.StringUtil;
import com.fangjia.datastore.driver.DSCollection;
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.util.DateUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection;
import com.mongodb.DBObject
import com.fangjia.script.util.MongoUtils;
import com.fangjia.core.util.MongoUtils;
import com.fangjia.script.Watcher;
import com.mongodb.Bytes
import org.apache.commons.io.FileUtils
/**
* ½«ÏßÉÏopen uriÊý¾Ýͬ²½µ½192.168.0.60ÉÏ
* @author gaojuan
@@ -31,7 +19,7 @@ import org.apache.commons.io.FileUtils
class CopyOpenUriLog extends BaseBatch{
static DBCollection DESC_OPEN = MongoUtils.getCollectionByName("business", "open_uri_log", "seawater.mongo.db.fangjia.com", 27020)
static DBCollection DESC_OPEN = MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
public static final DBCollection SOURCE_OPEN = MongoUtils.getCollectionByName("business", "open_uri_log", "log.mongo.db.fangjia.com", 52002)
static main(String[] args){
runBatch(new CopyOpenUriLog(), args)