Merge remote-tracking branch 'origin/master'
This commit is contained in:
+51
@@ -467,11 +467,15 @@ public class AddrSvcController {
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
map.put("addrArea", addrArea(map, type));
|
||||
map.put("countyLevelCity", countyLevelCity(map));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
map.put("addrArea", addrArea(map, type));
|
||||
map.put("countyLevelCity", countyLevelCity(map));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
|
||||
@@ -498,6 +502,45 @@ public class AddrSvcController {
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断县级市
|
||||
*/
|
||||
public String countyLevelCity(Map<String, String> map) {
|
||||
if (map.get("city") == null) {
|
||||
return "";
|
||||
}
|
||||
if (map.get("city").endsWith("区")) {
|
||||
return "否";
|
||||
}
|
||||
if (map.get("city").equals("东莞市") || map.get("city").equals("嘉峪关市") || map.get("city").equals("中山市") || map.get("city").equals("儋州市") ) {
|
||||
return "否";
|
||||
}
|
||||
return "是";
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断所属区域
|
||||
*
|
||||
* @param map
|
||||
* @return
|
||||
*/
|
||||
public String addrArea(Map<String, String> map, String type) {
|
||||
if (map.get("verifyFlag") == null ) {
|
||||
return "";
|
||||
}
|
||||
if (map.get("verifyFlag").equals("非农")) {
|
||||
return "城市区域";
|
||||
}
|
||||
if (map.get("verifyFlag").equals("农")) {
|
||||
return "农村区域";
|
||||
}
|
||||
if ("2".equals(type) && "是".equals(map.get("countyGov"))) {
|
||||
return "县城城区";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* 涉农识别-新规则-上海征信
|
||||
*
|
||||
@@ -565,11 +608,15 @@ public class AddrSvcController {
|
||||
basicModel.setResult(result);
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("addrArea", addrArea(map, type));
|
||||
map.put("countyLevelCity", countyLevelCity(map));
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("addrArea", addrArea(map, type));
|
||||
map.put("countyLevelCity", countyLevelCity(map));
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
@@ -703,11 +750,15 @@ public class AddrSvcController {
|
||||
basicModel.setResult(result);
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("addrArea", addrArea(map, type));
|
||||
map.put("countyLevelCity", countyLevelCity(map));
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("addrArea", addrArea(map, type));
|
||||
map.put("countyLevelCity", countyLevelCity(map));
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 将国信达及成交对标数据加入到第三方均价表,做为均价参考
|
||||
*/
|
||||
class AddThirdPriceSample {
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new AddThirdPriceSample()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
for(def i=1;i<19;i++){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d->
|
||||
if(d['A'] && d['C']){
|
||||
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
|
||||
if(dist){
|
||||
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407)
|
||||
def udata = new BasicDBObject("zhongyuan_esf_price", d['C'].toString().toDouble().toInteger())
|
||||
|
||||
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
|
||||
}else{
|
||||
println d['A'] +" 没查到"
|
||||
}
|
||||
}
|
||||
}, i)
|
||||
|
||||
println(i)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+6
-6
@@ -70,12 +70,12 @@ class BigdataDealBatch extends BaseBatch{
|
||||
//664b73800000000000000000 5-21
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
// println "curDate:"+curDate
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
println "curDate:"+curDate
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
|
||||
def rows = []
|
||||
|
||||
+58
-26
@@ -612,9 +612,9 @@ def findAllData(sqlInstance236, citycode){
|
||||
////},0)
|
||||
//
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d->
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\删除小区.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def dbo = CommonDatautils.findDistrictById(d['D'])
|
||||
// def dbo = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dbo){
|
||||
// def udata = new BasicDBObject()
|
||||
//// udata.put("name_cat", 11)
|
||||
@@ -630,6 +630,38 @@ def findAllData(sqlInstance236, citycode){
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
|
||||
if(d['A'] && d['B']){
|
||||
def dbo = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dbo){
|
||||
def alias = null
|
||||
if(!dbo.alias){
|
||||
alias = []
|
||||
}else{
|
||||
alias = dbo.alias
|
||||
}
|
||||
if(!alias.contains(d['B'])){
|
||||
alias << d['B']
|
||||
}
|
||||
if(alias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", dbo._id)
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d['A']+" "+"别名已存在"
|
||||
}
|
||||
|
||||
}else{
|
||||
println dbo._id.toString() +" 没找到"
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
|
||||
//for(def c : citys){
|
||||
// def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"])
|
||||
@@ -1299,34 +1331,34 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['J']){
|
||||
// def region = RegionData.formatRegion(d['A'], d['J'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['J']){
|
||||
//// def region = RegionData.formatRegion(d['A'], d['J'])
|
||||
//// if(region){
|
||||
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
|
||||
//// if(dist){
|
||||
//// d['B'] = dist.region
|
||||
//// d['C'] = dist.name
|
||||
//// d['L'] = dist.status-dist.name_cat
|
||||
//// }
|
||||
//// }
|
||||
//// if(d['J'] instanceof String){
|
||||
// def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
// if(dist){
|
||||
// d['A'] = dist.city
|
||||
// d['B'] = dist.region
|
||||
// d['C'] = dist.name
|
||||
// d['L'] = dist.status-dist.name_cat
|
||||
// d['K'] = dist.estate_type2
|
||||
// d['L'] = dist.status
|
||||
// d['M'] = dist.name_cat
|
||||
//
|
||||
//// d['H'] = dist.name
|
||||
// }
|
||||
// }
|
||||
// if(d['J'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
if(dist){
|
||||
d['A'] = dist.city
|
||||
d['B'] = dist.region
|
||||
d['C'] = dist.name
|
||||
d['K'] = dist.estate_type2
|
||||
d['L'] = dist.status
|
||||
d['M'] = dist.name_cat
|
||||
|
||||
// d['H'] = dist.name
|
||||
}
|
||||
// }
|
||||
}
|
||||
},0)
|
||||
//// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
|
||||
|
||||
+9
@@ -257,6 +257,9 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
if(third.get("5i5j_esf_price")){
|
||||
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
|
||||
}
|
||||
if(third.get("fccs_esf_price")){
|
||||
price_map.fccs_avg_price = third.get("fccs_esf_price")
|
||||
}
|
||||
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
allRows << dataMap
|
||||
}
|
||||
@@ -303,6 +306,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
web = "房多多"
|
||||
}else if(k=="wiwj_avg_price"){
|
||||
web = "我爱我家"
|
||||
}else if(k=="fccs_avg_price"){
|
||||
web = "房产超市"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
@@ -584,6 +589,10 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
|
||||
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
|
||||
}
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
|
||||
+1
@@ -1212,6 +1212,7 @@ class CommonDatautils {
|
||||
public static def findAllCount(String cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("cat", cat);
|
||||
q.put("status", 0)
|
||||
return SEAWEED_MONGO_COLL.count(q)
|
||||
}
|
||||
|
||||
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.google.gson.JsonObject
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.jsoup.Jsoup
|
||||
|
||||
class BaiduBaikeTown {
|
||||
|
||||
DBCollection dzb_COLL = MongoUtils.getCollectionByName("crawl", "development_zone_bdbk", "192.168.0.206",27020)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new BaiduBaikeTown()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",{row->
|
||||
if(!row['E'] && row['A']=="郑州经济技术开发区"){
|
||||
def q = new BasicDBObject()
|
||||
q.put("desc",new BasicDBObject('$ne', 'null'))
|
||||
q.put("name",row['A'])
|
||||
def d = dzb_COLL.findOne(q)
|
||||
def townMap = parse(d.desc, d.name)
|
||||
row['D'] = townMap.town_str1
|
||||
row['F'] = townMap.town_str2
|
||||
|
||||
// def ret = []
|
||||
// ret << ["名称", "基本信息", "目录", "全文"]
|
||||
// for(def d : dbcursor){
|
||||
// def townMap = parse(d.desc, d.name)
|
||||
// if(townMap.town){
|
||||
// println d.name+" "+townMap.town
|
||||
// }
|
||||
// ret << [d.name, townMap.town, townMap.towns, townMap.town_str1, townMap.town_str2]
|
||||
// }
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx", ret, "数据")
|
||||
}
|
||||
|
||||
def parse(desc, name){
|
||||
def map = [:]
|
||||
if(desc){
|
||||
def doc = Jsoup.parse(desc)
|
||||
def town = JQueryUtils.selectText(".basicInfoBlock_Ex6IU div:contains(下辖地区) dd", doc)
|
||||
|
||||
def s = doc.toString()
|
||||
def dataStr = DataHandleUtils.findMatchPattern(s,"PAGE_DATA=.*</script>").replace("PAGE_DATA= ","").replace("</script>","")
|
||||
def jsonObj = JSONObject.fromObject(dataStr)
|
||||
|
||||
def str = DataHandleUtils.findMatchPattern(doc.text(),"下辖.*[街道|镇]、?和?")
|
||||
def towns = []
|
||||
if(jsonObj && jsonObj.catalog){
|
||||
def catalogs = jsonObj.catalog
|
||||
def index = null
|
||||
for(def log : catalogs){
|
||||
def curIndex = log.index
|
||||
if(log.title=="行政区划"){
|
||||
index = log.index
|
||||
}
|
||||
if(index && curIndex.toString().startsWith(index+"-")){
|
||||
towns << log.title
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
map.town_str1 = str
|
||||
if(town){
|
||||
map.town = town
|
||||
}else{
|
||||
map.town = ""
|
||||
}
|
||||
|
||||
if(towns){
|
||||
map.towns = towns.join(",")
|
||||
}else{
|
||||
map.towns = ""
|
||||
}
|
||||
|
||||
|
||||
def str2 = DataHandleUtils.findMatchPattern(doc.text(),"开发区辖.*[,|。]")
|
||||
map.town_str2 = str2
|
||||
return map
|
||||
}
|
||||
map.town_str1 = ""
|
||||
map.town_str2 = ""
|
||||
map.town = ""
|
||||
map.towns = ""
|
||||
return map
|
||||
}
|
||||
}
|
||||
+71
-44
@@ -25,9 +25,11 @@ class DevProcess {
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
|
||||
DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
|
||||
DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
DBCollection DEV_MAPPING_CONN2 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping_2", MONGO100_URL)
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
@@ -54,7 +56,7 @@ class DevProcess {
|
||||
// task.createDevMapping()
|
||||
// task.deleteAddr()
|
||||
// task.deleteAddr2()
|
||||
// task.synchDevTo43()
|
||||
// task.synchDevTo100()
|
||||
// task.addNewDev2()
|
||||
|
||||
// task.addAlias()
|
||||
@@ -71,6 +73,8 @@ class DevProcess {
|
||||
// task.devToPark() //开发区下辖园区
|
||||
// task.checkTown()
|
||||
|
||||
// task.repeatSave()
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -692,13 +696,13 @@ class DevProcess {
|
||||
});
|
||||
}
|
||||
|
||||
def synchDevTo43(){
|
||||
def dbcursor = development_zone_CONN.find()
|
||||
def synchDevTo100(){
|
||||
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find()
|
||||
def rets = []
|
||||
for(def r : dbcursor){
|
||||
rets << r
|
||||
}
|
||||
DEVELOPMENT_ZONE_43_COLL.insert(rets)
|
||||
development_zone_CONN.insert(rets)
|
||||
}
|
||||
|
||||
|
||||
@@ -898,66 +902,89 @@ class DevProcess {
|
||||
}
|
||||
|
||||
def devToPark(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\开发区下辖街道-处理后表格.xlsx",0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",0, 1)
|
||||
def rets = []
|
||||
rets << ["开发区名称", "下辖街道", "城市", "区域"]
|
||||
for(def d : datas){
|
||||
if(d['E']){
|
||||
def arrs = d['E'].split(",")
|
||||
def q = new BasicDBObject().append("name", d['B'])
|
||||
def arrs = d['E'].split("、")
|
||||
def q = new BasicDBObject().append("name", d['A'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
for(def park : arrs){
|
||||
rets << [d['B'], park, obj.city, obj.district]
|
||||
rets << [d['A'], park, obj.city, obj.district]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx", rets, "下辖街道")
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx", rets, "下辖街道")
|
||||
}
|
||||
|
||||
|
||||
def checkTown(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d->
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx",{d->
|
||||
|
||||
if(!d['E']){
|
||||
if(d['A']){
|
||||
// if(d['A']=="长春高新技术产业开发区"){
|
||||
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['A'])
|
||||
if(jsonObject1) {
|
||||
if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
|
||||
def township = jsonObject1.get("township")
|
||||
def district = jsonObject1.get("district")
|
||||
d['E'] = "接口"
|
||||
if(district && district!='null'){
|
||||
d['F'] = district
|
||||
}
|
||||
if(township && township!='null'){
|
||||
d['G'] = township
|
||||
}
|
||||
|
||||
d['H'] = jsonObject1.get("townCode")
|
||||
if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){
|
||||
d['I'] = jsonObject1.get("provinceCode")
|
||||
}
|
||||
}
|
||||
// }
|
||||
}
|
||||
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city", d['C'])
|
||||
//// q.put("district", d['D'])
|
||||
// q.put("mzb_district_name", d['D'])
|
||||
// q.put("type", "乡镇街道")
|
||||
// q.put("street", d['B'])
|
||||
// def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
// if(obj){
|
||||
// d['E'] = "可入库2"
|
||||
// d['F'] = obj.district
|
||||
// def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B'])
|
||||
// if(jsonObject1) {
|
||||
// if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
|
||||
// def township = jsonObject1.get("township")
|
||||
// def district = jsonObject1.get("district")
|
||||
// d['E'] = "接口"
|
||||
// if(district && district!='null'){
|
||||
// d['F'] = district
|
||||
// }
|
||||
// if(township && township!='null'){
|
||||
// d['G'] = township
|
||||
// }
|
||||
//
|
||||
// d['H'] = jsonObject1.get("townCode")
|
||||
// if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){
|
||||
// d['I'] = jsonObject1.get("provinceCode")
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['C'])
|
||||
// q.put("district", d['D'])
|
||||
// q.put("mzb_district_name", d['D'])
|
||||
q.put("type", "乡镇街道")
|
||||
q.put('street',new BasicDBObject('$regex', Pattern.compile(d['B'].toString().trim()+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
// q.put("street", d['B'])
|
||||
// q.put('alias', new BasicDBObject('$in',[d['B']]))
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(obj){
|
||||
d['E'] = "可入库-忽略区域模糊2"
|
||||
d['F'] = obj.district
|
||||
d['G'] = obj.street
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},1)
|
||||
},0)
|
||||
}
|
||||
|
||||
def repeatSave(){
|
||||
def dbcursor = DEV_MAPPING_CONN.find(new BasicDBObject())
|
||||
def list = []
|
||||
for(def map : dbcursor){
|
||||
def columns = ["_id","province","city","district","name","type","address","dev_district", "township","park"]
|
||||
def udata = new BasicDBObject()
|
||||
map.each{k,v->
|
||||
if(columns.contains(k)){
|
||||
if(v instanceof String){
|
||||
if(k=="_id"){
|
||||
udata.put("_id", new ObjectId(v))
|
||||
}else{
|
||||
udata.put(k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
list << udata
|
||||
}
|
||||
DEV_MAPPING_CONN2.insert(list)
|
||||
}
|
||||
}
|
||||
|
||||
+13
-12
@@ -29,7 +29,7 @@ class DevUpdateTestBatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new DevUpdateTestBatch()
|
||||
def path = "D:\\数据\\数据建设\\开发区\\文心一言获取开发区下辖园区及街道.xlsx"//开发区-园区.xlsx
|
||||
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
|
||||
// task.addDev(path)
|
||||
task.addDevAddress(path)
|
||||
|
||||
@@ -137,26 +137,27 @@ class DevUpdateTestBatch {
|
||||
DEV_MAPPING_100_CONN.save(o)
|
||||
|
||||
|
||||
o.id = o._id.toString()
|
||||
o.remove("_id")
|
||||
o.remove("c_date")
|
||||
addEs(o, "192.168.0.89", "dev_mapping")
|
||||
try{
|
||||
addEs(o, "192.168.0.52", "dev_mapping")
|
||||
}catch(Exception e){
|
||||
println "52 es error: "+e.printStackTrace()
|
||||
}
|
||||
// o.id = o._id.toString()
|
||||
// o.remove("_id")
|
||||
// o.remove("c_date")
|
||||
// addEs(o, "192.168.0.89", "dev_mapping")
|
||||
// try{
|
||||
// addEs(o, "192.168.0.52", "dev_mapping")
|
||||
// }catch(Exception e){
|
||||
// println "52 es error: "+e.printStackTrace()
|
||||
// }
|
||||
|
||||
d['D'] = "已完成"
|
||||
}else{
|
||||
d['D'] = "存在"
|
||||
d['D'] = "重复"
|
||||
println d['A'] +" "+ d['C']+" 重复"
|
||||
}
|
||||
}else{
|
||||
d['D'] = "不存在"
|
||||
println d['A'] +" 不存在"
|
||||
}
|
||||
}
|
||||
},2)
|
||||
},1)
|
||||
}
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -40,11 +40,11 @@ class GXDdistrictMatch {
|
||||
|
||||
def addOldMatch(){
|
||||
//老匹配
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"//11城重新匹配.xlsx
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"//11城重新匹配.xlsx
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
println d['C']
|
||||
if(!d['AJ'] && d['I']){
|
||||
def jsonObject = OpenApiClient.addressMatch(d['C'], d['I'], "district", false,null, d['D'])
|
||||
def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D'])
|
||||
if (jsonObject != null && !jsonObject.containsKey("result")) {
|
||||
d['AJ'] = jsonObject.get("id")
|
||||
d['AK'] = jsonObject.get("city")
|
||||
|
||||
+230
@@ -0,0 +1,230 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 名录集
|
||||
*/
|
||||
class MinglujiData {
|
||||
|
||||
DBCollection mlj_COLL = MongoUtils.getCollectionByName("crawl", "kfq_mingluji_com", "192.168.0.206",27020)
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
|
||||
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
|
||||
static main(args) {
|
||||
def task = new MinglujiData()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
// exportDatas("湖州")
|
||||
// matchDev("浙江省","湖州")
|
||||
matchCommunity("湖州")
|
||||
}
|
||||
def exportDatas(city){
|
||||
def datas = findMinglujiDatas(city)
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\名录集\\${city}.xlsx", datas, "数据")
|
||||
}
|
||||
def matchDev(prov, city){
|
||||
def devs = loadAllDevs(prov)
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d->
|
||||
if(d['E'] && !d['N'] ){
|
||||
def obj = findDevByName(d['E'])
|
||||
if(!obj){
|
||||
obj = findDevByAlias(d['E'])
|
||||
if(!obj){
|
||||
obj = findDebBySimlar(devs, d['E'], d['B'])
|
||||
if(!obj){
|
||||
obj = findDevByPark(d['A'], d['B'], d['E'])
|
||||
if (!obj) {
|
||||
}else{
|
||||
d['N'] = "园区"
|
||||
}
|
||||
}else{
|
||||
d['N'] = "相似"
|
||||
}
|
||||
}else{
|
||||
d['N'] = "别名"
|
||||
}
|
||||
}else{
|
||||
d['N'] = "相同"
|
||||
}
|
||||
if(obj){
|
||||
d['M'] = obj.name
|
||||
d['O'] = obj.city
|
||||
d['P'] = obj.district
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def matchCommunity(city){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}-没配居委1.xlsx",{d->
|
||||
if(d['A'] && d['B'] && d['L'] && !d['Q']){
|
||||
def list = findCommunityByName(d['A'], d['B'], d['L'])
|
||||
if(!list){
|
||||
list = findCommunityBySimlar(d['A'], d['B'], d['L'])
|
||||
if(!list){
|
||||
|
||||
}else{
|
||||
d['T'] = "包含"
|
||||
}
|
||||
}else{
|
||||
d['T'] = "相同"
|
||||
}
|
||||
if(list){
|
||||
if(list.size()==1){
|
||||
def obj = list[0]
|
||||
d['Q'] = obj.name
|
||||
d['R'] = obj.street_name
|
||||
}else if(list.size()>=2){
|
||||
for(def l : list){
|
||||
if(d['C']==l.area_name){
|
||||
d['Q'] = l.name
|
||||
d['R'] = l.street_name
|
||||
break;
|
||||
}
|
||||
}
|
||||
d['S'] = list.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 0)
|
||||
}
|
||||
|
||||
def loadAllDevs(prov){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province", prov)
|
||||
return development_zone_CONN.find(q).toList()
|
||||
}
|
||||
|
||||
def findDebBySimlar(def devs, def name, def city){
|
||||
name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
|
||||
def last = null
|
||||
for(def d : devs){
|
||||
def alias1 = null
|
||||
if(!d.alias){
|
||||
alias1 = []
|
||||
}else{
|
||||
alias1 = d.alias
|
||||
}
|
||||
if(!alias1.contains(d.name)){
|
||||
alias1 << d.name
|
||||
}
|
||||
if(alias1){
|
||||
try {
|
||||
for(def n : alias1){
|
||||
n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
|
||||
if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){
|
||||
last = d
|
||||
break
|
||||
}
|
||||
}
|
||||
}catch(exception){
|
||||
println alias1
|
||||
}
|
||||
|
||||
}
|
||||
if(last){
|
||||
break
|
||||
}
|
||||
}
|
||||
return last
|
||||
}
|
||||
def findDevByName(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", name)
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevByAlias(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('alias', new BasicDBObject('$in',[name]))
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevBySimlar(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevByPark(prov, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put("type", "园区")
|
||||
return dev_mapping_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findCommunityByName(province, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province_name", province+"省")
|
||||
q.put("city_name", city+"市")
|
||||
q.put("name", name)
|
||||
def dbcursor = community_100_CONN.find(q)
|
||||
def list = []
|
||||
for(def d : dbcursor){
|
||||
list << d
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def findCommunityBySimlar(province, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province_name", province+"省")
|
||||
q.put("city_name", city+"市")
|
||||
q.put("name", new BasicDBObject('$regex', Pattern.compile('^'+name+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
def dbcursor = community_100_CONN.find(q)
|
||||
def list = []
|
||||
for(def d : dbcursor){
|
||||
list << d
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def findMinglujiDatas(city){
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("c_address",1)
|
||||
keys.put("c_city",1)
|
||||
keys.put("c_committee",1)
|
||||
keys.put("c_community",1)
|
||||
keys.put("c_company",1)
|
||||
keys.put("c_district",1)
|
||||
keys.put("c_street",1)
|
||||
keys.put("kind",1)
|
||||
keys.put("name",1)
|
||||
keys.put("province",1)
|
||||
def datas = mlj_COLL.find(new BasicDBObject("c_city", city), keys)
|
||||
def rets = []
|
||||
rets << ["省","城市","区域","街道","开发区名","类型","公司名","社区","居委","地址","拆分街道","拆分居委","匹配开发区名"]
|
||||
for(def d : datas){
|
||||
def street = d.c_street
|
||||
def splitStreet = ""
|
||||
if(street){
|
||||
if(street.contains(d.c_district)){
|
||||
def start = street.toString().lastIndexOf(d.c_district)+d.c_district.size()
|
||||
def end = street.toString().size()
|
||||
splitStreet = street.toString().substring(start, end)
|
||||
}
|
||||
}
|
||||
def splitCommunity = ""
|
||||
if(d.c_committee){
|
||||
if(d.c_committee.contains(d.c_district)){
|
||||
splitCommunity = d.c_committee.substring(d.c_committee.lastIndexOf(d.c_district)+d.c_district.size(), d.c_committee.toString().size())
|
||||
}
|
||||
}
|
||||
rets << [d.province, d.c_city, d.c_district, d.c_street, d.name, d.kind, d.c_company, d.c_community, d.c_committee, d.c_address, splitStreet, splitCommunity]
|
||||
}
|
||||
return rets
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -25,7 +25,7 @@ class FjwBigdataCommbuildtypeBatch extends BaseBatch{
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
//235为房价网内部计算均价的库
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-1
@@ -33,7 +33,7 @@ class FjwBigdataCommunityBatch extends BaseBatch{
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
//235为房价网内部计算均价的库
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-1
@@ -40,7 +40,7 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
|
||||
+4
-4
@@ -67,7 +67,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
|
||||
@@ -492,9 +492,9 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
|
||||
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
|
||||
def date = "20200101"//DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
||||
def startMdate = DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
|
||||
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
||||
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
+1
-1
@@ -34,7 +34,7 @@ class FjwGuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15";
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ class FjwPgsqlCommUtils {
|
||||
|
||||
public static def database = "fangjia"
|
||||
|
||||
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/"+database+"?stringtype=unspecified",
|
||||
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/"+database+"?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
public static def findAllDistrictIdsByOracle(def cityid){
|
||||
|
||||
+12
-1
@@ -36,7 +36,7 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
|
||||
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
@@ -265,6 +265,10 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
if(third.get("5i5j_esf_price")){
|
||||
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("fccs_esf_price")){
|
||||
price_map.fccs_avg_price = third.get("fccs_esf_price")
|
||||
}
|
||||
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
allRows << dataMap
|
||||
}
|
||||
@@ -311,6 +315,8 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
web = "房多多"
|
||||
}else if(k=="wiwj_avg_price"){
|
||||
web = "我爱我家"
|
||||
}else if(k=="fccs_avg_price"){
|
||||
web = "·¿²ú³¬ÊÐ"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
@@ -459,6 +465,11 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
|
||||
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
|
||||
}
|
||||
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
|
||||
+7
-7
@@ -33,11 +33,11 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
addFjwPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
// exportPkPrice()
|
||||
// addRegion()
|
||||
// addTradePrice(filePath)
|
||||
@@ -169,9 +169,9 @@ class AddPriceTest {
|
||||
def addCcbPrice(filepath){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(filepath,{ d->
|
||||
if(d['AJ']){
|
||||
def cityId = cityCodeDataMap.get(d['AK'])
|
||||
def map = findAvgPrice(d['AJ'], cityId)
|
||||
if(d['U']){
|
||||
def cityId = cityCodeDataMap.get(d['V'])
|
||||
def map = findAvgPrice(d['U'], cityId)
|
||||
if(map){
|
||||
d['F'] = map.avg_price
|
||||
// d['P'] = map.calc_type
|
||||
@@ -183,7 +183,7 @@ class AddPriceTest {
|
||||
|
||||
|
||||
def findAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202406 order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ class QiliangTest {
|
||||
// task.run()
|
||||
|
||||
def data = [
|
||||
'address': '江阴市澄杨路77号',
|
||||
'address': '景县开发区东区',
|
||||
'type': '1' //1:对公 0:对私
|
||||
]
|
||||
def s = task.getProxyResult(data);
|
||||
|
||||
+21
-8
@@ -15,6 +15,8 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.google.common.hash.BloomFilter
|
||||
import com.google.common.hash.Funnels
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
@@ -29,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
@@ -164,10 +167,17 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
'陵水黎族自治县', '随州市', '雅安市', '青岛市', '鞍山市', '韶关市', '马鞍山市', '驻马店市',
|
||||
'鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市',
|
||||
'黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市']
|
||||
|
||||
//"丽水市","杭州市",
|
||||
def runbuducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市"]
|
||||
for(def city : buducitys){
|
||||
if(runbuducitys.contains(city)){
|
||||
continue
|
||||
}
|
||||
def pois = findAllVillagePoiMap(city)
|
||||
def rows = createVillageData(city, pois)
|
||||
|
||||
|
||||
save(city, rows)
|
||||
// saveToMG(city, rows)
|
||||
println city +" "+pois.size()+" add:"+rows.size()
|
||||
@@ -182,8 +192,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
final CountDownLatch latch = new CountDownLatch(pois.size())
|
||||
// def townMap = findTownRange(city)
|
||||
def townMap = CommonDatautils.queryTownByCity(city)
|
||||
def poiKeys = findEsPoiByCity(city)
|
||||
|
||||
def poiKeysFilter = findEsPoiByCity(city)
|
||||
for(def i=0;i<pois.size();i++){
|
||||
def p = pois[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
@@ -192,7 +201,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
totalCount.addAndGet(1)
|
||||
// def obj = MAPPOI_CONN.findOne(new BasicDBObject("name", p.name).append("district", p.district).append("city", p.city))
|
||||
def key = p.city+"##"+p.district+"##"+p.name
|
||||
if(!poiKeys.contains(key)){
|
||||
if(!poiKeysFilter.mightContain(key)){
|
||||
def prov = ProvinceData.getProvince(p.city.replace("市",""))
|
||||
|
||||
def ret = createVillageData(prov, p, townMap)
|
||||
@@ -222,11 +231,13 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
//
|
||||
// })
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
def rets = new HashSet()
|
||||
// def rets = new HashSet()
|
||||
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi_single", queryBuilder, { String id, Map d->
|
||||
rets << d.city +"##"+d.district+"##"+d.name
|
||||
// rets << d.city +"##"+d.district+"##"+d.name
|
||||
filter.put(d.city +"##"+d.district+"##"+d.name)
|
||||
})
|
||||
return rets
|
||||
return filter
|
||||
}
|
||||
|
||||
def queryMapPoi(city){
|
||||
@@ -314,7 +325,6 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
// retMap.small_type = "村庄级地名"
|
||||
retMap.cat = map.std_tag
|
||||
|
||||
|
||||
def nameArr = map.std_tag.split(";|:")
|
||||
if(nameArr.size()>=1){
|
||||
retMap.type = nameArr[0]
|
||||
@@ -408,6 +418,9 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
}
|
||||
|
||||
def findAllVillagePoiMap(city){
|
||||
def q = new BasicDBObject("city", city);
|
||||
q.put("std_tag", new BasicDBObject('$exists', true))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
@@ -416,7 +429,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
keys.put("gcj02_x", 1)
|
||||
keys.put("gcj02_y", 1)
|
||||
keys.put("std_tag", 1)
|
||||
def list = BAIDU_POI_COLL.find(new BasicDBObject("city", city), keys).toList()
|
||||
def list = BAIDU_POI_COLL.find(q, keys).toList()
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user