Merge remote-tracking branch 'origin/master'

This commit is contained in:
majintao
2024-08-21 15:24:46 +08:00
21 changed files with 633 additions and 116 deletions
@@ -467,11 +467,15 @@ public class AddrSvcController {
} else {
map.put("addrVerify", "地址有效");
map.put("typeName", areaMap.get(map.get("typeCode")));
map.put("addrArea", addrArea(map, type));
map.put("countyLevelCity", countyLevelCity(map));
basicModel.setResult(map);
}
} else {
map.put("addrVerify", "地址有效");
map.put("typeName", areaMap.get(map.get("typeCode")));
map.put("addrArea", addrArea(map, type));
map.put("countyLevelCity", countyLevelCity(map));
basicModel.setResult(map);
}
@@ -498,6 +502,45 @@ public class AddrSvcController {
}
return mav;
}
/**
* 判断县级市
*/
public String countyLevelCity(Map<String, String> map) {
if (map.get("city") == null) {
return "";
}
if (map.get("city").endsWith("区")) {
return "否";
}
if (map.get("city").equals("东莞市") || map.get("city").equals("嘉峪关市") || map.get("city").equals("中山市") || map.get("city").equals("儋州市") ) {
return "否";
}
return "是";
}
/**
* 判断所属区域
*
* @param map
* @return
*/
public String addrArea(Map<String, String> map, String type) {
if (map.get("verifyFlag") == null ) {
return "";
}
if (map.get("verifyFlag").equals("非农")) {
return "城市区域";
}
if (map.get("verifyFlag").equals("农")) {
return "农村区域";
}
if ("2".equals(type) && "是".equals(map.get("countyGov"))) {
return "县城城区";
}
return "";
}
/**
* 涉农识别-新规则-上海征信
*
@@ -565,11 +608,15 @@ public class AddrSvcController {
basicModel.setResult(result);
} else {
map.put("addrVerify", "地址有效");
map.put("addrArea", addrArea(map, type));
map.put("countyLevelCity", countyLevelCity(map));
map.put("typeName", areaMap.get(map.get("typeCode")));
basicModel.setResult(map);
}
} else {
map.put("addrVerify", "地址有效");
map.put("addrArea", addrArea(map, type));
map.put("countyLevelCity", countyLevelCity(map));
map.put("typeName", areaMap.get(map.get("typeCode")));
basicModel.setResult(map);
}
@@ -703,11 +750,15 @@ public class AddrSvcController {
basicModel.setResult(result);
} else {
map.put("addrVerify", "地址有效");
map.put("addrArea", addrArea(map, type));
map.put("countyLevelCity", countyLevelCity(map));
map.put("typeName", areaMap.get(map.get("typeCode")));
basicModel.setResult(map);
}
} else {
map.put("addrVerify", "地址有效");
map.put("addrArea", addrArea(map, type));
map.put("countyLevelCity", countyLevelCity(map));
map.put("typeName", areaMap.get(map.get("typeCode")));
basicModel.setResult(map);
}
@@ -0,0 +1,40 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 将国信达及成交对标数据加入到第三方均价表,做为均价参考
*/
class AddThirdPriceSample {
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new AddThirdPriceSample()
task.run()
}
def run(){
for(def i=1;i<19;i++){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d->
if(d['A'] && d['C']){
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
if(dist){
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407)
def udata = new BasicDBObject("zhongyuan_esf_price", d['C'].toString().toDouble().toInteger())
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
}else{
println d['A'] +" 没查到"
}
}
}, i)
println(i)
}
}
}
@@ -70,12 +70,12 @@ class BigdataDealBatch extends BaseBatch{
//664b73800000000000000000 5-21
def citys = CcbCommonUtils.getRunningCitys(args)
// println "curDate:"+curDate
// def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
// addDealPriceMatch(city);
// }
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
println "curDate:"+curDate
def allCitys = CommonDatautils.findAllCity()
for(def city : allCitys){
addDealPriceMatch(city);
}
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
def rows = []
@@ -612,9 +612,9 @@ def findAllData(sqlInstance236, citycode){
////},0)
//
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d->
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\删除小区.xlsx",{d->
// if(d['A']){
// def dbo = CommonDatautils.findDistrictById(d['D'])
// def dbo = CommonDatautils.findDistrictById(d['A'])
// if(dbo){
// def udata = new BasicDBObject()
//// udata.put("name_cat", 11)
@@ -630,6 +630,38 @@ def findAllData(sqlInstance236, citycode){
// }
//},0)
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
if(d['A'] && d['B']){
def dbo = CommonDatautils.findDistrictById(d['A'])
if(dbo){
def alias = null
if(!dbo.alias){
alias = []
}else{
alias = dbo.alias
}
if(!alias.contains(d['B'])){
alias << d['B']
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", dbo._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{
println d['A']+" "+"别名已存在"
}
}else{
println dbo._id.toString() +" 没找到"
}
}
})
//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
//for(def c : citys){
// def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"])
@@ -1299,34 +1331,34 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['J']){
// def region = RegionData.formatRegion(d['A'], d['J'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
//
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['J']){
//// def region = RegionData.formatRegion(d['A'], d['J'])
//// if(region){
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
//// if(dist){
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['L'] = dist.status-dist.name_cat
//// }
//// }
//// if(d['J'] instanceof String){
// def dist = CommonDatautils.findDistrictById(d['J'])
// if(dist){
// d['A'] = dist.city
// d['B'] = dist.region
// d['C'] = dist.name
// d['L'] = dist.status-dist.name_cat
// d['K'] = dist.estate_type2
// d['L'] = dist.status
// d['M'] = dist.name_cat
//
//// d['H'] = dist.name
// }
// }
// if(d['J'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['J'])
if(dist){
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
d['K'] = dist.estate_type2
d['L'] = dist.status
d['M'] = dist.name_cat
// d['H'] = dist.name
}
// }
}
},0)
//// }
// }
//},0)
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
@@ -257,6 +257,9 @@ class ThirdPartyPriceBatch extends BaseBatch {
if(third.get("5i5j_esf_price")){
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
}
if(third.get("fccs_esf_price")){
price_map.fccs_avg_price = third.get("fccs_esf_price")
}
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
allRows << dataMap
}
@@ -303,6 +306,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
web = "房多多"
}else if(k=="wiwj_avg_price"){
web = "我爱我家"
}else if(k=="fccs_avg_price"){
web = "房产超市"
}
ret << sourceMap.get(web)
@@ -584,6 +589,10 @@ class ThirdPartyPriceBatch extends BaseBatch {
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
}
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
@@ -1212,6 +1212,7 @@ class CommonDatautils {
public static def findAllCount(String cat){
DBObject q = new BasicDBObject();
q.put("cat", cat);
q.put("status", 0)
return SEAWEED_MONGO_COLL.count(q)
}
@@ -0,0 +1,102 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.google.gson.JsonObject
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.jsoup.Jsoup
class BaiduBaikeTown {
DBCollection dzb_COLL = MongoUtils.getCollectionByName("crawl", "development_zone_bdbk", "192.168.0.206",27020)
static main(args) {
def task = new BaiduBaikeTown()
task.run()
}
def run(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",{row->
if(!row['E'] && row['A']=="郑州经济技术开发区"){
def q = new BasicDBObject()
q.put("desc",new BasicDBObject('$ne', 'null'))
q.put("name",row['A'])
def d = dzb_COLL.findOne(q)
def townMap = parse(d.desc, d.name)
row['D'] = townMap.town_str1
row['F'] = townMap.town_str2
// def ret = []
// ret << ["名称", "基本信息", "目录", "全文"]
// for(def d : dbcursor){
// def townMap = parse(d.desc, d.name)
// if(townMap.town){
// println d.name+" "+townMap.town
// }
// ret << [d.name, townMap.town, townMap.towns, townMap.town_str1, townMap.town_str2]
// }
}
})
// ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx", ret, "数据")
}
def parse(desc, name){
def map = [:]
if(desc){
def doc = Jsoup.parse(desc)
def town = JQueryUtils.selectText(".basicInfoBlock_Ex6IU div:contains(下辖地区) dd", doc)
def s = doc.toString()
def dataStr = DataHandleUtils.findMatchPattern(s,"PAGE_DATA=.*</script>").replace("PAGE_DATA= ","").replace("</script>","")
def jsonObj = JSONObject.fromObject(dataStr)
def str = DataHandleUtils.findMatchPattern(doc.text(),"下辖.*[街道|镇]、?和?")
def towns = []
if(jsonObj && jsonObj.catalog){
def catalogs = jsonObj.catalog
def index = null
for(def log : catalogs){
def curIndex = log.index
if(log.title=="行政区划"){
index = log.index
}
if(index && curIndex.toString().startsWith(index+"-")){
towns << log.title
}
}
}
map.town_str1 = str
if(town){
map.town = town
}else{
map.town = ""
}
if(towns){
map.towns = towns.join(",")
}else{
map.towns = ""
}
def str2 = DataHandleUtils.findMatchPattern(doc.text(),"开发区辖.*[,|。]")
map.town_str2 = str2
return map
}
map.town_str1 = ""
map.town_str2 = ""
map.town = ""
map.towns = ""
return map
}
}
@@ -25,9 +25,11 @@ class DevProcess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection DEV_MAPPING_CONN2 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping_2", MONGO100_URL)
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
@@ -54,7 +56,7 @@ class DevProcess {
// task.createDevMapping()
// task.deleteAddr()
// task.deleteAddr2()
// task.synchDevTo43()
// task.synchDevTo100()
// task.addNewDev2()
// task.addAlias()
@@ -71,6 +73,8 @@ class DevProcess {
// task.devToPark() //开发区下辖园区
// task.checkTown()
// task.repeatSave()
}
@@ -692,13 +696,13 @@ class DevProcess {
});
}
def synchDevTo43(){
def dbcursor = development_zone_CONN.find()
def synchDevTo100(){
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find()
def rets = []
for(def r : dbcursor){
rets << r
}
DEVELOPMENT_ZONE_43_COLL.insert(rets)
development_zone_CONN.insert(rets)
}
@@ -898,66 +902,89 @@ class DevProcess {
}
def devToPark(){
def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\开发区下辖街道-处理后表格.xlsx",0, 1)
def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",0, 1)
def rets = []
rets << ["开发区名称", "下辖街道", "城市", "区域"]
for(def d : datas){
if(d['E']){
def arrs = d['E'].split(",")
def q = new BasicDBObject().append("name", d['B'])
def arrs = d['E'].split("、")
def q = new BasicDBObject().append("name", d['A'])
def obj = development_zone_CONN.findOne(q)
if(obj){
for(def park : arrs){
rets << [d['B'], park, obj.city, obj.district]
rets << [d['A'], park, obj.city, obj.district]
}
}
}
}
ExcelUtils.write("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx", rets, "下辖街道")
ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx", rets, "下辖街道")
}
def checkTown(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx",{d->
if(!d['E']){
if(d['A']){
// if(d['A']=="长春高新技术产业开发区"){
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['A'])
if(jsonObject1) {
if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
def township = jsonObject1.get("township")
def district = jsonObject1.get("district")
d['E'] = "接口"
if(district && district!='null'){
d['F'] = district
}
if(township && township!='null'){
d['G'] = township
}
d['H'] = jsonObject1.get("townCode")
if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){
d['I'] = jsonObject1.get("provinceCode")
}
}
// }
}
// def q = new BasicDBObject()
// q.put("city", d['C'])
//// q.put("district", d['D'])
// q.put("mzb_district_name", d['D'])
// q.put("type", "乡镇街道")
// q.put("street", d['B'])
// def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
// if(obj){
// d['E'] = "可入库2"
// d['F'] = obj.district
// def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B'])
// if(jsonObject1) {
// if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
// def township = jsonObject1.get("township")
// def district = jsonObject1.get("district")
// d['E'] = "接口"
// if(district && district!='null'){
// d['F'] = district
// }
// if(township && township!='null'){
// d['G'] = township
// }
//
// d['H'] = jsonObject1.get("townCode")
// if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){
// d['I'] = jsonObject1.get("provinceCode")
// }
// }
// }
def q = new BasicDBObject()
q.put("city", d['C'])
// q.put("district", d['D'])
// q.put("mzb_district_name", d['D'])
q.put("type", "乡镇街道")
q.put('street',new BasicDBObject('$regex', Pattern.compile(d['B'].toString().trim()+'.*$', Pattern.CASE_INSENSITIVE)))
// q.put("street", d['B'])
// q.put('alias', new BasicDBObject('$in',[d['B']]))
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
d['E'] = "可入库-忽略区域模糊2"
d['F'] = obj.district
d['G'] = obj.street
}
}
}
},1)
},0)
}
def repeatSave(){
def dbcursor = DEV_MAPPING_CONN.find(new BasicDBObject())
def list = []
for(def map : dbcursor){
def columns = ["_id","province","city","district","name","type","address","dev_district", "township","park"]
def udata = new BasicDBObject()
map.each{k,v->
if(columns.contains(k)){
if(v instanceof String){
if(k=="_id"){
udata.put("_id", new ObjectId(v))
}else{
udata.put(k, v)
}
}
}
}
list << udata
}
DEV_MAPPING_CONN2.insert(list)
}
}
@@ -29,7 +29,7 @@ class DevUpdateTestBatch {
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\文心一言获取开发区下辖园区及街道.xlsx"//开发区-园区.xlsx
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
task.addDevAddress(path)
@@ -137,26 +137,27 @@ class DevUpdateTestBatch {
DEV_MAPPING_100_CONN.save(o)
o.id = o._id.toString()
o.remove("_id")
o.remove("c_date")
addEs(o, "192.168.0.89", "dev_mapping")
try{
addEs(o, "192.168.0.52", "dev_mapping")
}catch(Exception e){
println "52 es error: "+e.printStackTrace()
}
// o.id = o._id.toString()
// o.remove("_id")
// o.remove("c_date")
// addEs(o, "192.168.0.89", "dev_mapping")
// try{
// addEs(o, "192.168.0.52", "dev_mapping")
// }catch(Exception e){
// println "52 es error: "+e.printStackTrace()
// }
d['D'] = "已完成"
}else{
d['D'] = "存在"
d['D'] = "重复"
println d['A'] +" "+ d['C']+" 重复"
}
}else{
d['D'] = "不存在"
println d['A'] +" 不存在"
}
}
},2)
},1)
}
@@ -40,11 +40,11 @@ class GXDdistrictMatch {
def addOldMatch(){
//老匹配
def fileName = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"//11城重新匹配.xlsx
def fileName = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"//11城重新匹配.xlsx
ExcelUtils.multiThreadingModify(fileName, { d ->
println d['C']
if(!d['AJ'] && d['I']){
def jsonObject = OpenApiClient.addressMatch(d['C'], d['I'], "district", false,null, d['D'])
def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D'])
if (jsonObject != null && !jsonObject.containsKey("result")) {
d['AJ'] = jsonObject.get("id")
d['AK'] = jsonObject.get("city")
@@ -0,0 +1,230 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.regex.Pattern
/**
* 名录集
*/
class MinglujiData {
DBCollection mlj_COLL = MongoUtils.getCollectionByName("crawl", "kfq_mingluji_com", "192.168.0.206",27020)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
static main(args) {
def task = new MinglujiData()
task.run()
}
def run(){
// exportDatas("湖州")
// matchDev("浙江省","湖州")
matchCommunity("湖州")
}
def exportDatas(city){
def datas = findMinglujiDatas(city)
ExcelUtils.write("D:\\数据\\数据建设\\名录集\\${city}.xlsx", datas, "数据")
}
def matchDev(prov, city){
def devs = loadAllDevs(prov)
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d->
if(d['E'] && !d['N'] ){
def obj = findDevByName(d['E'])
if(!obj){
obj = findDevByAlias(d['E'])
if(!obj){
obj = findDebBySimlar(devs, d['E'], d['B'])
if(!obj){
obj = findDevByPark(d['A'], d['B'], d['E'])
if (!obj) {
}else{
d['N'] = "园区"
}
}else{
d['N'] = "相似"
}
}else{
d['N'] = "别名"
}
}else{
d['N'] = "相同"
}
if(obj){
d['M'] = obj.name
d['O'] = obj.city
d['P'] = obj.district
}
}
})
}
def matchCommunity(city){
ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}-没配居委1.xlsx",{d->
if(d['A'] && d['B'] && d['L'] && !d['Q']){
def list = findCommunityByName(d['A'], d['B'], d['L'])
if(!list){
list = findCommunityBySimlar(d['A'], d['B'], d['L'])
if(!list){
}else{
d['T'] = "包含"
}
}else{
d['T'] = "相同"
}
if(list){
if(list.size()==1){
def obj = list[0]
d['Q'] = obj.name
d['R'] = obj.street_name
}else if(list.size()>=2){
for(def l : list){
if(d['C']==l.area_name){
d['Q'] = l.name
d['R'] = l.street_name
break;
}
}
d['S'] = list.size()
}
}
}
}, 0)
}
def loadAllDevs(prov){
def q = new BasicDBObject()
q.put("province", prov)
return development_zone_CONN.find(q).toList()
}
def findDebBySimlar(def devs, def name, def city){
name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
def last = null
for(def d : devs){
def alias1 = null
if(!d.alias){
alias1 = []
}else{
alias1 = d.alias
}
if(!alias1.contains(d.name)){
alias1 << d.name
}
if(alias1){
try {
for(def n : alias1){
n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){
last = d
break
}
}
}catch(exception){
println alias1
}
}
if(last){
break
}
}
return last
}
def findDevByName(name){
def q = new BasicDBObject()
q.put("name", name)
return development_zone_CONN.findOne(q)
}
def findDevByAlias(name){
def q = new BasicDBObject()
q.put('alias', new BasicDBObject('$in',[name]))
return development_zone_CONN.findOne(q)
}
def findDevBySimlar(name){
def q = new BasicDBObject()
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
return development_zone_CONN.findOne(q)
}
def findDevByPark(prov, city, name){
def q = new BasicDBObject()
q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put("type", "园区")
return dev_mapping_CONN.findOne(q)
}
def findCommunityByName(province, city, name){
def q = new BasicDBObject()
q.put("province_name", province+"省")
q.put("city_name", city+"市")
q.put("name", name)
def dbcursor = community_100_CONN.find(q)
def list = []
for(def d : dbcursor){
list << d
}
return list
}
def findCommunityBySimlar(province, city, name){
def q = new BasicDBObject()
q.put("province_name", province+"省")
q.put("city_name", city+"市")
q.put("name", new BasicDBObject('$regex', Pattern.compile('^'+name+'.*$', Pattern.CASE_INSENSITIVE)))
def dbcursor = community_100_CONN.find(q)
def list = []
for(def d : dbcursor){
list << d
}
return list
}
def findMinglujiDatas(city){
def keys = new BasicDBObject()
keys.put("c_address",1)
keys.put("c_city",1)
keys.put("c_committee",1)
keys.put("c_community",1)
keys.put("c_company",1)
keys.put("c_district",1)
keys.put("c_street",1)
keys.put("kind",1)
keys.put("name",1)
keys.put("province",1)
def datas = mlj_COLL.find(new BasicDBObject("c_city", city), keys)
def rets = []
rets << ["省","城市","区域","街道","开发区名","类型","公司名","社区","居委","地址","拆分街道","拆分居委","匹配开发区名"]
for(def d : datas){
def street = d.c_street
def splitStreet = ""
if(street){
if(street.contains(d.c_district)){
def start = street.toString().lastIndexOf(d.c_district)+d.c_district.size()
def end = street.toString().size()
splitStreet = street.toString().substring(start, end)
}
}
def splitCommunity = ""
if(d.c_committee){
if(d.c_committee.contains(d.c_district)){
splitCommunity = d.c_committee.substring(d.c_committee.lastIndexOf(d.c_district)+d.c_district.size(), d.c_committee.toString().size())
}
}
rets << [d.province, d.c_city, d.c_district, d.c_street, d.name, d.kind, d.c_company, d.c_community, d.c_committee, d.c_address, splitStreet, splitCommunity]
}
return rets
}
}
@@ -25,7 +25,7 @@ class FjwBigdataCommbuildtypeBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
//235为房价网内部计算均价的库
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
@@ -33,7 +33,7 @@ class FjwBigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
//235为房价网内部计算均价的库
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
@@ -40,7 +40,7 @@ class FjwBigdataDealBatch extends BaseBatch{
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
@@ -67,7 +67,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
@@ -492,9 +492,9 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
def date = "20200101"//DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
@@ -34,7 +34,7 @@ class FjwGuiDingPriceBatch extends BaseBatch{
def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15";
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
@@ -13,7 +13,7 @@ class FjwPgsqlCommUtils {
public static def database = "fangjia"
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/"+database+"?stringtype=unspecified",
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/"+database+"?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
public static def findAllDistrictIdsByOracle(def cityid){
@@ -36,7 +36,7 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
@@ -265,6 +265,10 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
if(third.get("5i5j_esf_price")){
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
}
if(third.get("fccs_esf_price")){
price_map.fccs_avg_price = third.get("fccs_esf_price")
}
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
allRows << dataMap
}
@@ -311,6 +315,8 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
web = "房多多"
}else if(k=="wiwj_avg_price"){
web = "我爱我家"
}else if(k=="fccs_avg_price"){
web = "·¿²ú³¬ÊÐ"
}
ret << sourceMap.get(web)
@@ -459,6 +465,11 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
}
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
@@ -33,11 +33,11 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"
def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"
// addGXDPrice(filePath)
// addCcbPrice(filePath)
addCcbPrice(filePath)
// addThirdPrice(filePath)
addFjwPrice(filePath)
// addFjwPrice(filePath)
// exportPkPrice()
// addRegion()
// addTradePrice(filePath)
@@ -169,9 +169,9 @@ class AddPriceTest {
def addCcbPrice(filepath){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(filepath,{ d->
if(d['AJ']){
def cityId = cityCodeDataMap.get(d['AK'])
def map = findAvgPrice(d['AJ'], cityId)
if(d['U']){
def cityId = cityCodeDataMap.get(d['V'])
def map = findAvgPrice(d['U'], cityId)
if(map){
d['F'] = map.avg_price
// d['P'] = map.calc_type
@@ -183,7 +183,7 @@ class AddPriceTest {
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202406 order by gujiabds_clc_yrmo_val desc limit 1"
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
@@ -27,7 +27,7 @@ class QiliangTest {
// task.run()
def data = [
'address': '江阴市澄杨路77号',
'address': '景县开发区东区',
'type': '1' //1:对公 0:对私
]
def s = task.getProxyResult(data);
@@ -15,6 +15,8 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
@@ -29,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.nio.charset.StandardCharsets
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -164,10 +167,17 @@ class GaoDePoiDataProcess extends BaseBatch{
'陵水黎族自治县', '随州市', '雅安市', '青岛市', '鞍山市', '韶关市', '马鞍山市', '驻马店市',
'鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市',
'黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市']
//"丽水市","杭州市",
def runbuducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市"]
for(def city : buducitys){
if(runbuducitys.contains(city)){
continue
}
def pois = findAllVillagePoiMap(city)
def rows = createVillageData(city, pois)
save(city, rows)
// saveToMG(city, rows)
println city +" "+pois.size()+" add:"+rows.size()
@@ -182,8 +192,7 @@ class GaoDePoiDataProcess extends BaseBatch{
final CountDownLatch latch = new CountDownLatch(pois.size())
// def townMap = findTownRange(city)
def townMap = CommonDatautils.queryTownByCity(city)
def poiKeys = findEsPoiByCity(city)
def poiKeysFilter = findEsPoiByCity(city)
for(def i=0;i<pois.size();i++){
def p = pois[i]
fixedThreadPool.execute(new Runnable() {
@@ -192,7 +201,7 @@ class GaoDePoiDataProcess extends BaseBatch{
totalCount.addAndGet(1)
// def obj = MAPPOI_CONN.findOne(new BasicDBObject("name", p.name).append("district", p.district).append("city", p.city))
def key = p.city+"##"+p.district+"##"+p.name
if(!poiKeys.contains(key)){
if(!poiKeysFilter.mightContain(key)){
def prov = ProvinceData.getProvince(p.city.replace("市",""))
def ret = createVillageData(prov, p, townMap)
@@ -222,11 +231,13 @@ class GaoDePoiDataProcess extends BaseBatch{
//
// })
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
def rets = new HashSet()
// def rets = new HashSet()
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi_single", queryBuilder, { String id, Map d->
rets << d.city +"##"+d.district+"##"+d.name
// rets << d.city +"##"+d.district+"##"+d.name
filter.put(d.city +"##"+d.district+"##"+d.name)
})
return rets
return filter
}
def queryMapPoi(city){
@@ -314,7 +325,6 @@ class GaoDePoiDataProcess extends BaseBatch{
// retMap.small_type = "村庄级地名"
retMap.cat = map.std_tag
def nameArr = map.std_tag.split(";|:")
if(nameArr.size()>=1){
retMap.type = nameArr[0]
@@ -408,6 +418,9 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def findAllVillagePoiMap(city){
def q = new BasicDBObject("city", city);
q.put("std_tag", new BasicDBObject('$exists', true))
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("district", 1)
@@ -416,7 +429,7 @@ class GaoDePoiDataProcess extends BaseBatch{
keys.put("gcj02_x", 1)
keys.put("gcj02_y", 1)
keys.put("std_tag", 1)
def list = BAIDU_POI_COLL.find(new BasicDBObject("city", city), keys).toList()
def list = BAIDU_POI_COLL.find(q, keys).toList()
return list;
}