update
This commit is contained in:
+1
@@ -1212,6 +1212,7 @@ class CommonDatautils {
|
||||
public static def findAllCount(String cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("cat", cat);
|
||||
q.put("status", 0)
|
||||
return SEAWEED_MONGO_COLL.count(q)
|
||||
}
|
||||
|
||||
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.google.gson.JsonObject
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.jsoup.Jsoup
|
||||
|
||||
class BaiduBaikeTown {
|
||||
|
||||
DBCollection dzb_COLL = MongoUtils.getCollectionByName("crawl", "development_zone_bdbk", "192.168.0.206",27020)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new BaiduBaikeTown()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",{row->
|
||||
if(!row['E'] && row['A']=="郑州经济技术开发区"){
|
||||
def q = new BasicDBObject()
|
||||
q.put("desc",new BasicDBObject('$ne', 'null'))
|
||||
q.put("name",row['A'])
|
||||
def d = dzb_COLL.findOne(q)
|
||||
def townMap = parse(d.desc, d.name)
|
||||
row['D'] = townMap.town_str1
|
||||
row['F'] = townMap.town_str2
|
||||
|
||||
// def ret = []
|
||||
// ret << ["名称", "基本信息", "目录", "全文"]
|
||||
// for(def d : dbcursor){
|
||||
// def townMap = parse(d.desc, d.name)
|
||||
// if(townMap.town){
|
||||
// println d.name+" "+townMap.town
|
||||
// }
|
||||
// ret << [d.name, townMap.town, townMap.towns, townMap.town_str1, townMap.town_str2]
|
||||
// }
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx", ret, "数据")
|
||||
}
|
||||
|
||||
def parse(desc, name){
|
||||
def map = [:]
|
||||
if(desc){
|
||||
def doc = Jsoup.parse(desc)
|
||||
def town = JQueryUtils.selectText(".basicInfoBlock_Ex6IU div:contains(下辖地区) dd", doc)
|
||||
|
||||
def s = doc.toString()
|
||||
def dataStr = DataHandleUtils.findMatchPattern(s,"PAGE_DATA=.*</script>").replace("PAGE_DATA= ","").replace("</script>","")
|
||||
def jsonObj = JSONObject.fromObject(dataStr)
|
||||
|
||||
def str = DataHandleUtils.findMatchPattern(doc.text(),"下辖.*[街道|镇]、?和?")
|
||||
def towns = []
|
||||
if(jsonObj && jsonObj.catalog){
|
||||
def catalogs = jsonObj.catalog
|
||||
def index = null
|
||||
for(def log : catalogs){
|
||||
def curIndex = log.index
|
||||
if(log.title=="行政区划"){
|
||||
index = log.index
|
||||
}
|
||||
if(index && curIndex.toString().startsWith(index+"-")){
|
||||
towns << log.title
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
map.town_str1 = str
|
||||
if(town){
|
||||
map.town = town
|
||||
}else{
|
||||
map.town = ""
|
||||
}
|
||||
|
||||
if(towns){
|
||||
map.towns = towns.join(",")
|
||||
}else{
|
||||
map.towns = ""
|
||||
}
|
||||
|
||||
|
||||
def str2 = DataHandleUtils.findMatchPattern(doc.text(),"开发区辖.*[,|。]")
|
||||
map.town_str2 = str2
|
||||
return map
|
||||
}
|
||||
map.town_str1 = ""
|
||||
map.town_str2 = ""
|
||||
map.town = ""
|
||||
map.towns = ""
|
||||
return map
|
||||
}
|
||||
}
|
||||
+42
-41
@@ -37,7 +37,7 @@ class DevProcess {
|
||||
def task = new DevProcess();
|
||||
// task.run()
|
||||
// task.parseCityAndDist()
|
||||
task.devSplitRoad()
|
||||
// task.devSplitRoad()
|
||||
// task.devNameSplitRoad()
|
||||
// task.addFlag()
|
||||
// task.matchDev()
|
||||
@@ -69,7 +69,7 @@ class DevProcess {
|
||||
|
||||
// task.addKfc()
|
||||
// task.devToPark() //开发区下辖园区
|
||||
// task.checkTown()
|
||||
task.checkTown()
|
||||
|
||||
}
|
||||
|
||||
@@ -898,66 +898,67 @@ class DevProcess {
|
||||
}
|
||||
|
||||
def devToPark(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\开发区下辖街道-处理后表格.xlsx",0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",0, 1)
|
||||
def rets = []
|
||||
rets << ["开发区名称", "下辖街道", "城市", "区域"]
|
||||
for(def d : datas){
|
||||
if(d['E']){
|
||||
def arrs = d['E'].split(",")
|
||||
def q = new BasicDBObject().append("name", d['B'])
|
||||
def arrs = d['E'].split("、")
|
||||
def q = new BasicDBObject().append("name", d['A'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
for(def park : arrs){
|
||||
rets << [d['B'], park, obj.city, obj.district]
|
||||
rets << [d['A'], park, obj.city, obj.district]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx", rets, "下辖街道")
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx", rets, "下辖街道")
|
||||
}
|
||||
|
||||
|
||||
def checkTown(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d->
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx",{d->
|
||||
|
||||
if(!d['E']){
|
||||
if(d['A']){
|
||||
// if(d['A']=="长春高新技术产业开发区"){
|
||||
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['A'])
|
||||
if(jsonObject1) {
|
||||
if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
|
||||
def township = jsonObject1.get("township")
|
||||
def district = jsonObject1.get("district")
|
||||
d['E'] = "接口"
|
||||
if(district && district!='null'){
|
||||
d['F'] = district
|
||||
}
|
||||
if(township && township!='null'){
|
||||
d['G'] = township
|
||||
}
|
||||
|
||||
d['H'] = jsonObject1.get("townCode")
|
||||
if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){
|
||||
d['I'] = jsonObject1.get("provinceCode")
|
||||
}
|
||||
}
|
||||
// }
|
||||
}
|
||||
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city", d['C'])
|
||||
//// q.put("district", d['D'])
|
||||
// q.put("mzb_district_name", d['D'])
|
||||
// q.put("type", "乡镇街道")
|
||||
// q.put("street", d['B'])
|
||||
// def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
// if(obj){
|
||||
// d['E'] = "可入库2"
|
||||
// d['F'] = obj.district
|
||||
// def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B'])
|
||||
// if(jsonObject1) {
|
||||
// if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
|
||||
// def township = jsonObject1.get("township")
|
||||
// def district = jsonObject1.get("district")
|
||||
// d['E'] = "接口"
|
||||
// if(district && district!='null'){
|
||||
// d['F'] = district
|
||||
// }
|
||||
// if(township && township!='null'){
|
||||
// d['G'] = township
|
||||
// }
|
||||
//
|
||||
// d['H'] = jsonObject1.get("townCode")
|
||||
// if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){
|
||||
// d['I'] = jsonObject1.get("provinceCode")
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['C'])
|
||||
// q.put("district", d['D'])
|
||||
// q.put("mzb_district_name", d['D'])
|
||||
q.put("type", "乡镇街道")
|
||||
q.put('street',new BasicDBObject('$regex', Pattern.compile(d['B'].toString().trim()+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
// q.put("street", d['B'])
|
||||
// q.put('alias', new BasicDBObject('$in',[d['B']]))
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(obj){
|
||||
d['E'] = "可入库-忽略区域模糊2"
|
||||
d['F'] = obj.district
|
||||
d['G'] = obj.street
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},1)
|
||||
},0)
|
||||
}
|
||||
}
|
||||
|
||||
+13
-12
@@ -29,7 +29,7 @@ class DevUpdateTestBatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new DevUpdateTestBatch()
|
||||
def path = "D:\\数据\\数据建设\\开发区\\文心一言获取开发区下辖园区及街道.xlsx"//开发区-园区.xlsx
|
||||
def path = "D:\\数据\\数据建设\\开发区\\开发区下辖园区及街道\\前瞻网开发区下辖园区入库.xlsx"//开发区-园区.xlsx
|
||||
// task.addDev(path)
|
||||
task.addDevAddress(path)
|
||||
|
||||
@@ -137,26 +137,27 @@ class DevUpdateTestBatch {
|
||||
DEV_MAPPING_100_CONN.save(o)
|
||||
|
||||
|
||||
o.id = o._id.toString()
|
||||
o.remove("_id")
|
||||
o.remove("c_date")
|
||||
addEs(o, "192.168.0.89", "dev_mapping")
|
||||
try{
|
||||
addEs(o, "192.168.0.52", "dev_mapping")
|
||||
}catch(Exception e){
|
||||
println "52 es error: "+e.printStackTrace()
|
||||
}
|
||||
// o.id = o._id.toString()
|
||||
// o.remove("_id")
|
||||
// o.remove("c_date")
|
||||
// addEs(o, "192.168.0.89", "dev_mapping")
|
||||
// try{
|
||||
// addEs(o, "192.168.0.52", "dev_mapping")
|
||||
// }catch(Exception e){
|
||||
// println "52 es error: "+e.printStackTrace()
|
||||
// }
|
||||
|
||||
d['D'] = "已完成"
|
||||
}else{
|
||||
d['D'] = "存在"
|
||||
d['D'] = "重复"
|
||||
println d['A'] +" "+ d['C']+" 重复"
|
||||
}
|
||||
}else{
|
||||
d['D'] = "不存在"
|
||||
println d['A'] +" 不存在"
|
||||
}
|
||||
}
|
||||
},2)
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -40,11 +40,11 @@ class GXDdistrictMatch {
|
||||
|
||||
def addOldMatch(){
|
||||
//老匹配
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"//11城重新匹配.xlsx
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"//11城重新匹配.xlsx
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
println d['C']
|
||||
if(!d['AJ'] && d['I']){
|
||||
def jsonObject = OpenApiClient.addressMatch(d['C'], d['I'], "district", false,null, d['D'])
|
||||
def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D'])
|
||||
if (jsonObject != null && !jsonObject.containsKey("result")) {
|
||||
d['AJ'] = jsonObject.get("id")
|
||||
d['AK'] = jsonObject.get("city")
|
||||
|
||||
+7
-7
@@ -33,11 +33,11 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
addFjwPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
// exportPkPrice()
|
||||
// addRegion()
|
||||
// addTradePrice(filePath)
|
||||
@@ -169,9 +169,9 @@ class AddPriceTest {
|
||||
def addCcbPrice(filepath){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(filepath,{ d->
|
||||
if(d['AJ']){
|
||||
def cityId = cityCodeDataMap.get(d['AK'])
|
||||
def map = findAvgPrice(d['AJ'], cityId)
|
||||
if(d['U']){
|
||||
def cityId = cityCodeDataMap.get(d['V'])
|
||||
def map = findAvgPrice(d['U'], cityId)
|
||||
if(map){
|
||||
d['F'] = map.avg_price
|
||||
// d['P'] = map.calc_type
|
||||
@@ -183,7 +183,7 @@ class AddPriceTest {
|
||||
|
||||
|
||||
def findAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202406 order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
|
||||
Reference in New Issue
Block a user