Merge branch 'script_gj' into 'master'

update

See merge request !909
This commit is contained in:
2024-08-16 10:19:20 +08:00
6 changed files with 167 additions and 62 deletions
@@ -1212,6 +1212,7 @@ class CommonDatautils {
public static def findAllCount(String cat){
DBObject q = new BasicDBObject();
q.put("cat", cat);
q.put("status", 0)
return SEAWEED_MONGO_COLL.count(q)
}
@@ -0,0 +1,102 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.google.gson.JsonObject
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.jsoup.Jsoup
class BaiduBaikeTown {
DBCollection dzb_COLL = MongoUtils.getCollectionByName("crawl", "development_zone_bdbk", "192.168.0.206",27020)
static main(args) {
def task = new BaiduBaikeTown()
task.run()
}
def run(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",{row->
if(!row['E'] && row['A']=="郑州经济技术开发区"){
def q = new BasicDBObject()
q.put("desc",new BasicDBObject('$ne', 'null'))
q.put("name",row['A'])
def d = dzb_COLL.findOne(q)
def townMap = parse(d.desc, d.name)
row['D'] = townMap.town_str1
row['F'] = townMap.town_str2
// def ret = []
// ret << ["名称", "基本信息", "目录", "全文"]
// for(def d : dbcursor){
// def townMap = parse(d.desc, d.name)
// if(townMap.town){
// println d.name+" "+townMap.town
// }
// ret << [d.name, townMap.town, townMap.towns, townMap.town_str1, townMap.town_str2]
// }
}
})
// ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx", ret, "数据")
}
def parse(desc, name){
def map = [:]
if(desc){
def doc = Jsoup.parse(desc)
def town = JQueryUtils.selectText(".basicInfoBlock_Ex6IU div:contains(下辖地区) dd", doc)
def s = doc.toString()
def dataStr = DataHandleUtils.findMatchPattern(s,"PAGE_DATA=.*</script>").replace("PAGE_DATA= ","").replace("</script>","")
def jsonObj = JSONObject.fromObject(dataStr)
def str = DataHandleUtils.findMatchPattern(doc.text(),"下辖.*[街道|镇]、?和?")
def towns = []
if(jsonObj && jsonObj.catalog){
def catalogs = jsonObj.catalog
def index = null
for(def log : catalogs){
def curIndex = log.index
if(log.title=="行政区划"){
index = log.index
}
if(index && curIndex.toString().startsWith(index+"-")){
towns << log.title
}
}
}
map.town_str1 = str
if(town){
map.town = town
}else{
map.town = ""
}
if(towns){
map.towns = towns.join(",")
}else{
map.towns = ""
}
def str2 = DataHandleUtils.findMatchPattern(doc.text(),"开发区辖.*[,|。]")
map.town_str2 = str2
return map
}
map.town_str1 = ""
map.town_str2 = ""
map.town = ""
map.towns = ""
return map
}
}
@@ -37,7 +37,7 @@ class DevProcess {
def task = new DevProcess();
// task.run()
// task.parseCityAndDist()
task.devSplitRoad()
// task.devSplitRoad()
// task.devNameSplitRoad()
// task.addFlag()
// task.matchDev()
@@ -69,7 +69,7 @@ class DevProcess {
// task.addKfc()
// task.devToPark() //开发区下辖园区
// task.checkTown()
task.checkTown()
}
@@ -898,66 +898,67 @@ class DevProcess {
}
def devToPark(){
def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\开发区下辖街道-处理后表格.xlsx",0, 1)
def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",0, 1)
def rets = []
rets << ["开发区名称", "下辖街道", "城市", "区域"]
for(def d : datas){
if(d['E']){
def arrs = d['E'].split(",")
def q = new BasicDBObject().append("name", d['B'])
def arrs = d['E'].split("、")
def q = new BasicDBObject().append("name", d['A'])
def obj = development_zone_CONN.findOne(q)
if(obj){
for(def park : arrs){
rets << [d['B'], park, obj.city, obj.district]
rets << [d['A'], park, obj.city, obj.district]
}
}
}
}
ExcelUtils.write("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx", rets, "下辖街道")
ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx", rets, "下辖街道")
}
def checkTown(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx",{d->
if(!d['E']){
if(d['A']){
// if(d['A']=="长春高新技术产业开发区"){
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['A'])
if(jsonObject1) {
if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
def township = jsonObject1.get("township")
def district = jsonObject1.get("district")
d['E'] = "接口"
if(district && district!='null'){
d['F'] = district
}
if(township && township!='null'){
d['G'] = township
}
d['H'] = jsonObject1.get("townCode")
if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){
d['I'] = jsonObject1.get("provinceCode")
}
}
// }
}
// def q = new BasicDBObject()
// q.put("city", d['C'])
//// q.put("district", d['D'])
// q.put("mzb_district_name", d['D'])
// q.put("type", "乡镇街道")
// q.put("street", d['B'])
// def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
// if(obj){
// d['E'] = "可入库2"
// d['F'] = obj.district
// def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B'])
// if(jsonObject1) {
// if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
// def township = jsonObject1.get("township")
// def district = jsonObject1.get("district")
// d['E'] = "接口"
// if(district && district!='null'){
// d['F'] = district
// }
// if(township && township!='null'){
// d['G'] = township
// }
//
// d['H'] = jsonObject1.get("townCode")
// if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){
// d['I'] = jsonObject1.get("provinceCode")
// }
// }
// }
def q = new BasicDBObject()
q.put("city", d['C'])
// q.put("district", d['D'])
// q.put("mzb_district_name", d['D'])
q.put("type", "乡镇街道")
q.put('street',new BasicDBObject('$regex', Pattern.compile(d['B'].toString().trim()+'.*$', Pattern.CASE_INSENSITIVE)))
// q.put("street", d['B'])
// q.put('alias', new BasicDBObject('$in',[d['B']]))
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
d['E'] = "可入库-忽略区域模糊2"
d['F'] = obj.district
d['G'] = obj.street
}
}
}
},1)
},0)
}
}
@@ -29,7 +29,7 @@ class DevUpdateTestBatch {
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\文心一言获取开发区下辖园区及街道.xlsx"//开发区-园区.xlsx
def path = "D:\\数据\\数据建设\\开发区\\开发区下辖园区及街道\\前瞻网开发区下辖园区入库.xlsx"//开发区-园区.xlsx
// task.addDev(path)
task.addDevAddress(path)
@@ -137,26 +137,27 @@ class DevUpdateTestBatch {
DEV_MAPPING_100_CONN.save(o)
o.id = o._id.toString()
o.remove("_id")
o.remove("c_date")
addEs(o, "192.168.0.89", "dev_mapping")
try{
addEs(o, "192.168.0.52", "dev_mapping")
}catch(Exception e){
println "52 es error: "+e.printStackTrace()
}
// o.id = o._id.toString()
// o.remove("_id")
// o.remove("c_date")
// addEs(o, "192.168.0.89", "dev_mapping")
// try{
// addEs(o, "192.168.0.52", "dev_mapping")
// }catch(Exception e){
// println "52 es error: "+e.printStackTrace()
// }
d['D'] = "已完成"
}else{
d['D'] = "存在"
d['D'] = "重复"
println d['A'] +" "+ d['C']+" 重复"
}
}else{
d['D'] = "不存在"
println d['A'] +" 不存在"
}
}
},2)
},0)
}
@@ -40,11 +40,11 @@ class GXDdistrictMatch {
def addOldMatch(){
//老匹配
def fileName = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"//11城重新匹配.xlsx
def fileName = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"//11城重新匹配.xlsx
ExcelUtils.multiThreadingModify(fileName, { d ->
println d['C']
if(!d['AJ'] && d['I']){
def jsonObject = OpenApiClient.addressMatch(d['C'], d['I'], "district", false,null, d['D'])
def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D'])
if (jsonObject != null && !jsonObject.containsKey("result")) {
d['AJ'] = jsonObject.get("id")
d['AK'] = jsonObject.get("city")
@@ -33,11 +33,11 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"
def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"
// addGXDPrice(filePath)
// addCcbPrice(filePath)
addCcbPrice(filePath)
// addThirdPrice(filePath)
addFjwPrice(filePath)
// addFjwPrice(filePath)
// exportPkPrice()
// addRegion()
// addTradePrice(filePath)
@@ -169,9 +169,9 @@ class AddPriceTest {
def addCcbPrice(filepath){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(filepath,{ d->
if(d['AJ']){
def cityId = cityCodeDataMap.get(d['AK'])
def map = findAvgPrice(d['AJ'], cityId)
if(d['U']){
def cityId = cityCodeDataMap.get(d['V'])
def map = findAvgPrice(d['U'], cityId)
if(map){
d['F'] = map.avg_price
// d['P'] = map.calc_type
@@ -183,7 +183,7 @@ class AddPriceTest {
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202406 order by gujiabds_clc_yrmo_val desc limit 1"
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc