Merge remote-tracking branch 'origin/master'

This commit is contained in:
majintao
2024-08-22 13:46:42 +08:00
8 changed files with 183 additions and 59 deletions
@@ -7,7 +7,7 @@ public interface Constants {
String FANGJIA_CRAWL_MONGO_URI = "mongodb://192.168.0.235:27018";
String LOG_MONGO_URI = "mongodb://192.168.0.42:52002";
String SEABIRD_MONGO_URI = "mongodb://192.168.0.74:57007";
String SEAWATER_MONGO_URI = "mongodb://192.168.0.206:27020";
String SEAWATER_MONGO_URI = "mongodb://root:Goojia%40206!!#@192.168.0.206:27020";
String MSVC_FANG_JIA_DATA_SERVICE_BASE_PATH ="http://192.168.0.100:38080/FangJiaDataService";
}
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年6月深圳二手房成交.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年7月深圳二手房成交.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -33,7 +33,7 @@ class DevProcess {
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020)
// DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020)
static main(args) {
def task = new DevProcess();
@@ -31,9 +31,12 @@ class DevUpdateTestBatch {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
task.addDevAddress(path)
// task.addDevAddress(path)
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
}
def addDev(path){
@@ -1,9 +1,11 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import java.util.regex.Pattern
@@ -19,15 +21,25 @@ class MinglujiData {
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
DBCollection company_100_CONN = MongoUtils.getCollectionByName2("fangjia", "company", MONGO100_URL)
static main(args) {
def task = new MinglujiData()
task.run()
}
def run(){
// exportDatas("湖州")
// matchDev("浙江省","湖州")
matchCommunity("湖州")
def prov = "浙江省"
def city = "嘉兴"
// exportDatas(city)
// matchDev(prov, city)
matchCommunity(city)
// tempMatchDev(prov)
// saveMljData()
}
def exportDatas(city){
def datas = findMinglujiDatas(city)
@@ -35,71 +47,176 @@ class MinglujiData {
}
def matchDev(prov, city){
def devs = loadAllDevs(prov)
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d->
if(d['E'] && !d['N'] ){
def obj = findDevByName(d['E'])
if(!obj){
obj = findDevByAlias(d['E'])
if(!obj){
obj = findDebBySimlar(devs, d['E'], d['B'])
if(!obj){
obj = findDevByPark(d['A'], d['B'], d['E'])
if (!obj) {
}else{
d['N'] = "园区"
def i = 1;
while(i<20) {
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
}
File file=new File(fileName);
if(file.exists()){
ExcelUtils.multiThreadingModify(fileName, { d ->
if (d['D']) {
def street = d['D']
def splitStreet = ""
if (street) {
if (street.contains(d['C'])) {
def start = street.toString().lastIndexOf(d['C']) + d['C'].size()
def end = street.toString().size()
splitStreet = street.toString().substring(start, end)
} else {
splitStreet = street
}
}
d['K'] = splitStreet
}
if (d['I']) {
def splitCommunity = ""
if (d['C'] && d['I'].toString().contains(d['C'])) {
splitCommunity = d['I'].toString().substring(d['I'].toString().lastIndexOf(d['C']) + d['C'].toString().size(), d['I'].toString().size())
} else {
splitCommunity = d['I']
}
d['L'] = splitCommunity
}
if (d['E'] && !d['N']) {
def obj = findDevByName(d['E'])
if (!obj) {
obj = findDevByAlias(d['E'])
if (!obj) {
obj = findDebBySimlar(devs, d['E'], d['B'])
if (!obj) {
obj = findDevByPark(d['A'], d['B'], d['E'])
if (!obj) {
} else {
d['N'] = "园区"
}
} else {
d['N'] = "相似"
}
} else {
d['N'] = "别名"
}
} else {
d['N'] = "相同"
}
if (obj) {
d['M'] = obj.name
d['O'] = obj.city
d['P'] = obj.district
}
}
})
}
i++
println fileName +" 完成"
}
}
def tempMatchDev(prov){
def devs = loadAllDevs(prov)
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州-开发区-等级.xlsx",{d->
if(d['U'] && !d['W'] ){
def obj = findDevByName(d['U'])
if(!obj){
obj = findDevByAlias(d['U'])
if(!obj){
obj = findDebBySimlar(devs, d['U'], d['B'])
if(!obj){
obj = findDevByPark(d['A'], d['B'], d['U'])
if (!obj) {
}else{
d['X'] = "园区"
}
}else{
d['N'] = "相似"
d['X'] = "相似"
}
}else{
d['N'] = "别名"
d['X'] = "别名"
}
}else{
d['N'] = "相同"
d['X'] = "相同"
}
if(obj){
d['M'] = obj.name
d['O'] = obj.city
d['P'] = obj.district
d['W'] = obj.name
d['Y'] = obj.city
d['Z'] = obj.district
}
}
})
}
def matchCommunity(city){
ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}-没配居委1.xlsx",{d->
if(d['A'] && d['B'] && d['L'] && !d['Q']){
def list = findCommunityByName(d['A'], d['B'], d['L'])
if(!list){
list = findCommunityBySimlar(d['A'], d['B'], d['L'])
if(!list){
def i = 1;
while(i<20){
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
}
}else{
d['T'] = "包含"
}
}else{
d['T'] = "相同"
}
if(list){
if(list.size()==1){
def obj = list[0]
d['Q'] = obj.name
d['R'] = obj.street_name
}else if(list.size()>=2){
for(def l : list){
if(d['C']==l.area_name){
d['Q'] = l.name
d['R'] = l.street_name
break;
}
}
d['S'] = list.size()
}
}
}
}, 0)
File file=new File(fileName);
if(file.exists()){
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['A'] && d['B'] && d['L'] && !d['Q']){
def list = findCommunityByName(d['A'], d['B'], d['L'])
if(!list){
list = findCommunityBySimlar(d['A'], d['B'], d['L'])
if(!list){
}else{
d['T'] = "包含"
}
}else{
d['T'] = "相同"
}
if(list){
if(list.size()==1){
def obj = list[0]
d['Q'] = obj.name
d['R'] = obj.street_name
d['U'] = obj.area_name
d['V'] = obj.city_name
}else if(list.size()>=2){
for(def l : list){
if(d['C']==l.area_name){
d['Q'] = l.name
d['R'] = l.street_name
d['U'] = l.area_name
d['V'] = l.city_name
break;
}
}
d['S'] = list.size()
}
}
}
}, 0)
println fileName +" 远行完成"
}
i++
}
}
def saveMljData(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库数据.xlsx",{d->
def obj = new BasicDBObject()
obj.put("province", d['A'])
obj.put("city", d['B'])
obj.put("district", d['C'])
obj.put("name", d['D'])
obj.put("address", d['E'])
obj.put("dev_name", d['F'])
obj.put("committee", d['G'])
obj.put("street", d['H'])
company_100_CONN.save(obj)
})
}
def loadAllDevs(prov){
def q = new BasicDBObject()
q.put("province", prov)
@@ -27,7 +27,7 @@ class QiliangTest {
// task.run()
def data = [
'address': '景县开发区东区',
'address': '湖州市南太湖高新技术产业园区大港村陈家斗80号',
'type': '1' //1:对公 0:对私
]
def s = task.getProxyResult(data);
@@ -126,8 +126,7 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def sysnchVillage(){
def buducitys = [ '七台河市', '上海市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市',
'东营市', '中卫市', '中山市', '临夏回族自治州', '临汾市', '临沂市', '临沧市', '临高县', '丹东市',
def buducitys = [ '临汾市', '临沂市', '临沧市', '临高县', '丹东市',
'丽水市', '丽江市', '乌兰察布市', '乌海市', '乌鲁木齐市', '乐东黎族自治县', '乐山市', '九江市',
'云浮市', '五家渠市', '五指山市', '亳州市', '仙桃市', '伊春市', '伊犁哈萨克自治州', '佛山市',
'佳木斯市', '保亭黎族苗族自治县', '保定市', '保山市', '信阳市', '儋州市', '克孜勒苏柯尔克孜自治州',
@@ -169,7 +168,8 @@ class GaoDePoiDataProcess extends BaseBatch{
'黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市']
//"丽水市","杭州市",
def runbuducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市"]
def runbuducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市",'七台河市', '上海市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市',
'东营市', '中卫市', '中山市', '临夏回族自治州']
for(def city : buducitys){
if(runbuducitys.contains(city)){
continue
@@ -71,6 +71,10 @@ class MongoUtils {
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name)
}
if ((domainUrl == "192.168.0.206") && (port == 27020)) {
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI, db, name)
}
def key = db + name + domainUrl + port
if (collectionMap.containsKey(key)) {
return collectionMap[key]