房价网内部均价系统新增最近2个月的房源数据
This commit is contained in:
+1
-1
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\20240619--深圳成交数据.xlsx", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年6月深圳二手房成交.xlsx", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
|
||||
+57
-14
@@ -199,6 +199,16 @@ def findAllData(sqlInstance236, citycode){
|
||||
}
|
||||
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\保定202406.xlsx", { d ->
|
||||
// if(d['G']){
|
||||
// def obj = CommonDatautils.findDistrictById(d['G'])
|
||||
// if(obj){
|
||||
// d['H'] = obj.name
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
|
||||
////def url = "https://sh.anjuke.com"
|
||||
////def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
|
||||
@@ -1289,7 +1299,7 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240626人工指导价.xlsx"
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['J']){
|
||||
@@ -1308,9 +1318,9 @@ ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
d['A'] = dist.city
|
||||
d['B'] = dist.region
|
||||
d['C'] = dist.name
|
||||
// d['K'] = dist.estate_type2
|
||||
// d['L'] = dist.status
|
||||
// d['M'] = dist.name_cat
|
||||
d['K'] = dist.estate_type2
|
||||
d['L'] = dist.status
|
||||
d['M'] = dist.name_cat
|
||||
|
||||
// d['H'] = dist.name
|
||||
}
|
||||
@@ -1352,24 +1362,57 @@ ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
//def filePath = "D:\\数据\\建行\\房产估值\\指导价\\大庆删除小区.xlsx"
|
||||
//
|
||||
//def filePath = "D:\\数据\\数据建设\\小区\\嘉峪关处理.xlsx"
|
||||
//ExcelUtils.modify(filePath, { d ->
|
||||
// if(d['A']){
|
||||
//
|
||||
// def obj = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(obj){
|
||||
//
|
||||
// def data = new BasicDBObject()
|
||||
// data.put("status", 1)
|
||||
// data.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['A'])), new BasicDBObject('$set', data))
|
||||
//// def obj = CommonDatautils.findDistrictById(d['A'])
|
||||
// def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
// def list = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district")
|
||||
// if(list){
|
||||
// if(list.size()==1){
|
||||
// def dbo = list[0]
|
||||
// if(dbo){
|
||||
// def data = new BasicDBObject()
|
||||
// data.put("region", d['B'])
|
||||
// data.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', data))
|
||||
// }
|
||||
// }else{
|
||||
// println d['C'] +" 有多个"
|
||||
// }
|
||||
// }
|
||||
//// if(obj){
|
||||
//// def lng = d['G']
|
||||
//// def lat =d['H']
|
||||
////
|
||||
//// // 高德转百度
|
||||
//// def baidu = BaiDuApiUtils.geoconv(d['G'].toString().toDouble(), d['H'].toString().toDouble(), 3)
|
||||
//// def lng2 = baidu.x
|
||||
//// def lat2 = baidu.y
|
||||
////
|
||||
////
|
||||
//// def data = new BasicDBObject()
|
||||
//// Long lng_new = d['G'].toString().toDouble() *10000000000L
|
||||
//// Long lat_new = d['H'].toString().toDouble() *10000000000L
|
||||
//// data.put("lng", lng_new)
|
||||
//// data.put("lat", lat_new)
|
||||
////
|
||||
//// Long dLat2 = lat2 * 10000000000L
|
||||
//// Long dLng2 = lng2 * 10000000000L
|
||||
//// data.put("lat2", dLat2)
|
||||
//// data.put("lng2", dLng2)
|
||||
////
|
||||
////// data.put("status", 1)
|
||||
//// data.put("m_date", new Date())
|
||||
//// DbUtils.seaweed.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', data))
|
||||
//// }
|
||||
// }
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// if(d['B'] && !d['D']){
|
||||
|
||||
+67
-3
@@ -32,6 +32,10 @@ class TradeStatLog {
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
|
||||
def pssqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
@@ -41,7 +45,9 @@ class TradeStatLog {
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
task.exportTrade()//导出未入库成交
|
||||
// task.exportTrade()//导出未入库成交
|
||||
|
||||
task.exportTrade2()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
@@ -584,6 +590,36 @@ class TradeStatLog {
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据")
|
||||
}
|
||||
|
||||
|
||||
def exportTrade2(){
|
||||
|
||||
// def startDate = "2024-06-01"
|
||||
// def endDate = "2024-07-01"
|
||||
def ids = findAllDealIds();
|
||||
println ids.size()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d->
|
||||
if(ids.contains(d['A'])){
|
||||
d['V'] = "供"
|
||||
}
|
||||
if(d['S'] && !d['W']){
|
||||
def dist = CommonDatautils.findDistrictById(d['S'])
|
||||
if(!dist.name_cat && dist.name_cat!=0){
|
||||
d['W'] = "name_cat过滤"
|
||||
}
|
||||
if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){
|
||||
d['W'] = "小区过滤"
|
||||
}
|
||||
def key = d['B']+"##"+d['Q']+"##"+d['R']+"##"+d['D']+"##"+d['E'].toString().toDouble().toInteger()+"##"+d['J'].toString().substring(0,10)
|
||||
def map = findDealByCommId(d['B'], d['S'])
|
||||
def count = map.get(key)
|
||||
if(count>=2){
|
||||
d['W'] = "重复"
|
||||
}
|
||||
}
|
||||
println d['A']
|
||||
})
|
||||
}
|
||||
|
||||
def deleteDistrict(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
|
||||
if(d['G']){
|
||||
@@ -637,11 +673,39 @@ class TradeStatLog {
|
||||
return trades
|
||||
}
|
||||
|
||||
def findAllDealIds(){
|
||||
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'"
|
||||
def ids = []
|
||||
pssqlInstance.eachRow(sql,{r->
|
||||
if(!ids.contains(r.gujia_bds_mdl_id)){
|
||||
ids << r.gujia_bds_mdl_id
|
||||
}
|
||||
})
|
||||
return ids
|
||||
}
|
||||
|
||||
def findDealByCommId(city, commid){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_id", commid)
|
||||
q.put("city", city)
|
||||
q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2024-06-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2024-07-01")));
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
def count = map.get(key)
|
||||
if(!count){
|
||||
count = 0
|
||||
}
|
||||
map.put(key, count+1)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def findDealIds(city, startDate, endDate){
|
||||
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
|
||||
def ids = []
|
||||
def detailInfos = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
pssqlInstance.eachRow(sql,{r->
|
||||
if(!ids.contains(r.dealid)){
|
||||
ids << r.dealid
|
||||
|
||||
|
||||
@@ -12,8 +12,8 @@ class OpenUserBatch {
|
||||
def task = new OpenUserBatch();
|
||||
// task.run();
|
||||
// task.test();
|
||||
// task.addShenZhenFjtUser();
|
||||
task.addFujianFjtUser()
|
||||
task.addShenZhenFjtUser();
|
||||
// task.addFujianFjtUser()
|
||||
}
|
||||
|
||||
def test(){
|
||||
@@ -46,20 +46,20 @@ class OpenUserBatch {
|
||||
}
|
||||
|
||||
def addShenZhenFjtUser(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\渤海\\开通线上评估客户经理名单.xlsx",0,1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\黑龙江银行\\大正银行端新增账户模板中国银行.xls",0,1)
|
||||
def i = 0;
|
||||
for(def data : datas){
|
||||
i++;
|
||||
def id = 252608 + i;
|
||||
def id = 252893 + i;
|
||||
BigDecimal bd = new BigDecimal(data[3].toString());
|
||||
String mobile = bd.toPlainString();
|
||||
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)";
|
||||
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '18', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2025-12-31 00:00:00', '0', '', NULL)";
|
||||
onLineSqlInstance.execute(sql);
|
||||
|
||||
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '姚静梦')";
|
||||
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '王蕾')";
|
||||
onLineSqlInstance.execute(sql2)
|
||||
|
||||
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'上海','上海:')"
|
||||
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'哈尔滨,齐齐哈尔,大庆,牡丹江,佳木斯,绥化,鸡西,七台河,双鸭山,鹤岗,黑河,伊春,大兴安岭','哈尔滨:,齐齐哈尔:,大庆:,牡丹江:,佳木斯:,绥化:,鸡西:,七台河:,双鸭山:,鹤岗:,黑河:,伊春:,大兴安岭:')"
|
||||
onLineSqlInstance.execute(sql3)
|
||||
}
|
||||
}
|
||||
|
||||
+247
File diff suppressed because one or more lines are too long
+185
-7
@@ -7,6 +7,7 @@ import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
@@ -17,6 +18,8 @@ import net.sf.json.JSONObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DevProcess {
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
@@ -28,11 +31,13 @@ class DevProcess {
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020)
|
||||
|
||||
static main(args) {
|
||||
def task = new DevProcess();
|
||||
// task.run()
|
||||
// task.parseCityAndDist()
|
||||
task.devSplitRoad()
|
||||
// task.devSplitRoad()
|
||||
// task.devNameSplitRoad()
|
||||
// task.addFlag()
|
||||
// task.matchDev()
|
||||
@@ -41,6 +46,7 @@ class DevProcess {
|
||||
// task.updateCityAndDist()
|
||||
// task.addDevAddres1() //按添加地址中包含开发区名称的地址
|
||||
// task.addDevAddres2()//添加相差2公里范围内的地址
|
||||
// task.addDevAddres3()
|
||||
// task.matchDevAndAddAddress()
|
||||
|
||||
// task.stat()
|
||||
@@ -56,6 +62,15 @@ class DevProcess {
|
||||
// task.matchDevTownship()//添加乡镇街道
|
||||
|
||||
// task.addDevStreet()//添加区域为等级开发区下的街道到开发区对应表中
|
||||
|
||||
// task.matchNewCrawlDev()
|
||||
|
||||
// println DateUtils.add("2024-7-24", 7);
|
||||
|
||||
// task.addKfc()
|
||||
// task.devToPark() //开发区下辖园区
|
||||
task.checkTown()
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -334,15 +349,22 @@ class DevProcess {
|
||||
}
|
||||
|
||||
def matchDev(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\福建省.xlsx",{d->
|
||||
if(!d['Q'] && d['L']){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\省级开发区名单\\新疆开发区名单.xlsx",{d->
|
||||
if(!d['Q'] && d['A']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['L'])
|
||||
q.put("city",d['G'])
|
||||
// q.put("name", d['A'])
|
||||
// q.put("city",d['G'])
|
||||
// q.put('alias', new BasicDBObject('$in',[d['A']]))
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+d['A']+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['Q'] = obj._id.toString()
|
||||
d['R'] = obj.name
|
||||
d['C'] = obj._id.toString()
|
||||
d['D'] = obj.name
|
||||
if(obj.name == d['A']){
|
||||
d['E'] = "相同"
|
||||
}else{
|
||||
d['E'] = "包含"
|
||||
}
|
||||
}
|
||||
}
|
||||
},0);
|
||||
@@ -509,6 +531,38 @@ class DevProcess {
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
|
||||
}
|
||||
|
||||
|
||||
def addDevAddres3(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\新疆省入库.xlsx",{d->
|
||||
if(d['A']){
|
||||
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||
if(obj){
|
||||
d['C'] = obj._id.toString()
|
||||
|
||||
def q2 = new BasicDBObject()
|
||||
q2.put("name", d['A'])
|
||||
q2.put("address", d['B'])
|
||||
q2.put("type","地址")
|
||||
def sameObj = DEV_MAPPING_CONN.findOne(q2)
|
||||
if(!sameObj){
|
||||
index++
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province", obj.province)
|
||||
udata.put("city", obj.city)
|
||||
udata.put("district", obj.district)
|
||||
udata.put("name",obj.name)
|
||||
udata.put("type", "地址")
|
||||
udata.put("address", d['B'])
|
||||
DEV_MAPPING_CONN.save(udata)
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
println index
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def dbcursor = development_zone_CONN.find(new BasicDBObject())
|
||||
def total = 0
|
||||
@@ -777,4 +831,128 @@ class DevProcess {
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def matchNewCrawlDev(){
|
||||
def query = new BasicDBObject()
|
||||
query.put("match_id", new BasicDBObject('$exists', false))
|
||||
def list = kfq_COLL.find(query).toList()
|
||||
for(def d : list){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("name", d.name)
|
||||
q.put('alias', new BasicDBObject('$in',[d.name]))
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("match_id", obj._id.toString())
|
||||
udata.put("match_name", obj.name)
|
||||
kfq_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addKfc(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\投促中国开发区(1).xlsx",{d->
|
||||
if(d['I']){
|
||||
if(d['I'] == "添加"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("create_time" , new Date())
|
||||
udata.put("industry",d['D'])
|
||||
if(d['E'].contains("国家级")){
|
||||
udata.put("level" ,"国家级")
|
||||
}else if(d['E'].contains("省级")){
|
||||
udata.put("level" ,"省级")
|
||||
}
|
||||
udata.put("name" ,d['F'])
|
||||
udata.put("province" ,d['G'])
|
||||
udata.put("type" ,d['H'])
|
||||
udata.put("city" ,d['B'])
|
||||
udata.put("district" ,d['J'])
|
||||
development_zone_CONN.save(udata)
|
||||
}else{
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['I'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
def alias = []
|
||||
if(obj.alias){
|
||||
alias = obj.alias
|
||||
}
|
||||
if(!alias.contains(d['F'])){
|
||||
alias << d['F']
|
||||
}
|
||||
if(alias && alias.size()!=obj.alias.size()){
|
||||
def updateData = new BasicDBObject()
|
||||
updateData.put("alias", alias)
|
||||
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',updateData))
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def devToPark(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\开发区下辖街道-处理后表格.xlsx",0, 1)
|
||||
def rets = []
|
||||
rets << ["开发区名称", "下辖街道", "城市", "区域"]
|
||||
for(def d : datas){
|
||||
if(d['E']){
|
||||
def arrs = d['E'].split(",")
|
||||
def q = new BasicDBObject().append("name", d['B'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
for(def park : arrs){
|
||||
rets << [d['B'], park, obj.city, obj.district]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx", rets, "下辖街道")
|
||||
}
|
||||
|
||||
|
||||
def checkTown(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx",{d->
|
||||
|
||||
if(!d['E']){
|
||||
if(d['C'] && d['D']){
|
||||
// if(d['A']=="长春高新技术产业开发区"){
|
||||
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B'])
|
||||
if(jsonObject1) {
|
||||
if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
|
||||
def township = jsonObject1.get("township")
|
||||
def district = jsonObject1.get("district")
|
||||
d['E'] = "接口"
|
||||
if(district && district!='null'){
|
||||
d['F'] = district
|
||||
}
|
||||
if(township && township!='null'){
|
||||
d['G'] = township
|
||||
}
|
||||
|
||||
d['H'] = jsonObject1.get("townCode")
|
||||
if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){
|
||||
d['I'] = jsonObject1.get("provinceCode")
|
||||
}
|
||||
}
|
||||
// }
|
||||
}
|
||||
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city", d['C'])
|
||||
//// q.put("district", d['D'])
|
||||
// q.put("mzb_district_name", d['D'])
|
||||
// q.put("type", "乡镇街道")
|
||||
// q.put("street", d['B'])
|
||||
// def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
// if(obj){
|
||||
// d['E'] = "可入库2"
|
||||
// d['F'] = obj.district
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
},1)
|
||||
}
|
||||
}
|
||||
|
||||
+277
@@ -0,0 +1,277 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
import net.sf.json.JSONObject
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
class DevUpdateTestBatch {
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new DevUpdateTestBatch()
|
||||
def path = "D:\\数据\\数据建设\\开发区\\文心一言获取开发区下辖园区及街道.xlsx"//开发区-园区.xlsx
|
||||
// task.addDev(path)
|
||||
task.addDevAddress(path)
|
||||
|
||||
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
|
||||
}
|
||||
|
||||
def addDev(path){
|
||||
ExcelUtils.modify(path,{ d->
|
||||
if(!d['I'] && d['A']){
|
||||
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||
if(!obj){
|
||||
def o = new BasicDBObject()
|
||||
o.put("province", d['B'])
|
||||
o.put("city", d['C'])
|
||||
o.put("district", d['D'])
|
||||
o.put("name", d['A'])
|
||||
o.put("level", d['E'])
|
||||
o.put("type", d['F'])
|
||||
|
||||
if(d['G']){
|
||||
o.put("township", d['G'])
|
||||
}
|
||||
if(d['H']){
|
||||
def arr = d['H'].split(",")
|
||||
if(arr){
|
||||
o.put("alias", arr)
|
||||
}
|
||||
}
|
||||
def id = addMongoDB(o)
|
||||
o.remove("_id")
|
||||
o.id = id.toString()
|
||||
o.remove("create_time")
|
||||
addEs(o, "192.168.0.89", "development_zone")
|
||||
addEs(o, "192.168.0.52", "development_zone")
|
||||
d['I'] = "已完成"
|
||||
}else{
|
||||
println d['A'] +" 已存在"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addMongoDB(o){
|
||||
o.put("create_time", new Date())
|
||||
development_zone_100_CONN.save(o)
|
||||
DEVELOPMENT_ZONE_43_COLL.save(o)
|
||||
return o._id
|
||||
}
|
||||
|
||||
def addEs(map, host, indexName){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int processedCount = 0;
|
||||
def client = null
|
||||
if(host && host == "192.168.0.89"){
|
||||
client = ElasticUtils89.getClient(host)
|
||||
}else if(host && host == "192.168.0.52"){
|
||||
client = ElasticUtils.getClient(host)
|
||||
}
|
||||
|
||||
request.add(new IndexRequest(indexName, "_doc", map.id).source(map, XContentType.JSON));
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addDevAddress(path){
|
||||
ExcelUtils.modify(path,{ d->
|
||||
if(!d['D'] && d['A']){
|
||||
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||
if(obj){
|
||||
def q = new BasicDBObject("name", d['A']).append("type",d['B'])
|
||||
if(d['B'] == "乡镇街道"){
|
||||
q.put("township", d['C'])
|
||||
}else if(d['B'] == "地址"){
|
||||
q.put("address", d['C'])
|
||||
}else if(d['B'] == "园区"){
|
||||
q.put("park", d['C'])
|
||||
}
|
||||
def mappingObj = DEV_MAPPING_100_CONN.findOne(q)
|
||||
if(!mappingObj){
|
||||
def o = new BasicDBObject()
|
||||
o.put("province", obj.province)
|
||||
o.put("city", obj.city)
|
||||
o.put("district", obj.district)
|
||||
o.put("name", obj.name)
|
||||
o.put("type", d['B'])
|
||||
if(d['B'] == "乡镇街道"){
|
||||
o.put("township", d['C'])
|
||||
}else if(d['B'] == "地址"){
|
||||
o.put("address", d['C'])
|
||||
}else if(d['B'] == "园区"){
|
||||
o.put("park", d['C'])
|
||||
}
|
||||
def date = new Date()
|
||||
o.put("c_date", date)
|
||||
o.put("version", date.getTime())
|
||||
DEV_MAPPING_100_CONN.save(o)
|
||||
|
||||
|
||||
o.id = o._id.toString()
|
||||
o.remove("_id")
|
||||
o.remove("c_date")
|
||||
addEs(o, "192.168.0.89", "dev_mapping")
|
||||
try{
|
||||
addEs(o, "192.168.0.52", "dev_mapping")
|
||||
}catch(Exception e){
|
||||
println "52 es error: "+e.printStackTrace()
|
||||
}
|
||||
|
||||
d['D'] = "已完成"
|
||||
}else{
|
||||
d['D'] = "存在"
|
||||
println d['A'] +" "+ d['C']+" 重复"
|
||||
}
|
||||
}else{
|
||||
println d['A'] +" 不存在"
|
||||
}
|
||||
}
|
||||
},2)
|
||||
}
|
||||
|
||||
|
||||
def updateHasGov(province, city, district, street, hasGov){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("province", province)
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("street", street)
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(obj){
|
||||
//修改43
|
||||
// ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',new BasicDBObject('has_gov',hasGov)))
|
||||
|
||||
//修改89 administrative_mapping
|
||||
update89EsAdministrativeMapping(province, city, district, street, hasGov)
|
||||
//修改89 administrative_mapping
|
||||
update52EsAdministrativeMapping(province, city, district, street, hasGov)
|
||||
}else{
|
||||
println province+" "+city+" "+district+" "+street+" 没找到"
|
||||
}
|
||||
//修改100 community
|
||||
//修改89 community
|
||||
//修改52 community
|
||||
|
||||
}
|
||||
|
||||
|
||||
def update89EsAdministrativeMapping(province, city, district, street, hasGov){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.has_gov='${hasGov}'",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"match": {
|
||||
"province_name": "${province}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"city_name":"${city}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"area_name":"${district}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"street_name":"${street}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
// def response = JSONObject.fromObject(res)
|
||||
println(res)
|
||||
}
|
||||
|
||||
|
||||
def update52EsAdministrativeMapping(province, city, district, street, hasGov){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.52:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.has_gov='${hasGov}'",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"match": {
|
||||
"province_name": "${province}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"city_name":"${city}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"area_name":"${district}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"street_name":"${street}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
// def response = JSONObject.fromObject(res)
|
||||
println(res)
|
||||
}
|
||||
}
|
||||
+34
-1
@@ -10,7 +10,8 @@ class DistrictAliasProcess {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictAliasProcess();
|
||||
task.run()
|
||||
// task.run()
|
||||
task.fenQiProcess()
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -42,4 +43,36 @@ class DistrictAliasProcess {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def fenQiProcess(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("cat","district")
|
||||
q.put("status", 0)
|
||||
q.put("master_district_name", 'master')
|
||||
def dbcursor = DbUtils.seaweed.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << ["_id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name]
|
||||
}
|
||||
|
||||
for(def r : rets){
|
||||
def mainName1 = r.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
|
||||
def mainName2 = r.name.toString().replaceAll("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$", "")
|
||||
def lastObj = null
|
||||
for(def j : rets){
|
||||
if((j.name==mainName1 || mainName2==j.name) && r._id!=j._id && r.region==j.region && r.city==j.city){
|
||||
lastObj = j;
|
||||
break
|
||||
}
|
||||
}
|
||||
if(lastObj){
|
||||
println r
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("master_district_name","")
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(r._id)), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+5
-5
@@ -28,23 +28,23 @@ class GXDdistrictMatch {
|
||||
static main(args) {
|
||||
def task = new GXDdistrictMatch();
|
||||
// task.run()
|
||||
// task.addOldMatch()
|
||||
task.addOldMatch()
|
||||
// task.runDb()
|
||||
|
||||
// task.updateDistStatus()//恢复name_cat不正常数据
|
||||
// task.saveToDB() //保存新增小区
|
||||
// task.restoreDist()//恢复小区status=0
|
||||
|
||||
task.runMatch()
|
||||
// task.runMatch()
|
||||
}
|
||||
|
||||
def addOldMatch(){
|
||||
//老匹配
|
||||
def fileName = "D:\\数据\\国信达\\非正常数据.xlsx"//11城重新匹配.xlsx
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"//11城重新匹配.xlsx
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
println d['C']
|
||||
if(!d['AJ']){
|
||||
def jsonObject = OpenApiClient.addressMatch(d['AD'], d['H'], null, false, d['AE'], d['C'])
|
||||
if(!d['AJ'] && d['I']){
|
||||
def jsonObject = OpenApiClient.addressMatch(d['C'], d['I'], "district", false,null, d['D'])
|
||||
if (jsonObject != null && !jsonObject.containsKey("result")) {
|
||||
d['AJ'] = jsonObject.get("id")
|
||||
d['AK'] = jsonObject.get("city")
|
||||
|
||||
+88
-4
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
class PoiLocation {
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "didata_map_poi_single", MAP_POI_URL)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
|
||||
@@ -35,21 +35,52 @@ class PoiLocation {
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiLocation();
|
||||
task.run();
|
||||
// task.run();
|
||||
task.regionAndStreetProcess()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
def citys = ["芜湖市","重庆市","杭州市","泉州市","东莞市","福州市","厦门市","佛山市","天津市","武汉市","宁波市","郑州市","长沙市","青岛市","上海市","西安市","温州市","无锡市","南京市","湖州市","合肥市","临沂市","石家庄市","昆明市","金华市","保定市","成都市","济南市","北京市","沈阳市","潍坊市","广州市","惠州市","嘉兴市","赣州市","苏州市","南通市","南宁市","台州市","南阳市","徐州市","长春市","深圳市","哈尔滨市","漳州市","中山市","常州市","烟台市","大连市","邯郸市","济宁市","盐城市","绍兴市","唐山市","洛阳市","阜阳市","菏泽市","贵阳市","沧州市","遵义市","莆田市","南昌市","商丘市","周口市","邢台市","江门市","衡阳市","邵阳市","信阳市","廊坊市","扬州市","泰州市","新乡市","太原市","德州市","淄博市","宁德市","绵阳市","汕头市","聊城市","淮安市","驻马店市","黄冈市","宿迁市","三明市","六安市","龙岩市","桂林市","岳阳市","南平市","揭阳市","上饶市","九江市","南充市","连云港市","襄阳市","泰安市","常德市","郴州市","安阳市","安庆市","湛江市","永州市","茂名市","曲靖市","宜宾市","肇庆市","宿州市","兰州市","运城市","镇江市","滁州市","威海市","梅州市","宜春市","清远市","亳州市","怀化市","荆州市","平顶山市","开封市","玉林市","咸阳市","宜昌市","株洲市","乌鲁木齐市","滨州市","渭南市","临汾市","榆林市","银川市","衡水市","毕节市","枣庄市","吉安市","张家口市","柳州市","许昌市","泸州市","呼和浩特市","达州市","德阳市","赤峰市","珠海市","孝感市","益阳市","娄底市","乐山市","恩施土家族苗族自治州","焦作市","湘潭市","海口市","红河哈尼族彝族自治州","黔南布依族苗族自治州","河源市","汉中市","日照市","大理白族自治州","濮阳市","黔东南苗族侗族自治州","秦皇岛市","晋中市","宝鸡市","吉林市","包头市","蚌埠市","东营市","宣城市","韶关市","衢州市","十堰市","承德市","丽水市","眉山市","鞍山市","昭通市","凉山彝族自治州","长治市","抚州市","鄂尔多斯市","吕梁市","百色市","齐齐哈尔市","贵港市","潮州市","黔西南布依族苗族自治州","广安市","淮南市","河池市","大同市","铜仁市","阳江市","咸宁市","营口市","绥化市","马鞍山市","安康市","荆门市","忻州市","广元市","内江市","遂宁市","西宁市","大庆市","汕尾市","文山壮族苗族自治州","锦州市","延安市","巴中市","呼伦贝尔市","楚雄彝族自治州","黄石市","玉溪市","六盘水市","通辽市","钦州市","云浮市","梧州市","漯河市","晋城市","庆阳市","安顺市","丹东市","自贡市","朝阳市","葫芦岛市","天水市","三门峡市","湘西土家族苗族自治州","北海市","陇南市","普洱市","延边朝鲜族自治州","保山市","定西市","随州市","喀什地区","黄山市","佳木斯市","巴彦淖尔市","牡丹江市","舟山市","资阳市","商洛市","松原市","锡林郭勒盟","伊犁哈萨克自治州","萍乡市","铁岭市","乌兰察布市","丽江市","景德镇市","张家界市","抚顺市","通化市","淮北市","来宾市","临沧市","阿克苏地区","贺州市","盘锦市","昌吉回族自治州","池州市","雅安市","三亚市","平凉市","四平市","酒泉市","西双版纳傣族自治州","鹤壁市","辽阳市","白城市","吴忠市","崇左市","阜新市","朔州市","白银市","拉萨市","巴音郭楞蒙古自治州","张掖市","铜陵市","武威市","临夏回族自治州","德宏傣族景颇族自治州","新余市","甘孜藏族自治州","兴安盟","本溪市","鄂州市","黑河市","阿坝藏族羌族自治州","阳泉市","防城港市","鸡西市","仙桃市","鹰潭市","固原市","白山市","中卫市","双鸭山市","攀枝花市","石嘴山市","和田地区","海东市","塔城地区","日喀则市","济源市","伊春市","天门市","辽源市","潜江市","那曲市","铜川市","海西蒙古族藏族自治州","鹤岗市","昌都市","阿拉善盟","乌海市","儋州市","哈密市","克拉玛依市","七台河市","金昌市","甘南藏族自治州","琼海市","迪庆藏族自治州","石河子市","阿勒泰地区","文昌市","玉树藏族自治州","嘉峪关市","万宁市","怒江傈僳族自治州","林芝市","海南藏族自治州","吐鲁番市","澄迈县","陵水黎族自治县","博尔塔拉蒙古自治州","山南市","阿里地区","大兴安岭地区","海北藏族自治州","东方市","乐东黎族自治县","克孜勒苏柯尔克孜自治州","果洛藏族自治州","临高县","阿拉尔市","黄南藏族自治州","神农架林区","屯昌县","定安县","昌江黎族自治县","五家渠市","保亭黎族苗族自治县","琼中黎族苗族自治县","五指山市","白沙黎族自治县","北屯市","图木舒克市","可克达拉市","双河市","铁门关市","毕节地区","昆玉市","铜仁地区","海东地区","三沙市","吐鲁番地区","哈密地区","日喀则地区","林芝地区","昌都地区","山南地区","那曲地区"]
|
||||
def citys = [""]
|
||||
citys = citys.reverse()
|
||||
for(def city : citys){
|
||||
findPois(city)
|
||||
}
|
||||
}
|
||||
|
||||
def regionAndStreetProcess(){
|
||||
def citys = ['南平市','泉州市', '厦门市', '福州市', '龙岩市', '漳州市', '宁德市', '三明市', '莆田市']
|
||||
for(def city : citys){
|
||||
def datas = findAllPoisByCity(city)
|
||||
process(datas)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def findAllPoisByCity(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
keys.put("province", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("township", 1)
|
||||
keys.put("location", 1)
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findPois(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("run_status", new BasicDBObject('$ne', 1))
|
||||
// q.put("run_status", new BasicDBObject('$ne', 1))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
@@ -87,6 +118,32 @@ class PoiLocation {
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def process(datas){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
dataRun(d)
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def checkData(def index, def datas){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
@@ -190,6 +247,33 @@ class PoiLocation {
|
||||
return map;
|
||||
}
|
||||
|
||||
def dataRun(d){
|
||||
def regions = getFullRegion(d)
|
||||
if(d.location) {
|
||||
//判断区域否在围栏里
|
||||
def regionFlag = null
|
||||
if (d.district) {
|
||||
for (def rs : regions) {
|
||||
if (rs.shape && rs.shape.coordinates) {
|
||||
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (regionFlag) {//在围栏里
|
||||
//判断township是否在统计局中
|
||||
// if(d.township){
|
||||
// town = getSimilarTown(regions, d.city, d.township)
|
||||
// }
|
||||
|
||||
|
||||
}else{//不在转栏里,重新搜索
|
||||
|
||||
println(d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getFullRegion(d){
|
||||
def regions = regionMap.get(d.city)
|
||||
def names = []
|
||||
|
||||
+237
-2
@@ -2,8 +2,10 @@ package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
@@ -12,6 +14,8 @@ import com.mongodb.client.AggregateIterable
|
||||
import org.bson.Document
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 乡镇街道中间表
|
||||
* 包含市,区,乡镇街道别名,来源 统计局community表
|
||||
@@ -26,6 +30,9 @@ class TownMiddleProcess {
|
||||
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_43base_COLL = MongoUtils.getCollectionByName("fangjia_base", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
|
||||
DBCollection CITYPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
|
||||
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
@@ -67,7 +74,15 @@ class TownMiddleProcess {
|
||||
// checkTown()
|
||||
//区域民政部名称更新到乡镇街道中
|
||||
// updateTownshipByRegion()
|
||||
// addAliasFromExcel()
|
||||
|
||||
//增加简称及简拼
|
||||
// addSimpleName()
|
||||
|
||||
//根据民政部数据修改行政区划中间表
|
||||
// mzbDistrictUpate()
|
||||
|
||||
// addCity()
|
||||
}
|
||||
|
||||
|
||||
@@ -422,14 +437,16 @@ class TownMiddleProcess {
|
||||
for(def d : dbcursor){
|
||||
datas << d
|
||||
if (datas.size() >= 2000) {
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||
println datas.size()
|
||||
datas.clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (datas.size() >= 0) {
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||
datas.clear();
|
||||
}
|
||||
}
|
||||
@@ -929,4 +946,222 @@ class TownMiddleProcess {
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true)
|
||||
}
|
||||
}
|
||||
|
||||
def addAliasFromExcel(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道合并或者变更20240627.xlsx",{ row->
|
||||
def q = new BasicDBObject("type", "乡镇街道")
|
||||
q.put("city", row['A'])
|
||||
q.put("street", row['D'])
|
||||
if(row['B']){
|
||||
q.put("district", row['B'])
|
||||
}
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(obj){
|
||||
row['F'] = '是'
|
||||
def alias = obj.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(row['C'])){
|
||||
alias << row['C']
|
||||
}
|
||||
|
||||
if(alias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
|
||||
def updateQuery = new BasicDBObject("_id", obj._id)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addSimpleName(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
|
||||
def prov = ProvinceData.getSimpleProvince(d.province)
|
||||
if(prov){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province_short", prov)
|
||||
|
||||
def updateQuery = new BasicDBObject("_id", d._id)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
// def city = d.city.replace("市","").replace("区","").replace("县","")
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("city_short", city)
|
||||
// if(d.district && d.district.size()>=3){
|
||||
// def region= d.district.replace("区","").replace("镇","").replace("乡","")
|
||||
// udata.put("district_short", region)
|
||||
// }
|
||||
// def updateQuery = new BasicDBObject("_id", d._id)
|
||||
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
// def city = RegionData.formatCity(d.city)
|
||||
// if(city && d.mzb_district_name){
|
||||
// def region = RegionData.formatRegion(city, d.mzb_district_name)
|
||||
// if(region){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("city_short", city)
|
||||
// udata.put("district_short", region)
|
||||
//
|
||||
// def updateQuery = new BasicDBObject("_id", d._id)
|
||||
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
// }
|
||||
// }
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def mzbDistrictUpate(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\民政部\\2014-2022年.xlsx",{ row->
|
||||
if(!row['I']){
|
||||
def updateType = row['E']
|
||||
def prov = row['B']
|
||||
def newStreet = row['G']
|
||||
|
||||
if(["新设街道","新设镇","新增街道","新建街道","新建乡镇","新设乡","新增乡","新增建制"].contains(updateType)){
|
||||
def dbo = findStreets(prov, newStreet, row['F'].toString().toDouble().toInteger().toString())
|
||||
if(dbo){
|
||||
row['I'] = dbo.city
|
||||
row['J'] = dbo.district
|
||||
row['K'] = "存在"
|
||||
row['L'] = dbo.tjj_street_code
|
||||
row['M'] = dbo.pid
|
||||
}
|
||||
}else if(["撤乡设镇","撤乡建镇","撤乡设乡","撤乡、镇设镇","撤销设街道","撤销建街道","撤销街道建镇","撤镇设镇、街道","撤销镇新设镇","乡镇合并","整建制划归","撤镇建新镇","区划变更","撤镇、街道设镇","撤街道建镇","撤销乡、镇新设镇","撤镇、街道设街道","撤乡、街道设街道","撤乡设街道","撤乡建街道","撤镇设街道","撤镇建街道","名称变更","撤街道设镇","撤乡、镇设街道","撤街道、镇、乡","并入其他建制",'撤镇设镇',"合并","合并建镇","更名","撤销合并","合并建制","合并镇","合并街道","名称变更","合并调整","合并建街道","撤镇设街道、镇","撤乡、镇设乡","撤镇乡设街道","区划调整","合并乡","撤镇设市","区划代码调整","区划代码变更","撤销乡、镇设街道"].contains(updateType)){
|
||||
if(row['F']){
|
||||
def dbo = null
|
||||
def code = row['F'].toString()
|
||||
if(!code.contains(" ")){
|
||||
dbo = findStreets(prov, newStreet, code.toDouble().toInteger().toString())
|
||||
}
|
||||
if(dbo){
|
||||
row['I'] = dbo.city
|
||||
row['J'] = dbo.district
|
||||
row['L'] = dbo.tjj_street_code
|
||||
row['M'] = dbo.pid
|
||||
def alias = dbo.alias
|
||||
if(!dbo.alias){
|
||||
alias = []
|
||||
}
|
||||
if(row['D']){
|
||||
if(!alias.contains(row['D'])){
|
||||
row['K'] = "存在(加别名)"
|
||||
alias << row['D']
|
||||
addAlias(dbo._id, alias)
|
||||
}else{
|
||||
row['K'] = "存在"
|
||||
}
|
||||
}else{
|
||||
row['K'] = "存在"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}else if(updateType == "撤街道设街道"){
|
||||
row['K'] = "人工处理"
|
||||
}else if(updateType == "隶属关系变更"){
|
||||
row['K'] = "人工处理"
|
||||
}else if(["撤销镇","撤乡","撤销街道","撤销","撤街道设社区工作站","撤销建制"].contains(updateType)){
|
||||
row['K'] = "人工处理"
|
||||
}else if(updateType =='撤销乡'){
|
||||
if(!row['F']){
|
||||
row['K'] = "人工处理"
|
||||
}
|
||||
}
|
||||
|
||||
if(prov && newStreet && (!row['K'] || (row['K'] && !row['K'].contains("存在")) )){
|
||||
def dbo = findStreets(prov, newStreet, null)
|
||||
if(dbo){
|
||||
row['I'] = dbo.city
|
||||
row['J'] = dbo.district
|
||||
row['K'] = "存在-编码不同"
|
||||
row['L'] = dbo.tjj_street_code
|
||||
row['M'] = dbo.pid
|
||||
def alias = dbo.alias
|
||||
if(!dbo.alias){
|
||||
alias = []
|
||||
}
|
||||
if(row['D']){
|
||||
if(!alias.contains(row['D'])){
|
||||
row['K'] = "存在(加别名)"
|
||||
alias << row['D']
|
||||
// addAlias(dbo._id, alias)
|
||||
}else{
|
||||
row['K'] = "存在"
|
||||
}
|
||||
}else{
|
||||
row['K'] = "存在"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addCity(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\乡镇街道\\民政部\\2014-2022年.xlsx",{ row->
|
||||
def citycode = null
|
||||
if(row['F']){
|
||||
citycode = row['F'].toString().substring(0, 4)
|
||||
}else if(row['C']){
|
||||
citycode = row['C'].toString().substring(0, 4)
|
||||
}
|
||||
|
||||
if(citycode){
|
||||
def q = new BasicDBObject()
|
||||
q.put("mzb_city_code", citycode+"00")
|
||||
q.put("type", "城市")
|
||||
def dbo = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(dbo){
|
||||
row['O'] = dbo.city
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def findStreets(prov, newStreet, streetCode){
|
||||
def q = new BasicDBObject()
|
||||
Pattern provPattern = Pattern.compile(prov+'.*$', Pattern.CASE_INSENSITIVE);
|
||||
q.put("province", new BasicDBObject('$regex', provPattern))
|
||||
q.put("street", newStreet)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
def list = []
|
||||
def dbo = null
|
||||
for(def d : dbcursor){
|
||||
list << d
|
||||
}
|
||||
if(list && list.size()==1){
|
||||
dbo = list[0]
|
||||
}else if(list && list.size()>1){
|
||||
if(streetCode){
|
||||
for(def d : list){
|
||||
if(d.tjj_street_code && d.tjj_street_code.contains(streetCode)){
|
||||
dbo = d
|
||||
break;
|
||||
}else if(streetCode.contains(d.pid)){
|
||||
dbo = d
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return dbo
|
||||
}
|
||||
|
||||
def addAlias(id, alias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
udata.put("operator","高娟-批量")
|
||||
|
||||
def updateQuery = new BasicDBObject("_id", id)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
+243
-11
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
@@ -9,12 +10,14 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class TownshipStrandUpdate {
|
||||
|
||||
@@ -26,13 +29,26 @@ class TownshipStrandUpdate {
|
||||
def polylineMap = null
|
||||
def newAdvMappingMap = null
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection dev_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
static main(args) {
|
||||
def task = new TownshipStrandUpdate()
|
||||
// task.process()
|
||||
task.updateTown()
|
||||
// task.updateTown()
|
||||
// task.stat()
|
||||
|
||||
// task.updateDiffRegionTown()
|
||||
// task.cleanTown()
|
||||
// task.checkTownFromMongo()
|
||||
// task.updateTownMapping()
|
||||
// task.updateDevMapping()
|
||||
println new Date().getTime()
|
||||
// task.updateSpecCityTown()
|
||||
}
|
||||
|
||||
|
||||
@@ -43,21 +59,21 @@ class TownshipStrandUpdate {
|
||||
polylineMap = CommonDatautils.findAllTownRanges()
|
||||
newAdvMappingMap = CommonDatautils.queryNewTownByCity()
|
||||
// def citys = ["唐山","邯郸","石家庄","保定","秦皇岛","廊坊","沧州","张家口","衡水","邢台","承德"]//CommonDatautils.findAllCity()
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def citys = ["澄迈","临高"]//CommonDatautils.findAllCity(),"东方","澄迈","临高"
|
||||
citys = citys.reverse()
|
||||
for(def city : citys){
|
||||
//
|
||||
if(["中山","东莞"].contains(city)){
|
||||
continue
|
||||
}
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
|
||||
// if(["中山","东莞"].contains(city)){
|
||||
// continue
|
||||
// }
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\写字楼\\写字楼乡镇街道格式化-${city}.xlsx"
|
||||
File file=new File(fileName);
|
||||
if(!file.exists()){
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, "shop", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
|
||||
// if(!file.exists()){
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
|
||||
dists = checkData(dists)
|
||||
def datas = createDatas(dists)
|
||||
ExcelUtils.write(fileName, datas, "${city}")
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,6 +178,10 @@ class TownshipStrandUpdate {
|
||||
def running(d){
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
def regions = getFullRegion(fullCity, d)
|
||||
if(!regions && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(fullCity.city)){
|
||||
regions = []
|
||||
regions << fullCity.city
|
||||
}
|
||||
def gaodeMap = null
|
||||
def town = null
|
||||
def mapTown = null
|
||||
@@ -217,6 +237,10 @@ class TownshipStrandUpdate {
|
||||
}
|
||||
if( gaodeMap.district && gaodeMap.district!='[]'){
|
||||
mapRegion = gaodeMap.district
|
||||
}else{
|
||||
if(mapCity && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(mapCity)){
|
||||
mapRegion = mapCity
|
||||
}
|
||||
}
|
||||
if(gaodeMap.township && gaodeMap.township!='[]'){
|
||||
mapTownship = gaodeMap.township
|
||||
@@ -331,7 +355,7 @@ class TownshipStrandUpdate {
|
||||
def names = []
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
if(r.mzb_district_name.contains(d.region)){
|
||||
if(r.mzb_district_name.contains(d.region) || r.district_short.contains(d.region)){
|
||||
names << r.district
|
||||
}
|
||||
}
|
||||
@@ -381,7 +405,7 @@ class TownshipStrandUpdate {
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
if(township || district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
@@ -397,4 +421,212 @@ class TownshipStrandUpdate {
|
||||
map.format_addr = format_addr
|
||||
return map;
|
||||
}
|
||||
|
||||
def updateDiffRegionTown(){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop")
|
||||
d['AA'] = lists.size()
|
||||
//只处理库中只有一个的
|
||||
if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){
|
||||
def newRegion = RegionData.formatRegion(d['B'], d['M'])
|
||||
if(newRegion){
|
||||
if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
|
||||
def dbo = lists[0]
|
||||
def arrs = d['X'].split(" ")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", newRegion)
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", arrs[1])
|
||||
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
|
||||
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟")
|
||||
d['AB'] = "已处理"
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def cleanTown(){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
if(d['AC']){
|
||||
def dbo = CommonDatautils.findDistrict(d['B'], d['C'], d['D'], "district")
|
||||
if(dbo && dbo.township){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", "")
|
||||
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
|
||||
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "office", "update-town","高娟")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def checkTownFromMongo(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
fullCityMap = CommonDatautils.queryFullNameByCity()
|
||||
regionMap = CommonDatautils.queryDistNameByCity()
|
||||
commMap = CommonDatautils.queryCommunityByCity()
|
||||
citys = citys.reverse()
|
||||
def index = 0
|
||||
for(def city : citys) {
|
||||
index++
|
||||
if (["中山", "东莞"].contains(city)) {
|
||||
continue
|
||||
}
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city", "region", "name", "address", "cat", "estate_type2", "status", "township", "lng", "lat"])
|
||||
for (def d : dists) {
|
||||
if (d.township) {
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
def regions = getFullRegion(fullCity, d)
|
||||
def town = getSimilarTown(regions, fullCity, d.township)
|
||||
if (!town) {
|
||||
town = findStreet(d.city, d.city, d.township)
|
||||
if (!town) {
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", "")
|
||||
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',d._id), new BasicDBObject('$set', udata))
|
||||
CommonDatautils.saveOptLog(d.city, d.region, d.name, "office", "update-town","高娟")
|
||||
|
||||
println d.city + " " + d.region + " " + d.township + " " + d.name
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
println index + " " + city + " 完成"
|
||||
}
|
||||
}
|
||||
|
||||
def findStreet(city, district, street){
|
||||
def q = new BasicDBObject()
|
||||
Pattern provPattern = Pattern.compile(city+'.*$', Pattern.CASE_INSENSITIVE);
|
||||
q.put("city", new BasicDBObject('$regex', provPattern))
|
||||
q.put("district", new BasicDBObject('$regex', Pattern.compile(district+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
q.put("street", new BasicDBObject('$regex', Pattern.compile(street+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
def list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
|
||||
if(list){
|
||||
if(list.size()==1){
|
||||
return list[0]
|
||||
}else{
|
||||
println city+ " "+district+" "+street+" 有多个"
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def updateTownMapping(){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\双方区域对比.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
if(d['F']){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mzb_district_name", d['F'])
|
||||
//
|
||||
// def updateQuery = new BasicDBObject()
|
||||
// updateQuery.put("city", d['C'])
|
||||
// updateQuery.put("district", d['D'])
|
||||
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata), false, true)
|
||||
|
||||
def city = RegionData.formatCity(d['C'])
|
||||
if(city){
|
||||
def region = RegionData.formatRegion(city, d['D'])
|
||||
if(region){
|
||||
def updateQuery = new BasicDBObject()
|
||||
updateQuery.put("category", "region")
|
||||
updateQuery.put("city", city)
|
||||
updateQuery.put("name", region)
|
||||
|
||||
def list = REGION_BLOCK_COLL.find(updateQuery).toList()
|
||||
if(list && list.size()==1){
|
||||
def dbo = list[0]
|
||||
def query = new BasicDBObject()
|
||||
query.put("_id", dbo._id)
|
||||
|
||||
// def pattern = dbo.pattern
|
||||
// if(pattern.contains(d['F'])){
|
||||
// pattern.remove(d['F'])
|
||||
// }
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("org_name", d['F'])
|
||||
// udata.put("pattern",pattern)
|
||||
REGION_BLOCK_COLL.update(query, new BasicDBObject('$set',udata), false, true)
|
||||
}else{
|
||||
println city +" "+region+" "+list.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def updateDevMapping(){
|
||||
def fileName = "D:\\数据\\数据建设\\开发区\\开发区映射表重复数据.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("district", d['B'])
|
||||
q.put("name",d['C'])
|
||||
q.put("type", "乡镇街道")
|
||||
q.put("township", d['D'])
|
||||
def list = dev_MAPPING_COLL_100.find(q).toList()
|
||||
if(list && list.size()==2){
|
||||
def obj = list[0]
|
||||
def removeQuery = new BasicDBObject()
|
||||
removeQuery.put("_id", obj._id)
|
||||
dev_MAPPING_COLL_100.remove(removeQuery)
|
||||
}else{
|
||||
println q
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def updateSpecCityTown(){
|
||||
def citys = ["中山","东莞","潜江","天门","神农架","五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","阿拉尔","图木舒克","五家渠","北屯","铁门关","双河","可克达拉"]
|
||||
|
||||
// def citys = ["中山"]
|
||||
for(def city : citys){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
def list = CommonDatautils.findDistrictList(d['A'], null , d['C'], "office")
|
||||
if(list.size()>1){
|
||||
println d['A'] +" "+d['C'] + " 有多个"
|
||||
}else if(list.size()==1){
|
||||
def dbo = list[0]
|
||||
def town = null
|
||||
// if(d['X']){
|
||||
// town = d['X']
|
||||
// }else
|
||||
if(d['R']){
|
||||
town = d['R']
|
||||
}
|
||||
if(!town && d['M']){
|
||||
town = d['M']
|
||||
}
|
||||
def region = RegionData.formatRegion(d['A'], town)
|
||||
if(region){
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", region)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", town)
|
||||
udata.put("region", region)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d['A']+" "+d['C']+" "+d['X'] +" 无法格式化"
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+2
-3
@@ -86,7 +86,6 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
//目前查询库中是否已存在,用的是Oracle
|
||||
def index = 0
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
citys = ["三沙","临高","乐东","五指山","北屯","双河","可克达拉","定安","屯昌","昆玉","昌江","果洛州","海南州","玉树州","琼中","甘南","白沙","铁门关","阿里","黄南州"]
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
@@ -494,8 +493,8 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
|
||||
def date = "20200101"//DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
||||
def startMdate = DateUtils.parseFromTightByPattern("20200101","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern("20240518", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
|
||||
def startMdate = DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
+22
-22
@@ -33,7 +33,7 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\分期小区对比结果.xlsx"
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
@@ -167,14 +167,14 @@ class AddPriceTest {
|
||||
}
|
||||
|
||||
def addCcbPrice(filepath){
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(filepath,{ d->
|
||||
if(d['A']){
|
||||
// def cityId = cityCodeDataMap.get(d['N'])
|
||||
def map = findAvgPrice(d['G'], d['A'].toString().toDouble().toInteger())
|
||||
if(d['AJ']){
|
||||
def cityId = cityCodeDataMap.get(d['AK'])
|
||||
def map = findAvgPrice(d['AJ'], cityId)
|
||||
if(map){
|
||||
// d['AK'] = map.avg_price
|
||||
d['P'] = map.calc_type
|
||||
d['F'] = map.avg_price
|
||||
// d['P'] = map.calc_type
|
||||
// d['AM'] = map.credit_value
|
||||
}
|
||||
}
|
||||
@@ -183,7 +183,7 @@ class AddPriceTest {
|
||||
|
||||
|
||||
def findAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202406 order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
@@ -272,23 +272,23 @@ class AddPriceTest {
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
// if(!d['L']){
|
||||
// if(d['J'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
def dist = CommonDatautils.findDistrictById(d['AJ'])
|
||||
if(dist){
|
||||
d['A'] = dist.city
|
||||
d['B'] = dist.region
|
||||
d['C'] = dist.name
|
||||
d['D'] = dist.status
|
||||
d['E'] = dist.name_cat
|
||||
d['F'] = dist.estate_type2
|
||||
d['G'] = DateUtils.formatDateByPattern(dist.m_date,'yyyy-MM-dd')
|
||||
d['H'] = dist.master_district_name
|
||||
// d['A'] = dist.city
|
||||
// d['B'] = dist.region
|
||||
d['E'] = dist.name
|
||||
// d['D'] = dist.status
|
||||
// d['E'] = dist.name_cat
|
||||
// d['F'] = dist.estate_type2
|
||||
// d['G'] = DateUtils.formatDateByPattern(dist.m_date,'yyyy-MM-dd')
|
||||
// d['H'] = dist.master_district_name
|
||||
// d['N'] = dist.region
|
||||
// def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
// if(dbo){
|
||||
// d['D'] = dbo.avg_price
|
||||
//// d['X'] = dbo.cal_type
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(dbo){
|
||||
d['G'] = dbo.avg_price
|
||||
// d['X'] = dbo.cal_type
|
||||
// d['E'] = dbo.avg_price_2b
|
||||
// }
|
||||
}
|
||||
}
|
||||
// }
|
||||
// }
|
||||
|
||||
+2
-2
@@ -27,8 +27,8 @@ class QiliangTest {
|
||||
// task.run()
|
||||
|
||||
def data = [
|
||||
'address': '宁德市福安市中兴西路16号',
|
||||
'type': '2' //1:对公 0:对私
|
||||
'address': '江阴市澄杨路77号',
|
||||
'type': '1' //1:对公 0:对私
|
||||
]
|
||||
def s = task.getProxyResult(data);
|
||||
print(s);
|
||||
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import org.apache.http.HttpHost
|
||||
import org.apache.http.auth.AuthScope
|
||||
import org.apache.http.auth.UsernamePasswordCredentials
|
||||
import org.apache.http.client.CredentialsProvider
|
||||
import org.apache.http.client.config.RequestConfig
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
|
||||
import org.elasticsearch.action.ActionListener
|
||||
import org.elasticsearch.action.bulk.BackoffPolicy
|
||||
import org.elasticsearch.action.bulk.BulkProcessor
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.search.ClearScrollRequest
|
||||
import org.elasticsearch.action.search.SearchRequest
|
||||
import org.elasticsearch.action.search.SearchResponse
|
||||
import org.elasticsearch.action.search.SearchScrollRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestClient
|
||||
import org.elasticsearch.client.RestClientBuilder
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.function.BiConsumer
|
||||
|
||||
/**
|
||||
* 功能说明:Elasticsearch工具类
|
||||
*/
|
||||
class ElasticUtils89 {
|
||||
|
||||
|
||||
static getEsDataSource(String host){
|
||||
EsDataSource dataSource = new EsDataSource ();
|
||||
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
|
||||
Properties connectionProperties = new Properties();
|
||||
// connectionProperties.setProperty("user","developer");
|
||||
// connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
|
||||
|
||||
connectionProperties.setProperty("user","elastic");
|
||||
connectionProperties.setProperty("password","elasticsearch");//elasticsearch
|
||||
dataSource.setProperties(connectionProperties);
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.89"){
|
||||
EsDataSource dataSource = getEsDataSource(host)
|
||||
Connection connection = dataSource.getConnection()
|
||||
Statement statement = connection.createStatement()
|
||||
ResultSet rs = statement.executeQuery(sql)
|
||||
def columns = []
|
||||
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
|
||||
columns << rs.getMetaData().getColumnName(i)
|
||||
}
|
||||
rs.getMetaData().getColumnCount()
|
||||
while(rs.next()){
|
||||
def row = [:]
|
||||
for (String col : columns){
|
||||
row[col] = rs.getObject(col)
|
||||
}
|
||||
closure.call(row)
|
||||
}
|
||||
rs.close()
|
||||
connection.close()
|
||||
}
|
||||
|
||||
static RestHighLevelClient getClient(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient89(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param host
|
||||
* @return
|
||||
*/
|
||||
static createBulkProcessor(String host, RestHighLevelClient client = null) {
|
||||
|
||||
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
|
||||
@Override
|
||||
void beforeBulk(long executionId, BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
BulkResponse response) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
Throwable failure) {
|
||||
}
|
||||
};
|
||||
if(!client){
|
||||
client = getClient(host);
|
||||
}
|
||||
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
|
||||
@Override
|
||||
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
|
||||
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
|
||||
}
|
||||
}, listener).setBulkActions(10000)
|
||||
.setConcurrentRequests(20)
|
||||
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
|
||||
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭一个ES批处理器
|
||||
* @param
|
||||
* @return
|
||||
*/
|
||||
static closeBulkProcessor(BulkProcessor bulkProcessor) {
|
||||
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
|
||||
bulkProcessor.flush()
|
||||
bulkProcessor.close()
|
||||
}
|
||||
|
||||
|
||||
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
|
||||
|
||||
RestHighLevelClient client = getClient(host)
|
||||
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
final SearchHit hit = searchHits[k];
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try{
|
||||
if(closure.getMaximumNumberOfParameters() == 1){
|
||||
def update = closure.call(hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}else if(closure.getMaximumNumberOfParameters() == 2){
|
||||
def update = closure.call(hit.getId(), hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
executorService.shutdown()
|
||||
executorService.awaitTermination(1, TimeUnit.DAYS)
|
||||
bulkProcessor.close()
|
||||
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
|
||||
client.close()
|
||||
}
|
||||
|
||||
|
||||
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
|
||||
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
RestHighLevelClient client = getClient(host)
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def results = []
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
SearchHit searchHit = searchHits[k]
|
||||
searchHit.getSourceAsMap().put("id", searchHit.getId());
|
||||
results << searchHit.getSourceAsMap()
|
||||
}
|
||||
if(results.size()%100000==0){
|
||||
println "."+ new Date()
|
||||
}
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
client.close()
|
||||
return results;
|
||||
}
|
||||
}
|
||||
+184
-49
File diff suppressed because one or more lines are too long
+1
-1
@@ -129,7 +129,7 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
def run(){
|
||||
|
||||
fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
mcaDistMap = CommonDatautils.queryDistNameCity()
|
||||
mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
// priceMap = findAllDistrictSellPrice()
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import javax.crypto.Cipher;
|
||||
import javax.crypto.KeyGenerator;
|
||||
import javax.crypto.SecretKey;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.util.Base64;
|
||||
|
||||
public class DESUtil {
|
||||
|
||||
private static final String ALGORITHM = "DES";
|
||||
|
||||
// 生成密钥
|
||||
public static SecretKey generateKey() throws Exception {
|
||||
KeyGenerator keyGenerator = KeyGenerator.getInstance(ALGORITHM);
|
||||
keyGenerator.init(56); // DES密钥长度为56位
|
||||
return keyGenerator.generateKey();
|
||||
}
|
||||
|
||||
// 将密钥转换为字符串
|
||||
public static String keyToString(SecretKey secretKey) {
|
||||
return Base64.getEncoder().encodeToString(secretKey.getEncoded());
|
||||
}
|
||||
|
||||
// 将字符串转换为密钥
|
||||
public static SecretKey stringToKey(String keyStr) {
|
||||
byte[] decodedKey = Base64.getDecoder().decode(keyStr);
|
||||
return new SecretKeySpec(decodedKey, 0, decodedKey.length, ALGORITHM);
|
||||
}
|
||||
|
||||
// DES加密
|
||||
public static String encrypt(String data, SecretKey secretKey) throws Exception {
|
||||
Cipher cipher = Cipher.getInstance(ALGORITHM);
|
||||
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
|
||||
byte[] encryptedData = cipher.doFinal(data.getBytes());
|
||||
return Base64.getEncoder().encodeToString(encryptedData);
|
||||
}
|
||||
|
||||
// DES解密
|
||||
public static String decrypt(String encryptedData, SecretKey secretKey) throws Exception {
|
||||
Cipher cipher = Cipher.getInstance(ALGORITHM);
|
||||
cipher.init(Cipher.DECRYPT_MODE, secretKey);
|
||||
byte[] decodedData = Base64.getDecoder().decode(encryptedData);
|
||||
byte[] decryptedData = cipher.doFinal(decodedData);
|
||||
return new String(decryptedData);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
// 生成密钥
|
||||
// SecretKey secretKey = generateKey();
|
||||
// String keyStr = keyToString(secretKey);
|
||||
// System.out.println("密钥: " + keyStr);
|
||||
//
|
||||
//// // 加密数据
|
||||
// String data = "高娟";
|
||||
// String encryptedData = encrypt(data, secretKey);
|
||||
// System.out.println("加密后的数据: " + encryptedData);
|
||||
|
||||
// 解密数据
|
||||
SecretKey newSecretKey = stringToKey("tnb12345");
|
||||
String decryptedData = decrypt("5E55C890CED08EFB7E3A05A0BAECB7B100D71965E4AD0FE2", newSecretKey);
|
||||
System.out.println("解密后的数据: " + decryptedData);
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -12,7 +12,8 @@ import org.jsoup.Jsoup
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GaoDeApiUtils {
|
||||
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //15021384995
|
||||
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" //宝剑
|
||||
//15021384995:dfc2245bac8a7b2d5b62e060e59d4eca
|
||||
|
||||
/**
|
||||
* 冪帠僅蛌華硊<br>
|
||||
|
||||
Reference in New Issue
Block a user