房价网内部均价系统新增最近2个月的房源数据

This commit is contained in:
2024-08-12 11:50:45 +08:00
parent 9958587d12
commit f76253f983
20 changed files with 1995 additions and 133 deletions
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\20240619--深圳成交数据.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年6月深圳二手房成交.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -199,6 +199,16 @@ def findAllData(sqlInstance236, citycode){
}
//ExcelUtils.modify("D:\\数据\\建行\\保定202406.xlsx", { d ->
// if(d['G']){
// def obj = CommonDatautils.findDistrictById(d['G'])
// if(obj){
// d['H'] = obj.name
// }
// }
//})
////def url = "https://sh.anjuke.com"
////def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
@@ -1289,7 +1299,7 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240626人工指导价.xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['J']){
@@ -1308,9 +1318,9 @@ ExcelUtils.multiThreadingModify(fileName, {d->
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
// d['K'] = dist.estate_type2
// d['L'] = dist.status
// d['M'] = dist.name_cat
d['K'] = dist.estate_type2
d['L'] = dist.status
d['M'] = dist.name_cat
// d['H'] = dist.name
}
@@ -1352,24 +1362,57 @@ ExcelUtils.multiThreadingModify(fileName, {d->
//
//})
//def filePath = "D:\\数据\\建行\\房产估值\\指导价\\大庆删除小区.xlsx"
//
//def filePath = "D:\\数据\\数据建设\\小区\\嘉峪关处理.xlsx"
//ExcelUtils.modify(filePath, { d ->
// if(d['A']){
//
// def obj = CommonDatautils.findDistrictById(d['A'])
// if(obj){
//
// def data = new BasicDBObject()
// data.put("status", 1)
// data.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['A'])), new BasicDBObject('$set', data))
//// def obj = CommonDatautils.findDistrictById(d['A'])
// def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// def list = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district")
// if(list){
// if(list.size()==1){
// def dbo = list[0]
// if(dbo){
// def data = new BasicDBObject()
// data.put("region", d['B'])
// data.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', data))
// }
// }else{
// println d['C'] +" 有多个"
// }
// }
//// if(obj){
//// def lng = d['G']
//// def lat =d['H']
////
//// // 高德转百度
//// def baidu = BaiDuApiUtils.geoconv(d['G'].toString().toDouble(), d['H'].toString().toDouble(), 3)
//// def lng2 = baidu.x
//// def lat2 = baidu.y
////
////
//// def data = new BasicDBObject()
//// Long lng_new = d['G'].toString().toDouble() *10000000000L
//// Long lat_new = d['H'].toString().toDouble() *10000000000L
//// data.put("lng", lng_new)
//// data.put("lat", lat_new)
////
//// Long dLat2 = lat2 * 10000000000L
//// Long dLng2 = lng2 * 10000000000L
//// data.put("lat2", dLat2)
//// data.put("lng2", dLng2)
////
////// data.put("status", 1)
//// data.put("m_date", new Date())
//// DbUtils.seaweed.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', data))
//// }
// }
//
//})
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// if(d['B'] && !d['D']){
@@ -32,6 +32,10 @@ class TradeStatLog {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pssqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
@@ -41,7 +45,9 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
// task.run()//统计最近3个月抓取的成交数据量
task.exportTrade()//导出未入库成交
// task.exportTrade()//导出未入库成交
task.exportTrade2()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -584,6 +590,36 @@ class TradeStatLog {
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据")
}
def exportTrade2(){
// def startDate = "2024-06-01"
// def endDate = "2024-07-01"
def ids = findAllDealIds();
println ids.size()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d->
if(ids.contains(d['A'])){
d['V'] = "供"
}
if(d['S'] && !d['W']){
def dist = CommonDatautils.findDistrictById(d['S'])
if(!dist.name_cat && dist.name_cat!=0){
d['W'] = "name_cat过滤"
}
if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){
d['W'] = "小区过滤"
}
def key = d['B']+"##"+d['Q']+"##"+d['R']+"##"+d['D']+"##"+d['E'].toString().toDouble().toInteger()+"##"+d['J'].toString().substring(0,10)
def map = findDealByCommId(d['B'], d['S'])
def count = map.get(key)
if(count>=2){
d['W'] = "重复"
}
}
println d['A']
})
}
def deleteDistrict(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
if(d['G']){
@@ -637,11 +673,39 @@ class TradeStatLog {
return trades
}
def findAllDealIds(){
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'"
def ids = []
pssqlInstance.eachRow(sql,{r->
if(!ids.contains(r.gujia_bds_mdl_id)){
ids << r.gujia_bds_mdl_id
}
})
return ids
}
def findDealByCommId(city, commid){
def q = new BasicDBObject()
q.put("match_id", commid)
q.put("city", city)
q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2024-06-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2024-07-01")));
def dbcursor = DEAL_PRICE_COLL.find(q)
def map = [:]
for(def d : dbcursor){
def key = d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
def count = map.get(key)
if(!count){
count = 0
}
map.put(key, count+1)
}
return map
}
def findDealIds(city, startDate, endDate){
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
def ids = []
def detailInfos = []
sqlInstance.eachRow(sql,{r->
pssqlInstance.eachRow(sql,{r->
if(!ids.contains(r.dealid)){
ids << r.dealid
@@ -12,8 +12,8 @@ class OpenUserBatch {
def task = new OpenUserBatch();
// task.run();
// task.test();
// task.addShenZhenFjtUser();
task.addFujianFjtUser()
task.addShenZhenFjtUser();
// task.addFujianFjtUser()
}
def test(){
@@ -46,20 +46,20 @@ class OpenUserBatch {
}
def addShenZhenFjtUser(){
def datas = ExcelUtils.read("D:\\数据\\渤海\\开通线上评估客户经理名单.xlsx",0,1)
def datas = ExcelUtils.read("D:\\数据\\黑龙江银行\\大正银行端新增账户模板中国银行.xls",0,1)
def i = 0;
for(def data : datas){
i++;
def id = 252608 + i;
def id = 252893 + i;
BigDecimal bd = new BigDecimal(data[3].toString());
String mobile = bd.toPlainString();
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)";
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '18', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2025-12-31 00:00:00', '0', '', NULL)";
onLineSqlInstance.execute(sql);
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '姚静梦')";
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '王蕾')";
onLineSqlInstance.execute(sql2)
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'上海','上海:')"
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'哈尔滨,齐齐哈尔,大庆,牡丹江,佳木斯,绥化,鸡西,七台河,双鸭山,鹤岗,黑河,伊春,大兴安岭','哈尔滨:,齐齐哈尔:,大庆:,牡丹江:,佳木斯:,绥化:,鸡西:,七台河:,双鸭山:,鹤岗:,黑河:,伊春:,大兴安岭:')"
onLineSqlInstance.execute(sql3)
}
}
File diff suppressed because one or more lines are too long
@@ -7,6 +7,7 @@ import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
@@ -17,6 +18,8 @@ import net.sf.json.JSONObject
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import java.util.regex.Pattern
class DevProcess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
@@ -28,11 +31,13 @@ class DevProcess {
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020)
static main(args) {
def task = new DevProcess();
// task.run()
// task.parseCityAndDist()
task.devSplitRoad()
// task.devSplitRoad()
// task.devNameSplitRoad()
// task.addFlag()
// task.matchDev()
@@ -41,6 +46,7 @@ class DevProcess {
// task.updateCityAndDist()
// task.addDevAddres1() //按添加地址中包含开发区名称的地址
// task.addDevAddres2()//添加相差2公里范围内的地址
// task.addDevAddres3()
// task.matchDevAndAddAddress()
// task.stat()
@@ -56,6 +62,15 @@ class DevProcess {
// task.matchDevTownship()//添加乡镇街道
// task.addDevStreet()//添加区域为等级开发区下的街道到开发区对应表中
// task.matchNewCrawlDev()
// println DateUtils.add("2024-7-24", 7);
// task.addKfc()
// task.devToPark() //开发区下辖园区
task.checkTown()
}
@@ -334,15 +349,22 @@ class DevProcess {
}
def matchDev(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\福建省.xlsx",{d->
if(!d['Q'] && d['L']){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\省级开发区名单\\新疆开发区名单.xlsx",{d->
if(!d['Q'] && d['A']){
def q = new BasicDBObject()
q.put("name", d['L'])
q.put("city",d['G'])
// q.put("name", d['A'])
// q.put("city",d['G'])
// q.put('alias', new BasicDBObject('$in',[d['A']]))
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+d['A']+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
def obj = development_zone_CONN.findOne(q)
if(obj){
d['Q'] = obj._id.toString()
d['R'] = obj.name
d['C'] = obj._id.toString()
d['D'] = obj.name
if(obj.name == d['A']){
d['E'] = "相同"
}else{
d['E'] = "包含"
}
}
}
},0);
@@ -509,6 +531,38 @@ class DevProcess {
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
}
def addDevAddres3(){
def index = 0
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\新疆省入库.xlsx",{d->
if(d['A']){
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['A']))
if(obj){
d['C'] = obj._id.toString()
def q2 = new BasicDBObject()
q2.put("name", d['A'])
q2.put("address", d['B'])
q2.put("type","地址")
def sameObj = DEV_MAPPING_CONN.findOne(q2)
if(!sameObj){
index++
def udata = new BasicDBObject()
udata.put("province", obj.province)
udata.put("city", obj.city)
udata.put("district", obj.district)
udata.put("name",obj.name)
udata.put("type", "地址")
udata.put("address", d['B'])
DEV_MAPPING_CONN.save(udata)
}
}
}
},0)
println index
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
}
def stat(){
def dbcursor = development_zone_CONN.find(new BasicDBObject())
def total = 0
@@ -777,4 +831,128 @@ class DevProcess {
})
}
def matchNewCrawlDev(){
def query = new BasicDBObject()
query.put("match_id", new BasicDBObject('$exists', false))
def list = kfq_COLL.find(query).toList()
for(def d : list){
def q = new BasicDBObject()
// q.put("name", d.name)
q.put('alias', new BasicDBObject('$in',[d.name]))
def obj = development_zone_CONN.findOne(q)
if(obj){
def udata = new BasicDBObject()
udata.put("match_id", obj._id.toString())
udata.put("match_name", obj.name)
kfq_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
}
}
}
def addKfc(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\投促中国开发区(1).xlsx",{d->
if(d['I']){
if(d['I'] == "添加"){
def udata = new BasicDBObject()
udata.put("create_time" , new Date())
udata.put("industry",d['D'])
if(d['E'].contains("国家级")){
udata.put("level" ,"国家级")
}else if(d['E'].contains("省级")){
udata.put("level" ,"省级")
}
udata.put("name" ,d['F'])
udata.put("province" ,d['G'])
udata.put("type" ,d['H'])
udata.put("city" ,d['B'])
udata.put("district" ,d['J'])
development_zone_CONN.save(udata)
}else{
def q = new BasicDBObject()
q.put("name", d['I'])
def obj = development_zone_CONN.findOne(q)
if(obj){
def alias = []
if(obj.alias){
alias = obj.alias
}
if(!alias.contains(d['F'])){
alias << d['F']
}
if(alias && alias.size()!=obj.alias.size()){
def updateData = new BasicDBObject()
updateData.put("alias", alias)
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',updateData))
}
}
}
}
})
}
def devToPark(){
def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\开发区下辖街道-处理后表格.xlsx",0, 1)
def rets = []
rets << ["开发区名称", "下辖街道", "城市", "区域"]
for(def d : datas){
if(d['E']){
def arrs = d['E'].split(",")
def q = new BasicDBObject().append("name", d['B'])
def obj = development_zone_CONN.findOne(q)
if(obj){
for(def park : arrs){
rets << [d['B'], park, obj.city, obj.district]
}
}
}
}
ExcelUtils.write("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx", rets, "下辖街道")
}
def checkTown(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx",{d->
if(!d['E']){
if(d['C'] && d['D']){
// if(d['A']=="长春高新技术产业开发区"){
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B'])
if(jsonObject1) {
if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
def township = jsonObject1.get("township")
def district = jsonObject1.get("district")
d['E'] = "接口"
if(district && district!='null'){
d['F'] = district
}
if(township && township!='null'){
d['G'] = township
}
d['H'] = jsonObject1.get("townCode")
if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){
d['I'] = jsonObject1.get("provinceCode")
}
}
// }
}
// def q = new BasicDBObject()
// q.put("city", d['C'])
//// q.put("district", d['D'])
// q.put("mzb_district_name", d['D'])
// q.put("type", "乡镇街道")
// q.put("street", d['B'])
// def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
// if(obj){
// d['E'] = "可入库2"
// d['F'] = obj.district
// }
}
}
},1)
}
}
@@ -0,0 +1,277 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
class DevUpdateTestBatch {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\文心一言获取开发区下辖园区及街道.xlsx"//开发区-园区.xlsx
// task.addDev(path)
task.addDevAddress(path)
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
}
def addDev(path){
ExcelUtils.modify(path,{ d->
if(!d['I'] && d['A']){
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
if(!obj){
def o = new BasicDBObject()
o.put("province", d['B'])
o.put("city", d['C'])
o.put("district", d['D'])
o.put("name", d['A'])
o.put("level", d['E'])
o.put("type", d['F'])
if(d['G']){
o.put("township", d['G'])
}
if(d['H']){
def arr = d['H'].split(",")
if(arr){
o.put("alias", arr)
}
}
def id = addMongoDB(o)
o.remove("_id")
o.id = id.toString()
o.remove("create_time")
addEs(o, "192.168.0.89", "development_zone")
addEs(o, "192.168.0.52", "development_zone")
d['I'] = "已完成"
}else{
println d['A'] +" 已存在"
}
}
})
}
def addMongoDB(o){
o.put("create_time", new Date())
development_zone_100_CONN.save(o)
DEVELOPMENT_ZONE_43_COLL.save(o)
return o._id
}
def addEs(map, host, indexName){
BulkRequest request = new BulkRequest();
int processedCount = 0;
def client = null
if(host && host == "192.168.0.89"){
client = ElasticUtils89.getClient(host)
}else if(host && host == "192.168.0.52"){
client = ElasticUtils.getClient(host)
}
request.add(new IndexRequest(indexName, "_doc", map.id).source(map, XContentType.JSON));
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
}
def addDevAddress(path){
ExcelUtils.modify(path,{ d->
if(!d['D'] && d['A']){
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
if(obj){
def q = new BasicDBObject("name", d['A']).append("type",d['B'])
if(d['B'] == "乡镇街道"){
q.put("township", d['C'])
}else if(d['B'] == "地址"){
q.put("address", d['C'])
}else if(d['B'] == "园区"){
q.put("park", d['C'])
}
def mappingObj = DEV_MAPPING_100_CONN.findOne(q)
if(!mappingObj){
def o = new BasicDBObject()
o.put("province", obj.province)
o.put("city", obj.city)
o.put("district", obj.district)
o.put("name", obj.name)
o.put("type", d['B'])
if(d['B'] == "乡镇街道"){
o.put("township", d['C'])
}else if(d['B'] == "地址"){
o.put("address", d['C'])
}else if(d['B'] == "园区"){
o.put("park", d['C'])
}
def date = new Date()
o.put("c_date", date)
o.put("version", date.getTime())
DEV_MAPPING_100_CONN.save(o)
o.id = o._id.toString()
o.remove("_id")
o.remove("c_date")
addEs(o, "192.168.0.89", "dev_mapping")
try{
addEs(o, "192.168.0.52", "dev_mapping")
}catch(Exception e){
println "52 es error: "+e.printStackTrace()
}
d['D'] = "已完成"
}else{
d['D'] = "存在"
println d['A'] +" "+ d['C']+" 重复"
}
}else{
println d['A'] +" 不存在"
}
}
},2)
}
def updateHasGov(province, city, district, street, hasGov){
def q = new BasicDBObject()
q.put("province", province)
q.put("city", city)
q.put("district", district)
q.put("street", street)
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
//修改43
// ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',new BasicDBObject('has_gov',hasGov)))
//修改89 administrative_mapping
update89EsAdministrativeMapping(province, city, district, street, hasGov)
//修改89 administrative_mapping
update52EsAdministrativeMapping(province, city, district, street, hasGov)
}else{
println province+" "+city+" "+district+" "+street+" 没找到"
}
//修改100 community
//修改89 community
//修改52 community
}
def update89EsAdministrativeMapping(province, city, district, street, hasGov){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.has_gov='${hasGov}'",
"lang": "painless"
},
"query": {
"bool": {
"must": [
{
"match": {
"province_name": "${province}"
}
},
{
"match": {
"city_name":"${city}"
}
},
{
"match": {
"area_name":"${district}"
}
},
{
"match": {
"street_name":"${street}"
}
}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
// def response = JSONObject.fromObject(res)
println(res)
}
def update52EsAdministrativeMapping(province, city, district, street, hasGov){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.52:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.has_gov='${hasGov}'",
"lang": "painless"
},
"query": {
"bool": {
"must": [
{
"match": {
"province_name": "${province}"
}
},
{
"match": {
"city_name":"${city}"
}
},
{
"match": {
"area_name":"${district}"
}
},
{
"match": {
"street_name":"${street}"
}
}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
// def response = JSONObject.fromObject(res)
println(res)
}
}
@@ -10,7 +10,8 @@ class DistrictAliasProcess {
static main(args) {
def task = new DistrictAliasProcess();
task.run()
// task.run()
task.fenQiProcess()
}
def run(){
@@ -42,4 +43,36 @@ class DistrictAliasProcess {
}
})
}
def fenQiProcess(){
def q = new BasicDBObject()
q.put("cat","district")
q.put("status", 0)
q.put("master_district_name", 'master')
def dbcursor = DbUtils.seaweed.find(q)
def rets = []
for(def d : dbcursor){
rets << ["_id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name]
}
for(def r : rets){
def mainName1 = r.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
def mainName2 = r.name.toString().replaceAll("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$", "")
def lastObj = null
for(def j : rets){
if((j.name==mainName1 || mainName2==j.name) && r._id!=j._id && r.region==j.region && r.city==j.city){
lastObj = j;
break
}
}
if(lastObj){
println r
def udata = new BasicDBObject()
udata.put("master_district_name","")
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(r._id)), new BasicDBObject('$set', udata))
}
}
}
}
@@ -28,23 +28,23 @@ class GXDdistrictMatch {
static main(args) {
def task = new GXDdistrictMatch();
// task.run()
// task.addOldMatch()
task.addOldMatch()
// task.runDb()
// task.updateDistStatus()//恢复name_cat不正常数据
// task.saveToDB() //保存新增小区
// task.restoreDist()//恢复小区status=0
task.runMatch()
// task.runMatch()
}
def addOldMatch(){
//老匹配
def fileName = "D:\\数据\\国信达\\非正常数据.xlsx"//11城重新匹配.xlsx
def fileName = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"//11城重新匹配.xlsx
ExcelUtils.multiThreadingModify(fileName, { d ->
println d['C']
if(!d['AJ']){
def jsonObject = OpenApiClient.addressMatch(d['AD'], d['H'], null, false, d['AE'], d['C'])
if(!d['AJ'] && d['I']){
def jsonObject = OpenApiClient.addressMatch(d['C'], d['I'], "district", false,null, d['D'])
if (jsonObject != null && !jsonObject.containsKey("result")) {
d['AJ'] = jsonObject.get("id")
d['AK'] = jsonObject.get("city")
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicLong
class PoiLocation {
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "didata_map_poi_single", MAP_POI_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
@@ -35,21 +35,52 @@ class PoiLocation {
static main(args) {
def task = new PoiLocation();
task.run();
// task.run();
task.regionAndStreetProcess()
}
def run(){
def citys = ["芜湖市","重庆市","杭州市","泉州市","东莞市","福州市","厦门市","佛山市","天津市","武汉市","宁波市","郑州市","长沙市","青岛市","上海市","西安市","温州市","无锡市","南京市","湖州市","合肥市","临沂市","石家庄市","昆明市","金华市","保定市","成都市","济南市","北京市","沈阳市","潍坊市","广州市","惠州市","嘉兴市","赣州市","苏州市","南通市","南宁市","台州市","南阳市","徐州市","长春市","深圳市","哈尔滨市","漳州市","中山市","常州市","烟台市","大连市","邯郸市","济宁市","盐城市","绍兴市","唐山市","洛阳市","阜阳市","菏泽市","贵阳市","沧州市","遵义市","莆田市","南昌市","商丘市","周口市","邢台市","江门市","衡阳市","邵阳市","信阳市","廊坊市","扬州市","泰州市","新乡市","太原市","德州市","淄博市","宁德市","绵阳市","汕头市","聊城市","淮安市","驻马店市","黄冈市","宿迁市","三明市","六安市","龙岩市","桂林市","岳阳市","南平市","揭阳市","上饶市","九江市","南充市","连云港市","襄阳市","泰安市","常德市","郴州市","安阳市","安庆市","湛江市","永州市","茂名市","曲靖市","宜宾市","肇庆市","宿州市","兰州市","运城市","镇江市","滁州市","威海市","梅州市","宜春市","清远市","亳州市","怀化市","荆州市","平顶山市","开封市","玉林市","咸阳市","宜昌市","株洲市","乌鲁木齐市","滨州市","渭南市","临汾市","榆林市","银川市","衡水市","毕节市","枣庄市","吉安市","张家口市","柳州市","许昌市","泸州市","呼和浩特市","达州市","德阳市","赤峰市","珠海市","孝感市","益阳市","娄底市","乐山市","恩施土家族苗族自治州","焦作市","湘潭市","海口市","红河哈尼族彝族自治州","黔南布依族苗族自治州","河源市","汉中市","日照市","大理白族自治州","濮阳市","黔东南苗族侗族自治州","秦皇岛市","晋中市","宝鸡市","吉林市","包头市","蚌埠市","东营市","宣城市","韶关市","衢州市","十堰市","承德市","丽水市","眉山市","鞍山市","昭通市","凉山彝族自治州","长治市","抚州市","鄂尔多斯市","吕梁市","百色市","齐齐哈尔市","贵港市","潮州市","黔西南布依族苗族自治州","广安市","淮南市","河池市","大同市","铜仁市","阳江市","咸宁市","营口市","绥化市","马鞍山市","安康市","荆门市","忻州市","广元市","内江市","遂宁市","西宁市","大庆市","汕尾市","文山壮族苗族自治州","锦州市","延安市","巴中市","呼伦贝尔市","楚雄彝族自治州","黄石市","玉溪市","六盘水市","通辽市","钦州市","云浮市","梧州市","漯河市","晋城市","庆阳市","安顺市","丹东市","自贡市","朝阳市","葫芦岛市","天水市","三门峡市","湘西土家族苗族自治州","北海市","陇南市","普洱市","延边朝鲜族自治州","保山市","定西市","随州市","喀什地区","黄山市","佳木斯市","巴彦淖尔市","牡丹江市","舟山市","资阳市","商洛市","松原市","锡林郭勒盟","伊犁哈萨克自治州","萍乡市","铁岭市","乌兰察布市","丽江市","景德镇市","张家界市","抚顺市","通化市","淮北市","来宾市","临沧市","阿克苏地区","贺州市","盘锦市","昌吉回族自治州","池州市","雅安市","三亚市","平凉市","四平市","酒泉市","西双版纳傣族自治州","鹤壁市","辽阳市","白城市","吴忠市","崇左市","阜新市","朔州市","白银市","拉萨市","巴音郭楞蒙古自治州","张掖市","铜陵市","武威市","临夏回族自治州","德宏傣族景颇族自治州","新余市","甘孜藏族自治州","兴安盟","本溪市","鄂州市","黑河市","阿坝藏族羌族自治州","阳泉市","防城港市","鸡西市","仙桃市","鹰潭市","固原市","白山市","中卫市","双鸭山市","攀枝花市","石嘴山市","和田地区","海东市","塔城地区","日喀则市","济源市","伊春市","天门市","辽源市","潜江市","那曲市","铜川市","海西蒙古族藏族自治州","鹤岗市","昌都市","阿拉善盟","乌海市","儋州市","哈密市","克拉玛依市","七台河市","金昌市","甘南藏族自治州","琼海市","迪庆藏族自治州","石河子市","阿勒泰地区","文昌市","玉树藏族自治州","嘉峪关市","万宁市","怒江傈僳族自治州","林芝市","海南藏族自治州","吐鲁番市","澄迈县","陵水黎族自治县","博尔塔拉蒙古自治州","山南市","阿里地区","大兴安岭地区","海北藏族自治州","东方市","乐东黎族自治县","克孜勒苏柯尔克孜自治州","果洛藏族自治州","临高县","阿拉尔市","黄南藏族自治州","神农架林区","屯昌县","定安县","昌江黎族自治县","五家渠市","保亭黎族苗族自治县","琼中黎族苗族自治县","五指山市","白沙黎族自治县","北屯市","图木舒克市","可克达拉市","双河市","铁门关市","毕节地区","昆玉市","铜仁地区","海东地区","三沙市","吐鲁番地区","哈密地区","日喀则地区","林芝地区","昌都地区","山南地区","那曲地区"]
def citys = [""]
citys = citys.reverse()
for(def city : citys){
findPois(city)
}
}
def regionAndStreetProcess(){
def citys = ['南平市','泉州市', '厦门市', '福州市', '龙岩市', '漳州市', '宁德市', '三明市', '莆田市']
for(def city : citys){
def datas = findAllPoisByCity(city)
process(datas)
}
}
def findAllPoisByCity(city){
def q = new BasicDBObject()
q.put("city", city)
def keys = new BasicDBObject()
keys.put("_id", 1)
keys.put("province", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("township", 1)
keys.put("location", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
def findPois(city){
def q = new BasicDBObject()
q.put("city", city)
q.put("run_status", new BasicDBObject('$ne', 1))
// q.put("run_status", new BasicDBObject('$ne', 1))
def keys = new BasicDBObject()
keys.put("_id", 1)
@@ -87,6 +118,32 @@ class PoiLocation {
return rets;
}
def process(datas){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def d = datas[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
dataRun(d)
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
return rets
}
def checkData(def index, def datas){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
@@ -190,6 +247,33 @@ class PoiLocation {
return map;
}
def dataRun(d){
def regions = getFullRegion(d)
if(d.location) {
//判断区域否在围栏里
def regionFlag = null
if (d.district) {
for (def rs : regions) {
if (rs.shape && rs.shape.coordinates) {
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
}
}
}
if (regionFlag) {//在围栏里
//判断township是否在统计局中
// if(d.township){
// town = getSimilarTown(regions, d.city, d.township)
// }
}else{//不在转栏里,重新搜索
println(d)
}
}
}
def getFullRegion(d){
def regions = regionMap.get(d.city)
def names = []
@@ -2,8 +2,10 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@@ -12,6 +14,8 @@ import com.mongodb.client.AggregateIterable
import org.bson.Document
import org.bson.types.ObjectId
import java.util.regex.Pattern
/**
* 乡镇街道中间表
* 包含市,区,乡镇街道别名,来源 统计局community表
@@ -26,6 +30,9 @@ class TownMiddleProcess {
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_43base_COLL = MongoUtils.getCollectionByName("fangjia_base", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
DBCollection CITYPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
@@ -67,7 +74,15 @@ class TownMiddleProcess {
// checkTown()
//区域民政部名称更新到乡镇街道中
// updateTownshipByRegion()
// addAliasFromExcel()
//增加简称及简拼
// addSimpleName()
//根据民政部数据修改行政区划中间表
// mzbDistrictUpate()
// addCity()
}
@@ -422,14 +437,16 @@ class TownMiddleProcess {
for(def d : dbcursor){
datas << d
if (datas.size() >= 2000) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
println datas.size()
datas.clear();
}
}
if (datas.size() >= 0) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
datas.clear();
}
}
@@ -929,4 +946,222 @@ class TownMiddleProcess {
ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true)
}
}
def addAliasFromExcel(){
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道合并或者变更20240627.xlsx",{ row->
def q = new BasicDBObject("type", "乡镇街道")
q.put("city", row['A'])
q.put("street", row['D'])
if(row['B']){
q.put("district", row['B'])
}
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
row['F'] = '是'
def alias = obj.alias
if(!alias){
alias = []
}
if(!alias.contains(row['C'])){
alias << row['C']
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
def updateQuery = new BasicDBObject("_id", obj._id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
})
}
def addSimpleName(){
def q = new BasicDBObject()
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
for(def d : dbcursor){
def prov = ProvinceData.getSimpleProvince(d.province)
if(prov){
def udata = new BasicDBObject()
udata.put("province_short", prov)
def updateQuery = new BasicDBObject("_id", d._id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
// def city = d.city.replace("市","").replace("区","").replace("县","")
// def udata = new BasicDBObject()
// udata.put("city_short", city)
// if(d.district && d.district.size()>=3){
// def region= d.district.replace("区","").replace("镇","").replace("乡","")
// udata.put("district_short", region)
// }
// def updateQuery = new BasicDBObject("_id", d._id)
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
// def city = RegionData.formatCity(d.city)
// if(city && d.mzb_district_name){
// def region = RegionData.formatRegion(city, d.mzb_district_name)
// if(region){
// def udata = new BasicDBObject()
// udata.put("city_short", city)
// udata.put("district_short", region)
//
// def updateQuery = new BasicDBObject("_id", d._id)
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
// }
// }
}
}
def mzbDistrictUpate(){
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\民政部\\2014-2022年.xlsx",{ row->
if(!row['I']){
def updateType = row['E']
def prov = row['B']
def newStreet = row['G']
if(["新设街道","新设镇","新增街道","新建街道","新建乡镇","新设乡","新增乡","新增建制"].contains(updateType)){
def dbo = findStreets(prov, newStreet, row['F'].toString().toDouble().toInteger().toString())
if(dbo){
row['I'] = dbo.city
row['J'] = dbo.district
row['K'] = "存在"
row['L'] = dbo.tjj_street_code
row['M'] = dbo.pid
}
}else if(["撤乡设镇","撤乡建镇","撤乡设乡","撤乡、镇设镇","撤销设街道","撤销建街道","撤销街道建镇","撤镇设镇、街道","撤销镇新设镇","乡镇合并","整建制划归","撤镇建新镇","区划变更","撤镇、街道设镇","撤街道建镇","撤销乡、镇新设镇","撤镇、街道设街道","撤乡、街道设街道","撤乡设街道","撤乡建街道","撤镇设街道","撤镇建街道","名称变更","撤街道设镇","撤乡、镇设街道","撤街道、镇、乡","并入其他建制",'撤镇设镇',"合并","合并建镇","更名","撤销合并","合并建制","合并镇","合并街道","名称变更","合并调整","合并建街道","撤镇设街道、镇","撤乡、镇设乡","撤镇乡设街道","区划调整","合并乡","撤镇设市","区划代码调整","区划代码变更","撤销乡、镇设街道"].contains(updateType)){
if(row['F']){
def dbo = null
def code = row['F'].toString()
if(!code.contains(" ")){
dbo = findStreets(prov, newStreet, code.toDouble().toInteger().toString())
}
if(dbo){
row['I'] = dbo.city
row['J'] = dbo.district
row['L'] = dbo.tjj_street_code
row['M'] = dbo.pid
def alias = dbo.alias
if(!dbo.alias){
alias = []
}
if(row['D']){
if(!alias.contains(row['D'])){
row['K'] = "存在(加别名)"
alias << row['D']
addAlias(dbo._id, alias)
}else{
row['K'] = "存在"
}
}else{
row['K'] = "存在"
}
}
}
}else if(updateType == "撤街道设街道"){
row['K'] = "人工处理"
}else if(updateType == "隶属关系变更"){
row['K'] = "人工处理"
}else if(["撤销镇","撤乡","撤销街道","撤销","撤街道设社区工作站","撤销建制"].contains(updateType)){
row['K'] = "人工处理"
}else if(updateType =='撤销乡'){
if(!row['F']){
row['K'] = "人工处理"
}
}
if(prov && newStreet && (!row['K'] || (row['K'] && !row['K'].contains("存在")) )){
def dbo = findStreets(prov, newStreet, null)
if(dbo){
row['I'] = dbo.city
row['J'] = dbo.district
row['K'] = "存在-编码不同"
row['L'] = dbo.tjj_street_code
row['M'] = dbo.pid
def alias = dbo.alias
if(!dbo.alias){
alias = []
}
if(row['D']){
if(!alias.contains(row['D'])){
row['K'] = "存在(加别名)"
alias << row['D']
// addAlias(dbo._id, alias)
}else{
row['K'] = "存在"
}
}else{
row['K'] = "存在"
}
}
}
}
})
}
def addCity(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\乡镇街道\\民政部\\2014-2022年.xlsx",{ row->
def citycode = null
if(row['F']){
citycode = row['F'].toString().substring(0, 4)
}else if(row['C']){
citycode = row['C'].toString().substring(0, 4)
}
if(citycode){
def q = new BasicDBObject()
q.put("mzb_city_code", citycode+"00")
q.put("type", "城市")
def dbo = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(dbo){
row['O'] = dbo.city
}
}
})
}
def findStreets(prov, newStreet, streetCode){
def q = new BasicDBObject()
Pattern provPattern = Pattern.compile(prov+'.*$', Pattern.CASE_INSENSITIVE);
q.put("province", new BasicDBObject('$regex', provPattern))
q.put("street", newStreet)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
def list = []
def dbo = null
for(def d : dbcursor){
list << d
}
if(list && list.size()==1){
dbo = list[0]
}else if(list && list.size()>1){
if(streetCode){
for(def d : list){
if(d.tjj_street_code && d.tjj_street_code.contains(streetCode)){
dbo = d
break;
}else if(streetCode.contains(d.pid)){
dbo = d
break;
}
}
}
}
return dbo
}
def addAlias(id, alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
udata.put("operator","高娟-批量")
def updateQuery = new BasicDBObject("_id", id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.CosineSimilarAlgorithm
@@ -9,12 +10,14 @@ import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class TownshipStrandUpdate {
@@ -26,13 +29,26 @@ class TownshipStrandUpdate {
def polylineMap = null
def newAdvMappingMap = null
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com")
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection dev_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
static main(args) {
def task = new TownshipStrandUpdate()
// task.process()
task.updateTown()
// task.updateTown()
// task.stat()
// task.updateDiffRegionTown()
// task.cleanTown()
// task.checkTownFromMongo()
// task.updateTownMapping()
// task.updateDevMapping()
println new Date().getTime()
// task.updateSpecCityTown()
}
@@ -43,21 +59,21 @@ class TownshipStrandUpdate {
polylineMap = CommonDatautils.findAllTownRanges()
newAdvMappingMap = CommonDatautils.queryNewTownByCity()
// def citys = ["唐山","邯郸","石家庄","保定","秦皇岛","廊坊","沧州","张家口","衡水","邢台","承德"]//CommonDatautils.findAllCity()
def citys = CommonDatautils.findAllCity()
def citys = ["澄迈","临高"]//CommonDatautils.findAllCity(),"东方","澄迈","临高"
citys = citys.reverse()
for(def city : citys){
//
if(["中山","东莞"].contains(city)){
continue
}
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
// if(["中山","东莞"].contains(city)){
// continue
// }
def fileName = "D:\\数据\\数据建设\\乡镇街道\\写字楼\\写字楼乡镇街道格式化-${city}.xlsx"
File file=new File(fileName);
if(!file.exists()){
def dists = CommonDatautils.findAllDistrictsFromMg(city, "shop", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
// if(!file.exists()){
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
dists = checkData(dists)
def datas = createDatas(dists)
ExcelUtils.write(fileName, datas, "${city}")
}
// }
}
}
@@ -162,6 +178,10 @@ class TownshipStrandUpdate {
def running(d){
def fullCity = fullCityMap.get(d.city)
def regions = getFullRegion(fullCity, d)
if(!regions && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(fullCity.city)){
regions = []
regions << fullCity.city
}
def gaodeMap = null
def town = null
def mapTown = null
@@ -217,6 +237,10 @@ class TownshipStrandUpdate {
}
if( gaodeMap.district && gaodeMap.district!='[]'){
mapRegion = gaodeMap.district
}else{
if(mapCity && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(mapCity)){
mapRegion = mapCity
}
}
if(gaodeMap.township && gaodeMap.township!='[]'){
mapTownship = gaodeMap.township
@@ -331,7 +355,7 @@ class TownshipStrandUpdate {
def names = []
if(regions){
for(def r : regions){
if(r.mzb_district_name.contains(d.region)){
if(r.mzb_district_name.contains(d.region) || r.district_short.contains(d.region)){
names << r.district
}
}
@@ -381,7 +405,7 @@ class TownshipStrandUpdate {
}
}
}
if(district || count>10){
if(township || district || count>10){
stop = true
}else{
count ++
@@ -397,4 +421,212 @@ class TownshipStrandUpdate {
map.format_addr = format_addr
return map;
}
def updateDiffRegionTown(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx"
ExcelUtils.multiThreadingModify(fileName,{d->
def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop")
d['AA'] = lists.size()
//只处理库中只有一个的
if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){
def newRegion = RegionData.formatRegion(d['B'], d['M'])
if(newRegion){
if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
def dbo = lists[0]
def arrs = d['X'].split(" ")
def udata = new BasicDBObject()
udata.put("region", newRegion)
udata.put("m_date", new Date())
udata.put("township", arrs[1])
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟")
d['AB'] = "已处理"
}
}
}
})
}
def cleanTown(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx"
ExcelUtils.modify(fileName,{d->
if(d['AC']){
def dbo = CommonDatautils.findDistrict(d['B'], d['C'], d['D'], "district")
if(dbo && dbo.township){
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("township", "")
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "office", "update-town","高娟")
}
}
})
}
def checkTownFromMongo(){
def citys = CommonDatautils.findAllCity()
fullCityMap = CommonDatautils.queryFullNameByCity()
regionMap = CommonDatautils.queryDistNameByCity()
commMap = CommonDatautils.queryCommunityByCity()
citys = citys.reverse()
def index = 0
for(def city : citys) {
index++
if (["中山", "东莞"].contains(city)) {
continue
}
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city", "region", "name", "address", "cat", "estate_type2", "status", "township", "lng", "lat"])
for (def d : dists) {
if (d.township) {
def fullCity = fullCityMap.get(d.city)
def regions = getFullRegion(fullCity, d)
def town = getSimilarTown(regions, fullCity, d.township)
if (!town) {
town = findStreet(d.city, d.city, d.township)
if (!town) {
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("township", "")
DbUtils.seaweed.update(new BasicDBObject('_id',d._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(d.city, d.region, d.name, "office", "update-town","高娟")
println d.city + " " + d.region + " " + d.township + " " + d.name
}
}
}
}
println index + " " + city + " 完成"
}
}
def findStreet(city, district, street){
def q = new BasicDBObject()
Pattern provPattern = Pattern.compile(city+'.*$', Pattern.CASE_INSENSITIVE);
q.put("city", new BasicDBObject('$regex', provPattern))
q.put("district", new BasicDBObject('$regex', Pattern.compile(district+'.*$', Pattern.CASE_INSENSITIVE)))
q.put("street", new BasicDBObject('$regex', Pattern.compile(street+'.*$', Pattern.CASE_INSENSITIVE)))
def list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
if(list){
if(list.size()==1){
return list[0]
}else{
println city+ " "+district+" "+street+" 有多个"
}
}
return null
}
def updateTownMapping(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\双方区域对比.xlsx"
ExcelUtils.modify(fileName,{d->
if(d['F']){
// def udata = new BasicDBObject()
// udata.put("mzb_district_name", d['F'])
//
// def updateQuery = new BasicDBObject()
// updateQuery.put("city", d['C'])
// updateQuery.put("district", d['D'])
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata), false, true)
def city = RegionData.formatCity(d['C'])
if(city){
def region = RegionData.formatRegion(city, d['D'])
if(region){
def updateQuery = new BasicDBObject()
updateQuery.put("category", "region")
updateQuery.put("city", city)
updateQuery.put("name", region)
def list = REGION_BLOCK_COLL.find(updateQuery).toList()
if(list && list.size()==1){
def dbo = list[0]
def query = new BasicDBObject()
query.put("_id", dbo._id)
// def pattern = dbo.pattern
// if(pattern.contains(d['F'])){
// pattern.remove(d['F'])
// }
def udata = new BasicDBObject()
udata.put("org_name", d['F'])
// udata.put("pattern",pattern)
REGION_BLOCK_COLL.update(query, new BasicDBObject('$set',udata), false, true)
}else{
println city +" "+region+" "+list.size()
}
}
}
}
})
}
def updateDevMapping(){
def fileName = "D:\\数据\\数据建设\\开发区\\开发区映射表重复数据.xlsx"
ExcelUtils.modify(fileName,{d->
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("district", d['B'])
q.put("name",d['C'])
q.put("type", "乡镇街道")
q.put("township", d['D'])
def list = dev_MAPPING_COLL_100.find(q).toList()
if(list && list.size()==2){
def obj = list[0]
def removeQuery = new BasicDBObject()
removeQuery.put("_id", obj._id)
dev_MAPPING_COLL_100.remove(removeQuery)
}else{
println q
}
})
}
def updateSpecCityTown(){
def citys = ["中山","东莞","潜江","天门","神农架","五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","阿拉尔","图木舒克","五家渠","北屯","铁门关","双河","可克达拉"]
// def citys = ["中山"]
for(def city : citys){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
ExcelUtils.modify(fileName,{d->
def list = CommonDatautils.findDistrictList(d['A'], null , d['C'], "office")
if(list.size()>1){
println d['A'] +" "+d['C'] + " 有多个"
}else if(list.size()==1){
def dbo = list[0]
def town = null
// if(d['X']){
// town = d['X']
// }else
if(d['R']){
town = d['R']
}
if(!town && d['M']){
town = d['M']
}
def region = RegionData.formatRegion(d['A'], town)
if(region){
def q = new BasicDBObject()
q.put("region", region)
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("township", town)
udata.put("region", region)
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
}else{
println d['A']+" "+d['C']+" "+d['X'] +" 无法格式化"
}
}
})
}
}
}
@@ -86,7 +86,6 @@ class FjwBigdataForsaleBatch extends BaseBatch{
//目前查询库中是否已存在,用的是Oracle
def index = 0
def citys = CcbCommonUtils.getRunningCitys(args)
citys = ["三沙","临高","乐东","五指山","北屯","双河","可克达拉","定安","屯昌","昆玉","昌江","果洛州","海南州","玉树州","琼中","甘南","白沙","铁门关","阿里","黄南州"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -494,8 +493,8 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
def date = "20200101"//DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern("20200101","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern("20240518", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
@@ -33,7 +33,7 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\分期小区对比结果.xlsx"
def filePath = "D:\\数据\\建行\\房产估值\\20240616-20240715系统业务反馈.xlsx"
// addGXDPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
@@ -167,14 +167,14 @@ class AddPriceTest {
}
def addCcbPrice(filepath){
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(filepath,{ d->
if(d['A']){
// def cityId = cityCodeDataMap.get(d['N'])
def map = findAvgPrice(d['G'], d['A'].toString().toDouble().toInteger())
if(d['AJ']){
def cityId = cityCodeDataMap.get(d['AK'])
def map = findAvgPrice(d['AJ'], cityId)
if(map){
// d['AK'] = map.avg_price
d['P'] = map.calc_type
d['F'] = map.avg_price
// d['P'] = map.calc_type
// d['AM'] = map.credit_value
}
}
@@ -183,7 +183,7 @@ class AddPriceTest {
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202406 order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
@@ -272,23 +272,23 @@ class AddPriceTest {
ExcelUtils.multiThreadingModify(path,{d->
// if(!d['L']){
// if(d['J'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['J'])
def dist = CommonDatautils.findDistrictById(d['AJ'])
if(dist){
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
d['D'] = dist.status
d['E'] = dist.name_cat
d['F'] = dist.estate_type2
d['G'] = DateUtils.formatDateByPattern(dist.m_date,'yyyy-MM-dd')
d['H'] = dist.master_district_name
// d['A'] = dist.city
// d['B'] = dist.region
d['E'] = dist.name
// d['D'] = dist.status
// d['E'] = dist.name_cat
// d['F'] = dist.estate_type2
// d['G'] = DateUtils.formatDateByPattern(dist.m_date,'yyyy-MM-dd')
// d['H'] = dist.master_district_name
// d['N'] = dist.region
// def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
// if(dbo){
// d['D'] = dbo.avg_price
//// d['X'] = dbo.cal_type
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){
d['G'] = dbo.avg_price
// d['X'] = dbo.cal_type
// d['E'] = dbo.avg_price_2b
// }
}
}
// }
// }
@@ -27,8 +27,8 @@ class QiliangTest {
// task.run()
def data = [
'address': '宁德市福安市中兴西路16号',
'type': '2' //1:对公 0:对私
'address': '江阴市澄杨路77号',
'type': '1' //1:对公 0:对私
]
def s = task.getProxyResult(data);
print(s);
@@ -0,0 +1,265 @@
package com.fangjia.script.gj.xiamen
import org.apache.http.HttpHost
import org.apache.http.auth.AuthScope
import org.apache.http.auth.UsernamePasswordCredentials
import org.apache.http.client.CredentialsProvider
import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.client.BasicCredentialsProvider
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.elasticsearch.action.ActionListener
import org.elasticsearch.action.bulk.BackoffPolicy
import org.elasticsearch.action.bulk.BulkProcessor
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.search.ClearScrollRequest
import org.elasticsearch.action.search.SearchRequest
import org.elasticsearch.action.search.SearchResponse
import org.elasticsearch.action.search.SearchScrollRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestClient
import org.elasticsearch.client.RestClientBuilder
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.function.BiConsumer
/**
* 功能说明:Elasticsearch工具类
*/
class ElasticUtils89 {
static getEsDataSource(String host){
EsDataSource dataSource = new EsDataSource ();
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
Properties connectionProperties = new Properties();
// connectionProperties.setProperty("user","developer");
// connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
connectionProperties.setProperty("user","elastic");
connectionProperties.setProperty("password","elasticsearch");//elasticsearch
dataSource.setProperties(connectionProperties);
return dataSource;
}
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.89"){
EsDataSource dataSource = getEsDataSource(host)
Connection connection = dataSource.getConnection()
Statement statement = connection.createStatement()
ResultSet rs = statement.executeQuery(sql)
def columns = []
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
columns << rs.getMetaData().getColumnName(i)
}
rs.getMetaData().getColumnCount()
while(rs.next()){
def row = [:]
for (String col : columns){
row[col] = rs.getObject(col)
}
closure.call(row)
}
rs.close()
connection.close()
}
static RestHighLevelClient getClient(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
static RestHighLevelClient getClient89(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
/**
* 创建一个ES批处理器
* @param host
* @return
*/
static createBulkProcessor(String host, RestHighLevelClient client = null) {
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
@Override
void beforeBulk(long executionId, BulkRequest request) {
}
@Override
void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
}
@Override
void afterBulk(long executionId, BulkRequest request,
Throwable failure) {
}
};
if(!client){
client = getClient(host);
}
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
@Override
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
}
}, listener).setBulkActions(10000)
.setConcurrentRequests(20)
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
}
/**
* 关闭一个ES批处理器
* @param
* @return
*/
static closeBulkProcessor(BulkProcessor bulkProcessor) {
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
bulkProcessor.flush()
bulkProcessor.close()
}
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
RestHighLevelClient client = getClient(host)
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
def nThread = Runtime.getRuntime().availableProcessors() * 5
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
while (searchHits != null && searchHits.length > 0) {
for (int k=0; k < searchHits.size(); k++) {
final SearchHit hit = searchHits[k];
executorService.submit(new Runnable() {
@Override
void run() {
try{
if(closure.getMaximumNumberOfParameters() == 1){
def update = closure.call(hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}else if(closure.getMaximumNumberOfParameters() == 2){
def update = closure.call(hit.getId(), hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}
}catch (Exception e){
e.printStackTrace()
}
}
})
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
executorService.shutdown()
executorService.awaitTermination(1, TimeUnit.DAYS)
bulkProcessor.close()
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
client.close()
}
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
RestHighLevelClient client = getClient(host)
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
def results = []
while (searchHits != null && searchHits.length > 0) {
for (int k=0; k < searchHits.size(); k++) {
SearchHit searchHit = searchHits[k]
searchHit.getSourceAsMap().put("id", searchHit.getId());
results << searchHit.getSourceAsMap()
}
if(results.size()%100000==0){
println "."+ new Date()
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
client.close()
return results;
}
}
File diff suppressed because one or more lines are too long
@@ -129,7 +129,7 @@ class SeaweedDataSynch extends BaseBatch{
def run(){
fullCityMap = CommonDatautils.queryFullNameCity()
mcaDistMap = CommonDatautils.queryDistNameCity()
mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
// priceMap = findAllDistrictSellPrice()
def citys = CommonDatautils.findAllCity().reverse()
@@ -0,0 +1,69 @@
package com.fangjia.script.util
import javax.crypto.Cipher;
import javax.crypto.KeyGenerator;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import java.util.Base64;
public class DESUtil {
private static final String ALGORITHM = "DES";
// 生成密钥
public static SecretKey generateKey() throws Exception {
KeyGenerator keyGenerator = KeyGenerator.getInstance(ALGORITHM);
keyGenerator.init(56); // DES密钥长度为56位
return keyGenerator.generateKey();
}
// 将密钥转换为字符串
public static String keyToString(SecretKey secretKey) {
return Base64.getEncoder().encodeToString(secretKey.getEncoded());
}
// 将字符串转换为密钥
public static SecretKey stringToKey(String keyStr) {
byte[] decodedKey = Base64.getDecoder().decode(keyStr);
return new SecretKeySpec(decodedKey, 0, decodedKey.length, ALGORITHM);
}
// DES加密
public static String encrypt(String data, SecretKey secretKey) throws Exception {
Cipher cipher = Cipher.getInstance(ALGORITHM);
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
byte[] encryptedData = cipher.doFinal(data.getBytes());
return Base64.getEncoder().encodeToString(encryptedData);
}
// DES解密
public static String decrypt(String encryptedData, SecretKey secretKey) throws Exception {
Cipher cipher = Cipher.getInstance(ALGORITHM);
cipher.init(Cipher.DECRYPT_MODE, secretKey);
byte[] decodedData = Base64.getDecoder().decode(encryptedData);
byte[] decryptedData = cipher.doFinal(decodedData);
return new String(decryptedData);
}
public static void main(String[] args) {
try {
// 生成密钥
// SecretKey secretKey = generateKey();
// String keyStr = keyToString(secretKey);
// System.out.println("密钥: " + keyStr);
//
//// // 加密数据
// String data = "高娟";
// String encryptedData = encrypt(data, secretKey);
// System.out.println("加密后的数据: " + encryptedData);
// 解密数据
SecretKey newSecretKey = stringToKey("tnb12345");
String decryptedData = decrypt("5E55C890CED08EFB7E3A05A0BAECB7B100D71965E4AD0FE2", newSecretKey);
System.out.println("解密后的数据: " + decryptedData);
} catch (Exception e) {
e.printStackTrace();
}
}
}
@@ -12,7 +12,8 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //15021384995
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" //宝剑
//15021384995:dfc2245bac8a7b2d5b62e060e59d4eca
/**
* 冪帠僅蛌華硊<br>