This commit is contained in:
2024-10-22 15:32:18 +08:00
parent 1d8e9fa913
commit e1fbc1ee6a
13 changed files with 556 additions and 187 deletions
@@ -13,19 +13,24 @@ class AddThirdPriceSample {
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static final DBCollection NO_PRICE_DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com")
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
static main(args) {
def task = new AddThirdPriceSample()
task.run()
task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据
// task.addTrade() //添加成交
}
def run(){
for(def i=1;i<19;i++){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d->
if(d['A'] && d['C']){
for(def i=0;i<19;i++){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达参考价作为第三方入库.xlsx",{d->
if(d['A'] && d['B']){
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
if(dist){
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407)
def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202408)
// def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
def udata = new BasicDBObject("fccs_esf_price", d['B'].toString().toDouble().toInteger())
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
}else{
println d['A'] +" 没查到"
@@ -37,4 +42,67 @@ class AddThirdPriceSample {
}
}
def addTrade(){
def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"]
def rets = findTrades(citys)
// DEAL_PRICE_COLL.insert(rets)
}
def findTrades(citys){
def q = new BasicDBObject()
q.put("city", new BasicDBObject('$in', citys))
q.put("flag", 1)
def dbcursor = NO_PRICE_DEAL_PRICE_COLL.find(q)
def rets = []
def index = 0
for(def db : dbcursor){
index++
def udata = new BasicDBObject()
udata.put("_id",db._id)
udata.put("url",db.url)
udata.put("area",db.area)
udata.put("avg_price",db.avg_price.toString().toDouble().toInteger())
udata.put("build_structure",db.build_structure)
udata.put("build_type",db.build_type)
udata.put("build_year",db.build_year)
udata.put("city",db.city)
udata.put("direction", db.direction)
udata.put("district_name",db.district_name)
udata.put("elevator",db.elevator)
udata.put("fitment",db.fitment)
udata.put("floor_info",db.floor_info)
udata.put("hall",db.hall.toString().toInteger())
udata.put("height",db.height.toString().toInteger())
udata.put("housing_age",db.housing_age)
udata.put("lift",db.lift.toString().toInteger())
udata.put("lift_household",db.lift_household.toString().toInteger())
udata.put("property_ownership",db.property_ownership)
udata.put("region",db.region)
udata.put("room", db.room.toString().toInteger())
udata.put("room_structure",db.room_structure)
udata.put("source",db.source)
udata.put("total_price",db.total_price.toString().toInteger())
udata.put("trade_date",db.trade_date)
udata.put("trade_property",db.trade_property)
udata.put("house_using",db.house_using)
udata.put("heating_type",db.heating_type)
udata.put("community_id",db.community_id)
udata.put("create_date",db.create_time)
udata.put("m_date", new Date())
udata.put("address",db.address)
udata.put("match_city",db.match_city)
udata.put("match_region",db.match_region)
udata.put("match_name",db.match_name)
udata.put("match_id",db.match_id)
udata.put("price_flag", "国信达成交-推导")
DEAL_PRICE_COLL.save(udata)
println index
// rets << udata
}
return rets
}
}
@@ -631,36 +631,36 @@ def findAllData(sqlInstance236, citycode){
//},0)
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
if(d['A'] && d['B']){
def dbo = CommonDatautils.findDistrictById(d['A'])
if(dbo){
def alias = null
if(!dbo.alias){
alias = []
}else{
alias = dbo.alias
}
if(!alias.contains(d['B'])){
alias << d['B']
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", dbo._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{
println d['A']+" "+"别名已存在"
}
}else{
println dbo._id.toString() +" 没找到"
}
}
})
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
// if(d['A'] && d['B']){
// def dbo = CommonDatautils.findDistrictById(d['A'])
// if(dbo){
// def alias = null
// if(!dbo.alias){
// alias = []
// }else{
// alias = dbo.alias
// }
// if(!alias.contains(d['B'])){
// alias << d['B']
// }
// if(alias){
// def udata = new BasicDBObject()
// udata.put("alias", alias)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", dbo._id)
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }else{
// println d['A']+" "+"别名已存在"
// }
//
// }else{
// println dbo._id.toString() +" 没找到"
// }
// }
//})
//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
//for(def c : citys){
@@ -1331,34 +1331,34 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx"
//
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['J']){
//// def region = RegionData.formatRegion(d['A'], d['J'])
//// if(region){
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
//// if(dist){
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['L'] = dist.status-dist.name_cat
//// }
//// }
//// if(d['J'] instanceof String){
// def dist = CommonDatautils.findDistrictById(d['J'])
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['G']){
// def region = RegionData.formatRegion(d['A'], d['J'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
// if(dist){
// d['B'] = dist.region
// d['C'] = dist.name
// d['L'] = dist.status-dist.name_cat
// }
// }
if(d['G'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['G'])
if(dist){
// d['A'] = dist.city
// d['B'] = dist.region
// d['C'] = dist.name
// d['K'] = dist.estate_type2
// d['L'] = dist.status
// d['M'] = dist.name_cat
//
//// d['H'] = dist.name
// }
//// }
// }
//},0)
d['H'] = dist.name
}
}
}
},0)
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
@@ -47,7 +47,8 @@ class TradeStatLog {
// task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
task.exportTrade2()//导出未入库成交
// task.exportTrade2()//导出未入库成交
task.updateDealPriceRegion()//修改成交区域数据
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -597,7 +598,7 @@ class TradeStatLog {
// def endDate = "2024-07-01"
def ids = findAllDealIds();
println ids.size()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\廊坊.xlsx",{d->
if(ids.contains(d['A'])){
d['V'] = "供"
}
@@ -674,7 +675,7 @@ class TradeStatLog {
}
def findAllDealIds(){
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'"
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='廊坊'"
def ids = []
pssqlInstance.eachRow(sql,{r->
if(!ids.contains(r.gujia_bds_mdl_id)){
@@ -801,6 +802,20 @@ class TradeStatLog {
},0)
}
def updateDealPriceRegion(){
ExcelUtils.modify("D:\\数据\\建行\\建行成交\\成交待修改区域数据.xlsx",{d->
if(d['A'] && d['G']){
def region = RegionData.formatRegion(d['D'], d['G'])
if(region){
def udata = new BasicDBObject()
udata.put("region", region)
DEAL_PRICE_COLL.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}
}
},0)
}
def findTradeById(id){
def q = new BasicDBObject()
@@ -24,6 +24,13 @@ class AddressCut extends BaseBatch{
final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
final DBCollection thirdPartyPriceBkColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price_history_bk_new", "192.168.0.235", 27018)
final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018)
final DBCollection dataProcessBkColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process_history_bk", "192.168.0.235", 27018)
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
@@ -39,50 +46,66 @@ class AddressCut extends BaseBatch{
@Override
public void process(Object args) {
<<<<<<< HEAD
// sameAddr()
seaweedStreetUpdate()
def dates = [202408]
for(def date : dates){
// synch(date)
synchDataProcess(date)
println date + "同步完成! "
}
}
def seaweedStreetUpdate(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区地址拆分\\已核实-路2.xlsx",{d->
if(d['A']){
def dbo = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(dbo){
def udate = new BasicDBObject()
def street = null
def address = ""
if(d['E']){
street = d['E']
udate.put("street", street)
address = street
}
def streetNum = null
if(d['F']){
streetNum = d['F']
udate.put("street_number", streetNum)
address = address + streetNum
}
if(dbo.address!=address){
udate.put("address", address)
}
if(udate){
udate.put("m_date", new Date())
DBUtils.seaweed.update(new BasicDBObject('_id', dbo.get("_id")), new BasicDBObject('$set', udate))
}
}
def synch(date){
def q = new BasicDBObject()
q.put("s_date", date)
def dbcursor = thirdPartyPriceColl.find(q).batchSize(5000)
def index = 0
def rets = []
def num = 0
for(def d : dbcursor){
num++
index++
rets << d
if(index%10000==0){
thirdPartyPriceBkColl.insert(rets)
println(num)
rets.clear()
}
})
=======
// sameAddr() //相同地址小区
// roadMatch()
addAnjukeRoad()
}
if(rets.size()>0){
thirdPartyPriceBkColl.insert(rets)
rets.clear()
}
}
def synchDataProcess(date){
def q = new BasicDBObject()
q.put("s_date", date)
def dbcursor = dataProcessColl.find(q).batchSize(5000)
def index = 0
def rets = []
def num = 0
for(def d : dbcursor){
num++
index++
rets << d
if(index%10000==0){
dataProcessBkColl.insert(rets)
println(num)
rets.clear()
}
}
if(rets.size()>0){
dataProcessBkColl.insert(rets)
rets.clear()
}
}
def addAnjukeRoad(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d ->
def roads = findThirdPartyPrice(d['A'], d['B'], d['C'], "链家")
@@ -130,8 +153,6 @@ class AddressCut extends BaseBatch{
}
})
>>>>>>> c1a5ab8c6aa973a91e6be5547ec0925db3fa500f
}
def sameAddr() {
@@ -122,6 +122,8 @@ class CommonDatautils {
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
return house_dataStoreConnection
@@ -820,6 +822,7 @@ class CommonDatautils {
DBObject q = new BasicDBObject();
q.put("status",0)
// q.put("status",new BasicDBObject('$ne', 0))
q.put("cat", cat);
q.put("city", city)
@@ -829,7 +832,7 @@ class CommonDatautils {
keys.put(field, 1);
}
}
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
def rets = []
for(def d : dbsour){
rets << d
@@ -959,13 +962,7 @@ class CommonDatautils {
keys.put(field, 1);
}
}
def cursor = getDataStoreConnection().find(q, keys).batchSize(2000)
if(cursor){
for(def obj : cursor){
return obj;
}
}
return null;
return SEAWEED_COLL.findOne(q, keys)
}
@@ -1132,6 +1129,10 @@ class CommonDatautils {
q.put("city", city);
q.put("cat", cat);
q.put("name", name);
if(region){
q.put("region", region);
}
q.put("status",new BasicDBObject('$ne', 0))
DBObject keys = new BasicDBObject();
if(fields){
@@ -1680,7 +1681,7 @@ class CommonDatautils {
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("category","region");
queryCondition.put("show",1);
// queryCondition.put("show",1);
def dbCursor = REGION_BLOCK_COLL.find(queryCondition)
def regions = [];
for(DBObject obj : dbCursor){
@@ -2385,6 +2386,9 @@ class CommonDatautils {
q.put("city", city);
if(region){
q.put("region", region);
q.put("cat", "区域")
}else{
q.put("cat", "城市")
}
q.put('weekend', weekend);
def dbCursor = REGION_STAT_COLL.find(q).sort(new BasicDBObject().append("weekend", -1)).limit(1);
@@ -3735,9 +3739,9 @@ class CommonDatautils {
if(!region){
if(city){
q.put("cat", "city")
}else{
q.put("cat", "region")
}
}else{
q.put("cat", "region")
}
}
@@ -4095,6 +4099,21 @@ class CommonDatautils {
return retMap;
}
static def findHouseCount(def city, def region, def name){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
.must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("district_name", name)).must(QueryBuilders.termQuery("cat", "sell"))
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(5);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
return houses.size();
}
/**
* 是否是别墅
* @return
@@ -5060,4 +5079,38 @@ class CommonDatautils {
}
static def findStatIndexBySort(type, city, region, sort){
def q = new BasicDBObject()
q.put("cat", "district")
q.put("type", type)
if(region){
q.put("district", region)
}
if(city){
q.put("city", city)
}
def cur = STATS_INDEX_COLL.find(q).sort(new BasicDBObject("yearmonth", sort)).limit(1)
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def obj = 0;
while(cur.hasNext()) {
obj = cur.next()
}
return obj;
}
static def findStatIndexByMonth(type, city, region, month){
def q = new BasicDBObject()
q.put("cat", "district")
q.put("type", type)
if(region){
q.put("district", region)
}
if(city){
q.put("city", city)
}
q.put("yearmonth", month)
return STATS_INDEX_COLL.findOne(q)
}
}
@@ -2,8 +2,10 @@ package com.fangjia.script.gj
import com.fangjia.core.BlockData
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
@@ -531,6 +533,28 @@ class FiledStrand {
return null
}
static def processLngAndLat(val) {
if (val) {
def map = [:]
def lng = DataHandleUtils.getDoubleRound(6, val.x)
def lat = DataHandleUtils.getDoubleRound(6, val.y)
Long gd_lat = lat * 10000000000L
Long gd_lng = lng * 10000000000L
map.lat = gd_lat
map.lng = gd_lng
def baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
Long bd_lng = baidu.x * 10000000000L
Long bd_lat = baidu.y * 10000000000L
map.lat2 = bd_lat
map.lng2 = bd_lng
return map
}
return null
}
static def isRoad(def s){
def pattern = Pattern.compile("[路|街|大道|巷]");
def matcher = pattern.matcher(s)
@@ -2,33 +2,97 @@ package com.fangjia.script.gj
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d->
if(d['A'] && !d['E']){
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district")
// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district")
println d['A'] +" "+d['C']
// def total = findDatas(d['A'], d['B'], d['C'])
// d['D'] = total
def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·",""))
d['E'] = c
}
})
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
//ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d->
// if(d['A'] && !d['E']){
//// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district")
//// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district")
// println d['A'] +" "+d['C']
//// def total = findDatas(d['A'], d['B'], d['C'])
//// d['D'] = total
// def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·",""))
// d['E'] = c
// }
//})
def URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection seaweed_coll = MongoUtils.getCollectionByName2("fangjia", "seaweed", URL)
def q = new BasicDBObject()
q.put("city", "上海市")
q.put("category", "district")
q.put("status", 0)
def ret = []
def list = seaweed_coll.find(q).batchSize(5000).toList()
println "总计:"+list.size()
//for(def l : list){
// def obj = getAmapLocation(l.city, l.region, l.name)
// if(obj){
// ret << [l.city, l.region, l.name, obj.cityname, obj.adname, obj.name,obj.type]
// }
//}
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(list.size())
def total = list.size()
for(def i=0;i<list.size();i++){
final def l = list[i]
def dist = CommonDatautils.findDistrict(l.fangjia_city, l.fangjia_region, l.name, "district")
fixedThreadPool.execute(new Runnable() {
void run() {
try{
addCount.addAndGet(1)
def alias = []
alias << dist.name
if(l.alias){
alias.addAll(l.alias)
}
for(def alia : alias){
def obj = getAmapLocation(l.city, l.region, alia)
if(obj){
ret << [l.city, l.region, l.name, obj.cityname, obj.adname, obj.name,obj.type]
break
}
}
println addCount.get()+"/"+total+" "+ ret.size()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
ExcelUtils.write("D:\\数据\\数据建设\\小区\\高德地图有围栏数据.xlsx", ret, "高德")
def getAmapLocation(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
def total = 0
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name.contains(name) && d.type.contains("住宅")){
total ++
if(d.name.contains(name) && d.type.contains("住宅") && d.shape){
obj = d
}
})
return total;
return obj;
}
@@ -46,7 +46,7 @@ class DevProcess {
def task = new DevProcess();
// task.run()
// task.parseCityAndDist()
task.devSplitRoad()
// task.devSplitRoad()
// task.devNameSplitRoad()
// task.addFlag()
// task.matchDev()
@@ -63,7 +63,7 @@ class DevProcess {
// task.createDevMapping()
// task.deleteAddr()
// task.deleteAddr2()
// task.synchDevTo100()
task.synchDevTo100()
// task.addNewDev2()
// task.addAlias()
@@ -20,6 +20,10 @@ class DevUpdateTestBatch {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_43_CONN = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
@@ -30,8 +34,8 @@ class DevUpdateTestBatch {
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
task.addDevAddress(path)
task.addDev(path)
// task.addDevAddress(path)
//
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
@@ -42,7 +46,7 @@ class DevUpdateTestBatch {
def addDev(path){
ExcelUtils.modify(path,{ d->
if(!d['I'] && d['A']){
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
def obj = development_zone_43_CONN.findOne(new BasicDBObject("name", d['A']))
if(!obj){
def o = new BasicDBObject()
o.put("province", d['B'])
@@ -28,7 +28,7 @@ class GXDdistrictMatch {
static main(args) {
def task = new GXDdistrictMatch();
// task.run()
task.addOldMatch()
task.addOldMatch2()
// task.runDb()
// task.updateDistStatus()//恢复name_cat不正常数据
@@ -68,6 +68,32 @@ class GXDdistrictMatch {
}
def addOldMatch2(){
//老匹配
def fileName = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx"//11城重新匹配.xlsx
ExcelUtils.multiThreadingModify(fileName, { d ->
println d['C']
if(!d['AJ']){
def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D'])
if (jsonObject != null && !jsonObject.containsKey("result")) {
d['AJ'] = jsonObject.get("id")
d['AK'] = jsonObject.get("city")
d['AL'] = jsonObject.get("district")
d['AM'] = jsonObject.get("name")
d['AN'] = jsonObject.get("address")
def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
if(dbo){
d['AO'] = dbo.status
d['AP'] = dbo.name_cat
d['AQ'] = dbo.estate_type2
}
}
}
});
}
def runDb(){
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
ExcelUtils.modify(fileName, { d ->
@@ -35,7 +35,7 @@ class ThirdPriceMappingBatch extends BaseBatch{
final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018)
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
static main(args) {
runBatch(new ThirdPriceMappingBatch(), args);
@@ -44,9 +44,9 @@ class ThirdPriceMappingBatch extends BaseBatch{
@Override
public void process(Object args) {
// //将新增数据同步过来
// addNemCommunity()
addNemCommunity()
// //接口进行匹配
match()
// match()
// addOldMatchResult()
}
@@ -151,8 +151,9 @@ class ThirdPriceMappingBatch extends BaseBatch{
sd.each {k,v->
udata.put(k,v)
}
udata.removeField("_id")
udata.put("m_date", new Date())
thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata))
thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata),false, false)
}else{
sd.put("m_date", new Date())
thirdSeaweedMappingColl.save(sd)
@@ -1,6 +1,7 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
@@ -900,41 +901,92 @@ class TownMiddleProcess {
}
def updateTownPolineBatch(){
def i = 0
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx",{ row->
if(row['R'] && row['R']=="新增围栏"){
def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D'])
println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q)
def d = TOWNPOLYLINE_COLL.findOne(q)
if(d){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble())
def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble())
pointList << [lon, lat]
i++
if(i>=134){
def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D'])
println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q)
def d = TOWNPOLYLINE_COLL.findOne(q)
if(d){
def resultMap = [:]
def bd_resultMap = [:]
resultMap.type = "polygon"
bd_resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
def bd_rets = []
println row['N'] +" 共有:"+arrs.size()+" 个围栏"
def index = 1
for(def s : arrs){
def points = s.split(";")
def pointList = []
def bd_pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble())
def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble())
pointList << [lon, lat]
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lon, lat, 3)
def bd_lng = baidu.x
def bd_lat = baidu.y
bd_pointList << [bd_lng, bd_lat]
}
if(index<=1){
resultMap.coordinates = pointList
bd_resultMap.coordinates = bd_pointList
def udata = new BasicDBObject()
udata.put("shape", resultMap)
udata.put("operator", "高娟")
udata.put("m_date", new Date())
udata.put("bd_shape",bd_resultMap)
def updateQuery = new BasicDBObject("name",row['N'])
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}else{
def dbo = DEVELOPMENT_ZONE_43_COLL.findOne(new BasicDBObject("name",row['N']))
def obj = new BasicDBObject()
dbo.each {k,v->
if(k!='_id'){
obj.put(k,v)
}
}
resultMap.coordinates = pointList
bd_resultMap.coordinates = bd_pointList
obj.put("shape", resultMap)
obj.put("operator", "高娟")
obj.put("m_date", new Date())
obj.put("bd_shape",bd_resultMap)
obj.put("create_time", new Date())
obj.put("block", dbo.name+"-地块"+index)
DEVELOPMENT_ZONE_43_COLL.save(obj)
}
// rets << pointList
// bd_rets << bd_pointList
index++
}
rets << pointList
// if(rets.size()>1) {
// resultMap.type = "MultiPolygon"
// bd_resultMap.type = "MultiPolygon"
// }
// println d.city+" "+d.region+" "+d.town+" "+rets.size()
row['S'] = "已完成"
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+d.town+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def updateQuery = new BasicDBObject("name",row['N'])
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
row['S'] = "ÒÑÍê³É"
}
}
})
}
@@ -53,7 +53,7 @@ class ImportIndexStats extends BaseBatch{
public static final def dingjiYear = 2020;
public static def dingjiYear = 2020;
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final EsDataSource dataSource = new EsDataSource()
@@ -76,22 +76,23 @@ class ImportIndexStats extends BaseBatch{
// synch()
// synchTjjIndex()//同步统计局抓取过来的指数
// Calendar calendar=Calendar.getInstance();
// def month= calendar.get(Calendar.MONTH);
// def year= calendar.get(Calendar.YEAR);
// def date = createDateString(year, month, 0, "");
// calculatePriceIndex(year, month, date);
Calendar calendar=Calendar.getInstance();
def month= calendar.get(Calendar.MONTH);
def year= calendar.get(Calendar.YEAR);
def date = createDateString(year, month, 0, "");
calculatePriceIndex(year, month, date);
// addLostIndex()
addHistoryIndex()
// addHistoryIndex()
}
def addLostIndex(){
def citys = CommonDatautils.findAllCity();
def index = 0
for(def city : citys){
index++
if(cityList.contains(city)){
println city +" Êý¾ÝÀ´Ô´ÓÚͳ¼Æ¾Ö"
println "[${index}] "+ city +" 数据来源于统计局"
continue;
}
def start = findStartOrEndIndex(city, 1)
@@ -117,21 +118,48 @@ class ImportIndexStats extends BaseBatch{
if(nextObj && nextObj.chain){
indexs << nextObj.chain
}
udata.put("chain",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
udata.put("remark","ÍÆµ¼")
if(indexs){
udata.put("chain",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
udata.put("remark","推导")
}
}
if(!dbo.yeartoyear){
def indexs = []
if(lastMonthObj && lastMonthObj.yeartoyear){
indexs << lastMonthObj.yeartoyear
}
if(nextObj && nextObj.yeartoyear){
indexs << nextObj.yeartoyear
}
if(indexs){
udata.put("yeartoyear",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
udata.put("remark","推导")
}
}
if(!dbo.fixedbase){
def indexs = []
if(lastMonthObj && lastMonthObj.fixedbase){
indexs << lastMonthObj.fixedbase
}
if(nextObj && nextObj.fixedbase){
indexs << nextObj.fixedbase
}
if(indexs){
udata.put("fixedbase",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
udata.put("remark","推导")
}
}
if(udata){
println city +" 补充"
STATS_INDEX_COLL.update(new BasicDBObject("_id", dbo._id), new BasicDBObject('$set', udata),false, true);
}
// if(!dbo.yeartoyear){
//
// }
// if(!dbo.fixedbase){
//
// }
}
}
}
println "[${index}] "+ city +" 已处理完"
}
}
@@ -154,7 +182,7 @@ class ImportIndexStats extends BaseBatch{
}
def addHistoryIndex(){
def s = "202101"
def s = "202001"
def startDate = null;
def stop = false;
def runDates = []
@@ -171,7 +199,7 @@ class ImportIndexStats extends BaseBatch{
runDates << map
s = DateUtils.addMonth(date, 1, "yyyyMM")
if(year==2024 && month==9){
if(year==2024 && month==8){
stop = true;
}
}
@@ -182,6 +210,7 @@ class ImportIndexStats extends BaseBatch{
//计算统计局没有发布的城市的价格指数
def calculatePriceIndex(def year, def month, def date){
// def citys = ["五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]//
def citys = CommonDatautils.findAllCity();
// def date = createDateString(year, month, 0, "");
def haveIndexCitys = findPriceIndexCitys(date.toInteger());
@@ -193,7 +222,7 @@ class ImportIndexStats extends BaseBatch{
def newCitys = []
for(def i=0;i<citys.size();i++) {
def city = citys[i]
if (!haveIndexCitys.contains(city)) {
if (!haveIndexCitys.contains(city) && !cityList.contains(city)) {
newCitys << city
}
}
@@ -204,6 +233,15 @@ class ImportIndexStats extends BaseBatch{
if(id){//已计算
continue;
}
if(date.toInteger()>=201601){
dingjiYear = 2015
}else if(date.toInteger()>=202101){
dingjiYear = 2020
}else if(date.toInteger()>=202601){
dingjiYear = 2025
}else if(date.toInteger()>=203101){
dingjiYear = 2030
}
fixedThreadPool.execute(new Runnable() {
void run() {
try{
@@ -220,7 +258,7 @@ class ImportIndexStats extends BaseBatch{
}
});
}
latch.await(20, TimeUnit.SECONDS);
latch.await(1, TimeUnit.MINUTES);
}
def calculating(city, year, month, date){
@@ -647,6 +685,7 @@ class ImportIndexStats extends BaseBatch{
DBObject q = new BasicDBObject();
q.put("yearmonth", date);
q.put("cat", "district")
q.put("type", "城市")
DBObject keys = new BasicDBObject();
keys.put("city", 1)
@@ -666,6 +705,7 @@ class ImportIndexStats extends BaseBatch{
q.put("yearmonth", date);
q.put("city", city);
q.put("cat", "district");
q.put("type", "城市")
DBObject keys = new BasicDBObject();
keys.put("city", 1)
@@ -684,6 +724,7 @@ class ImportIndexStats extends BaseBatch{
dbo.put("source",result.source);
dbo.put("remark",result.remark);
dbo.put("cat",result.cat);
dbo.put("type",result.type);
dbo.put("c_date",new Date());
if(result.markup){
dbo.put("chain", result.markup.toDouble());