This commit is contained in:
2024-10-22 15:32:18 +08:00
parent 1d8e9fa913
commit e1fbc1ee6a
13 changed files with 556 additions and 187 deletions
@@ -13,19 +13,24 @@ class AddThirdPriceSample {
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static final DBCollection NO_PRICE_DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com")
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
static main(args) { static main(args) {
def task = new AddThirdPriceSample() def task = new AddThirdPriceSample()
task.run() task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据
// task.addTrade() //添加成交
} }
def run(){ def run(){
for(def i=1;i<19;i++){ for(def i=0;i<19;i++){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d-> ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达参考价作为第三方入库.xlsx",{d->
if(d['A'] && d['C']){ if(d['A'] && d['B']){
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim()) def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
if(dist){ if(dist){
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407) def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202408)
def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger()) // def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
def udata = new BasicDBObject("fccs_esf_price", d['B'].toString().toDouble().toInteger())
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false) AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
}else{ }else{
println d['A'] +" 没查到" println d['A'] +" 没查到"
@@ -37,4 +42,67 @@ class AddThirdPriceSample {
} }
} }
def addTrade(){
def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"]
def rets = findTrades(citys)
// DEAL_PRICE_COLL.insert(rets)
}
def findTrades(citys){
def q = new BasicDBObject()
q.put("city", new BasicDBObject('$in', citys))
q.put("flag", 1)
def dbcursor = NO_PRICE_DEAL_PRICE_COLL.find(q)
def rets = []
def index = 0
for(def db : dbcursor){
index++
def udata = new BasicDBObject()
udata.put("_id",db._id)
udata.put("url",db.url)
udata.put("area",db.area)
udata.put("avg_price",db.avg_price.toString().toDouble().toInteger())
udata.put("build_structure",db.build_structure)
udata.put("build_type",db.build_type)
udata.put("build_year",db.build_year)
udata.put("city",db.city)
udata.put("direction", db.direction)
udata.put("district_name",db.district_name)
udata.put("elevator",db.elevator)
udata.put("fitment",db.fitment)
udata.put("floor_info",db.floor_info)
udata.put("hall",db.hall.toString().toInteger())
udata.put("height",db.height.toString().toInteger())
udata.put("housing_age",db.housing_age)
udata.put("lift",db.lift.toString().toInteger())
udata.put("lift_household",db.lift_household.toString().toInteger())
udata.put("property_ownership",db.property_ownership)
udata.put("region",db.region)
udata.put("room", db.room.toString().toInteger())
udata.put("room_structure",db.room_structure)
udata.put("source",db.source)
udata.put("total_price",db.total_price.toString().toInteger())
udata.put("trade_date",db.trade_date)
udata.put("trade_property",db.trade_property)
udata.put("house_using",db.house_using)
udata.put("heating_type",db.heating_type)
udata.put("community_id",db.community_id)
udata.put("create_date",db.create_time)
udata.put("m_date", new Date())
udata.put("address",db.address)
udata.put("match_city",db.match_city)
udata.put("match_region",db.match_region)
udata.put("match_name",db.match_name)
udata.put("match_id",db.match_id)
udata.put("price_flag", "国信达成交-推导")
DEAL_PRICE_COLL.save(udata)
println index
// rets << udata
}
return rets
}
} }
@@ -631,36 +631,36 @@ def findAllData(sqlInstance236, citycode){
//},0) //},0)
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d-> //ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
if(d['A'] && d['B']){ // if(d['A'] && d['B']){
def dbo = CommonDatautils.findDistrictById(d['A']) // def dbo = CommonDatautils.findDistrictById(d['A'])
if(dbo){ // if(dbo){
def alias = null // def alias = null
if(!dbo.alias){ // if(!dbo.alias){
alias = [] // alias = []
}else{ // }else{
alias = dbo.alias // alias = dbo.alias
} // }
if(!alias.contains(d['B'])){ // if(!alias.contains(d['B'])){
alias << d['B'] // alias << d['B']
} // }
if(alias){ // if(alias){
def udata = new BasicDBObject() // def udata = new BasicDBObject()
udata.put("alias", alias) // udata.put("alias", alias)
udata.put("m_date",new Date()) // udata.put("m_date",new Date())
//
def q = new BasicDBObject() // def q = new BasicDBObject()
q.put("_id", dbo._id) // q.put("_id", dbo._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) // DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{ // }else{
println d['A']+" "+"别名已存在" // println d['A']+" "+"别名已存在"
} // }
//
}else{ // }else{
println dbo._id.toString() +" 没找到" // println dbo._id.toString() +" 没找到"
} // }
} // }
}) //})
//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] //def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
//for(def c : citys){ //for(def c : citys){
@@ -1331,34 +1331,34 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx" def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx"
// //
//ExcelUtils.multiThreadingModify(fileName, {d-> ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['J']){ if(d['G']){
//// def region = RegionData.formatRegion(d['A'], d['J']) // def region = RegionData.formatRegion(d['A'], d['J'])
//// if(region){ // if(region){
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") // def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
//// if(dist){
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['L'] = dist.status-dist.name_cat
//// }
//// }
//// if(d['J'] instanceof String){
// def dist = CommonDatautils.findDistrictById(d['J'])
// if(dist){ // if(dist){
// d['B'] = dist.region
// d['C'] = dist.name
// d['L'] = dist.status-dist.name_cat
// }
// }
if(d['G'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['G'])
if(dist){
// d['A'] = dist.city // d['A'] = dist.city
// d['B'] = dist.region // d['B'] = dist.region
// d['C'] = dist.name // d['C'] = dist.name
// d['K'] = dist.estate_type2 // d['K'] = dist.estate_type2
// d['L'] = dist.status // d['L'] = dist.status
// d['M'] = dist.name_cat // d['M'] = dist.name_cat
//
//// d['H'] = dist.name d['H'] = dist.name
// } }
//// } }
// } }
//},0) },0)
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx" //def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
@@ -47,7 +47,8 @@ class TradeStatLog {
// task.run()//统计最近3个月抓取的成交数据量 // task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交 // task.exportTrade()//导出未入库成交
task.exportTrade2()//导出未入库成交 // task.exportTrade2()//导出未入库成交
task.updateDealPriceRegion()//修改成交区域数据
// task.thirdPartyStat() // task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR) // println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat() // task.averageStat()
@@ -597,7 +598,7 @@ class TradeStatLog {
// def endDate = "2024-07-01" // def endDate = "2024-07-01"
def ids = findAllDealIds(); def ids = findAllDealIds();
println ids.size() println ids.size()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d-> ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\廊坊.xlsx",{d->
if(ids.contains(d['A'])){ if(ids.contains(d['A'])){
d['V'] = "供" d['V'] = "供"
} }
@@ -674,7 +675,7 @@ class TradeStatLog {
} }
def findAllDealIds(){ def findAllDealIds(){
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'" def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='廊坊'"
def ids = [] def ids = []
pssqlInstance.eachRow(sql,{r-> pssqlInstance.eachRow(sql,{r->
if(!ids.contains(r.gujia_bds_mdl_id)){ if(!ids.contains(r.gujia_bds_mdl_id)){
@@ -801,6 +802,20 @@ class TradeStatLog {
},0) },0)
} }
def updateDealPriceRegion(){
ExcelUtils.modify("D:\\数据\\建行\\建行成交\\成交待修改区域数据.xlsx",{d->
if(d['A'] && d['G']){
def region = RegionData.formatRegion(d['D'], d['G'])
if(region){
def udata = new BasicDBObject()
udata.put("region", region)
DEAL_PRICE_COLL.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}
}
},0)
}
def findTradeById(id){ def findTradeById(id){
def q = new BasicDBObject() def q = new BasicDBObject()
@@ -24,6 +24,13 @@ class AddressCut extends BaseBatch{
final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
final DBCollection thirdPartyPriceBkColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price_history_bk_new", "192.168.0.235", 27018)
final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018)
final DBCollection dataProcessBkColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process_history_bk", "192.168.0.235", 27018)
public static EsDataSource dataSource = new EsDataSource() public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static { static {
@@ -39,50 +46,66 @@ class AddressCut extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
<<<<<<< HEAD
// sameAddr() // sameAddr()
seaweedStreetUpdate() def dates = [202408]
for(def date : dates){
// synch(date)
synchDataProcess(date)
println date + "同步完成! "
}
} }
def seaweedStreetUpdate(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区地址拆分\\已核实-路2.xlsx",{d->
if(d['A']){
def dbo = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(dbo){
def udate = new BasicDBObject()
def street = null
def address = ""
if(d['E']){
street = d['E']
udate.put("street", street)
address = street
}
def streetNum = null
if(d['F']){
streetNum = d['F']
udate.put("street_number", streetNum)
address = address + streetNum
}
if(dbo.address!=address){
udate.put("address", address)
}
if(udate){
udate.put("m_date", new Date())
DBUtils.seaweed.update(new BasicDBObject('_id', dbo.get("_id")), new BasicDBObject('$set', udate))
}
} def synch(date){
def q = new BasicDBObject()
q.put("s_date", date)
def dbcursor = thirdPartyPriceColl.find(q).batchSize(5000)
def index = 0
def rets = []
def num = 0
for(def d : dbcursor){
num++
index++
rets << d
if(index%10000==0){
thirdPartyPriceBkColl.insert(rets)
println(num)
rets.clear()
} }
}
}) if(rets.size()>0){
======= thirdPartyPriceBkColl.insert(rets)
// sameAddr() //相同地址小区 rets.clear()
// roadMatch() }
addAnjukeRoad()
} }
def synchDataProcess(date){
def q = new BasicDBObject()
q.put("s_date", date)
def dbcursor = dataProcessColl.find(q).batchSize(5000)
def index = 0
def rets = []
def num = 0
for(def d : dbcursor){
num++
index++
rets << d
if(index%10000==0){
dataProcessBkColl.insert(rets)
println(num)
rets.clear()
}
}
if(rets.size()>0){
dataProcessBkColl.insert(rets)
rets.clear()
}
}
def addAnjukeRoad(){ def addAnjukeRoad(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d -> ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d ->
def roads = findThirdPartyPrice(d['A'], d['B'], d['C'], "链家") def roads = findThirdPartyPrice(d['A'], d['B'], d['C'], "链家")
@@ -130,8 +153,6 @@ class AddressCut extends BaseBatch{
} }
}) })
>>>>>>> c1a5ab8c6aa973a91e6be5547ec0925db3fa500f
} }
def sameAddr() { def sameAddr() {
@@ -122,6 +122,8 @@ class CommonDatautils {
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com") public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
public static DSCollection getHouseDataStoreConnection(){ public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) { if(house_dataStoreConnection != null) {
return house_dataStoreConnection return house_dataStoreConnection
@@ -820,6 +822,7 @@ class CommonDatautils {
DBObject q = new BasicDBObject(); DBObject q = new BasicDBObject();
q.put("status",0) q.put("status",0)
// q.put("status",new BasicDBObject('$ne', 0))
q.put("cat", cat); q.put("cat", cat);
q.put("city", city) q.put("city", city)
@@ -829,7 +832,7 @@ class CommonDatautils {
keys.put(field, 1); keys.put(field, 1);
} }
} }
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000) def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
def rets = [] def rets = []
for(def d : dbsour){ for(def d : dbsour){
rets << d rets << d
@@ -959,13 +962,7 @@ class CommonDatautils {
keys.put(field, 1); keys.put(field, 1);
} }
} }
def cursor = getDataStoreConnection().find(q, keys).batchSize(2000) return SEAWEED_COLL.findOne(q, keys)
if(cursor){
for(def obj : cursor){
return obj;
}
}
return null;
} }
@@ -1132,6 +1129,10 @@ class CommonDatautils {
q.put("city", city); q.put("city", city);
q.put("cat", cat); q.put("cat", cat);
q.put("name", name); q.put("name", name);
if(region){
q.put("region", region);
}
q.put("status",new BasicDBObject('$ne', 0))
DBObject keys = new BasicDBObject(); DBObject keys = new BasicDBObject();
if(fields){ if(fields){
@@ -1680,7 +1681,7 @@ class CommonDatautils {
DBObject queryCondition = new BasicDBObject() DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city); queryCondition.put("city", city);
queryCondition.put("category","region"); queryCondition.put("category","region");
queryCondition.put("show",1); // queryCondition.put("show",1);
def dbCursor = REGION_BLOCK_COLL.find(queryCondition) def dbCursor = REGION_BLOCK_COLL.find(queryCondition)
def regions = []; def regions = [];
for(DBObject obj : dbCursor){ for(DBObject obj : dbCursor){
@@ -2385,6 +2386,9 @@ class CommonDatautils {
q.put("city", city); q.put("city", city);
if(region){ if(region){
q.put("region", region); q.put("region", region);
q.put("cat", "区域")
}else{
q.put("cat", "城市")
} }
q.put('weekend', weekend); q.put('weekend', weekend);
def dbCursor = REGION_STAT_COLL.find(q).sort(new BasicDBObject().append("weekend", -1)).limit(1); def dbCursor = REGION_STAT_COLL.find(q).sort(new BasicDBObject().append("weekend", -1)).limit(1);
@@ -3735,9 +3739,9 @@ class CommonDatautils {
if(!region){ if(!region){
if(city){ if(city){
q.put("cat", "city") q.put("cat", "city")
}else{
q.put("cat", "region")
} }
}else{
q.put("cat", "region")
} }
} }
@@ -4095,6 +4099,21 @@ class CommonDatautils {
return retMap; return retMap;
} }
static def findHouseCount(def city, def region, def name){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
.must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("district_name", name)).must(QueryBuilders.termQuery("cat", "sell"))
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(5);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
return houses.size();
}
/** /**
* 是否是别墅 * 是否是别墅
* @return * @return
@@ -5060,4 +5079,38 @@ class CommonDatautils {
} }
static def findStatIndexBySort(type, city, region, sort){
def q = new BasicDBObject()
q.put("cat", "district")
q.put("type", type)
if(region){
q.put("district", region)
}
if(city){
q.put("city", city)
}
def cur = STATS_INDEX_COLL.find(q).sort(new BasicDBObject("yearmonth", sort)).limit(1)
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def obj = 0;
while(cur.hasNext()) {
obj = cur.next()
}
return obj;
}
static def findStatIndexByMonth(type, city, region, month){
def q = new BasicDBObject()
q.put("cat", "district")
q.put("type", type)
if(region){
q.put("district", region)
}
if(city){
q.put("city", city)
}
q.put("yearmonth", month)
return STATS_INDEX_COLL.findOne(q)
}
} }
@@ -2,8 +2,10 @@ package com.fangjia.script.gj
import com.fangjia.core.BlockData import com.fangjia.core.BlockData
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DateUtil import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
@@ -531,6 +533,28 @@ class FiledStrand {
return null return null
} }
static def processLngAndLat(val) {
if (val) {
def map = [:]
def lng = DataHandleUtils.getDoubleRound(6, val.x)
def lat = DataHandleUtils.getDoubleRound(6, val.y)
Long gd_lat = lat * 10000000000L
Long gd_lng = lng * 10000000000L
map.lat = gd_lat
map.lng = gd_lng
def baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
Long bd_lng = baidu.x * 10000000000L
Long bd_lat = baidu.y * 10000000000L
map.lat2 = bd_lat
map.lng2 = bd_lng
return map
}
return null
}
static def isRoad(def s){ static def isRoad(def s){
def pattern = Pattern.compile("[路|街|大道|巷]"); def pattern = Pattern.compile("[路|街|大道|巷]");
def matcher = pattern.matcher(s) def matcher = pattern.matcher(s)
@@ -2,33 +2,97 @@ package com.fangjia.script.gj
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d-> import java.util.concurrent.CountDownLatch
if(d['A'] && !d['E']){ import java.util.concurrent.ExecutorService
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district") import java.util.concurrent.Executors
// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district") import java.util.concurrent.atomic.AtomicLong
println d['A'] +" "+d['C']
// def total = findDatas(d['A'], d['B'], d['C']) //ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d->
// d['D'] = total // if(d['A'] && !d['E']){
def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·","")) //// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district")
d['E'] = c //// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district")
} // println d['A'] +" "+d['C']
}) //// def total = findDatas(d['A'], d['B'], d['C'])
//// d['D'] = total
// def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·",""))
// d['E'] = c
// }
//})
def URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection seaweed_coll = MongoUtils.getCollectionByName2("fangjia", "seaweed", URL)
def q = new BasicDBObject()
q.put("city", "上海市")
q.put("category", "district")
q.put("status", 0)
def ret = []
def list = seaweed_coll.find(q).batchSize(5000).toList()
println "总计:"+list.size()
//for(def l : list){
// def obj = getAmapLocation(l.city, l.region, l.name)
// if(obj){
// ret << [l.city, l.region, l.name, obj.cityname, obj.adname, obj.name,obj.type]
// }
//}
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(list.size())
def total = list.size()
for(def i=0;i<list.size();i++){
final def l = list[i]
def dist = CommonDatautils.findDistrict(l.fangjia_city, l.fangjia_region, l.name, "district")
fixedThreadPool.execute(new Runnable() {
void run() {
try{
addCount.addAndGet(1)
def alias = []
alias << dist.name
if(l.alias){
alias.addAll(l.alias)
}
for(def alia : alias){
def obj = getAmapLocation(l.city, l.region, alia)
if(obj){
ret << [l.city, l.region, l.name, obj.cityname, obj.adname, obj.name,obj.type]
break
}
}
println addCount.get()+"/"+total+" "+ ret.size()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
ExcelUtils.write("D:\\数据\\数据建设\\小区\\高德地图有围栏数据.xlsx", ret, "高德")
def getAmapLocation(city, region, name){ def getAmapLocation(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') " String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def total = 0 def obj = null
ElasticUtils.executeSqlQuery(sql,{ d-> ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name.contains(name) && d.type.contains("住宅")){ if(d.name.contains(name) && d.type.contains("住宅") && d.shape){
total ++ obj = d
} }
}) })
return total; return obj;
} }
@@ -46,7 +46,7 @@ class DevProcess {
def task = new DevProcess(); def task = new DevProcess();
// task.run() // task.run()
// task.parseCityAndDist() // task.parseCityAndDist()
task.devSplitRoad() // task.devSplitRoad()
// task.devNameSplitRoad() // task.devNameSplitRoad()
// task.addFlag() // task.addFlag()
// task.matchDev() // task.matchDev()
@@ -63,7 +63,7 @@ class DevProcess {
// task.createDevMapping() // task.createDevMapping()
// task.deleteAddr() // task.deleteAddr()
// task.deleteAddr2() // task.deleteAddr2()
// task.synchDevTo100() task.synchDevTo100()
// task.addNewDev2() // task.addNewDev2()
// task.addAlias() // task.addAlias()
@@ -20,6 +20,10 @@ class DevUpdateTestBatch {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_43_CONN = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") DBCollection development_zone_43_CONN = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
@@ -30,8 +34,8 @@ class DevUpdateTestBatch {
static main(args) { static main(args) {
def task = new DevUpdateTestBatch() def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path) task.addDev(path)
task.addDevAddress(path) // task.addDevAddress(path)
// //
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是") // task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
@@ -42,7 +46,7 @@ class DevUpdateTestBatch {
def addDev(path){ def addDev(path){
ExcelUtils.modify(path,{ d-> ExcelUtils.modify(path,{ d->
if(!d['I'] && d['A']){ if(!d['I'] && d['A']){
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A'])) def obj = development_zone_43_CONN.findOne(new BasicDBObject("name", d['A']))
if(!obj){ if(!obj){
def o = new BasicDBObject() def o = new BasicDBObject()
o.put("province", d['B']) o.put("province", d['B'])
@@ -28,7 +28,7 @@ class GXDdistrictMatch {
static main(args) { static main(args) {
def task = new GXDdistrictMatch(); def task = new GXDdistrictMatch();
// task.run() // task.run()
task.addOldMatch() task.addOldMatch2()
// task.runDb() // task.runDb()
// task.updateDistStatus()//恢复name_cat不正常数据 // task.updateDistStatus()//恢复name_cat不正常数据
@@ -68,6 +68,32 @@ class GXDdistrictMatch {
} }
def addOldMatch2(){
//老匹配
def fileName = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx"//11城重新匹配.xlsx
ExcelUtils.multiThreadingModify(fileName, { d ->
println d['C']
if(!d['AJ']){
def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D'])
if (jsonObject != null && !jsonObject.containsKey("result")) {
d['AJ'] = jsonObject.get("id")
d['AK'] = jsonObject.get("city")
d['AL'] = jsonObject.get("district")
d['AM'] = jsonObject.get("name")
d['AN'] = jsonObject.get("address")
def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
if(dbo){
d['AO'] = dbo.status
d['AP'] = dbo.name_cat
d['AQ'] = dbo.estate_type2
}
}
}
});
}
def runDb(){ def runDb(){
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx" def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
ExcelUtils.modify(fileName, { d -> ExcelUtils.modify(fileName, { d ->
@@ -35,7 +35,7 @@ class ThirdPriceMappingBatch extends BaseBatch{
final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018) final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018)
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
static main(args) { static main(args) {
runBatch(new ThirdPriceMappingBatch(), args); runBatch(new ThirdPriceMappingBatch(), args);
@@ -44,9 +44,9 @@ class ThirdPriceMappingBatch extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
// //将新增数据同步过来 // //将新增数据同步过来
// addNemCommunity() addNemCommunity()
// //接口进行匹配 // //接口进行匹配
match() // match()
// addOldMatchResult() // addOldMatchResult()
} }
@@ -151,8 +151,9 @@ class ThirdPriceMappingBatch extends BaseBatch{
sd.each {k,v-> sd.each {k,v->
udata.put(k,v) udata.put(k,v)
} }
udata.removeField("_id")
udata.put("m_date", new Date()) udata.put("m_date", new Date())
thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata)) thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata),false, false)
}else{ }else{
sd.put("m_date", new Date()) sd.put("m_date", new Date())
thirdSeaweedMappingColl.save(sd) thirdSeaweedMappingColl.save(sd)
@@ -1,6 +1,7 @@
package com.fangjia.script.gj.data_handle package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeocodeUtils import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
@@ -900,41 +901,92 @@ class TownMiddleProcess {
} }
def updateTownPolineBatch(){ def updateTownPolineBatch(){
def i = 0
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx",{ row-> ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx",{ row->
if(row['R'] && row['R']=="新增围栏"){ if(row['R'] && row['R']=="新增围栏"){
def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D']) i++
println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q) if(i>=134){
def d = TOWNPOLYLINE_COLL.findOne(q) def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D'])
if(d){ println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q)
def resultMap = [:] def d = TOWNPOLYLINE_COLL.findOne(q)
resultMap.type = "polygon" if(d){
def arrs = d.polyline.split("\\|") def resultMap = [:]
def rets = [] def bd_resultMap = [:]
for(def s : arrs){
def points = s.split(";") resultMap.type = "polygon"
def pointList = [] bd_resultMap.type = "polygon"
for(def pointStr : points){
def pointArr = pointStr.split(",") def arrs = d.polyline.split("\\|")
def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble()) def rets = []
def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble()) def bd_rets = []
pointList << [lon, lat]
println row['N'] +" 共有:"+arrs.size()+" 个围栏"
def index = 1
for(def s : arrs){
def points = s.split(";")
def pointList = []
def bd_pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble())
def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble())
pointList << [lon, lat]
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lon, lat, 3)
def bd_lng = baidu.x
def bd_lat = baidu.y
bd_pointList << [bd_lng, bd_lat]
}
if(index<=1){
resultMap.coordinates = pointList
bd_resultMap.coordinates = bd_pointList
def udata = new BasicDBObject()
udata.put("shape", resultMap)
udata.put("operator", "高娟")
udata.put("m_date", new Date())
udata.put("bd_shape",bd_resultMap)
def updateQuery = new BasicDBObject("name",row['N'])
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}else{
def dbo = DEVELOPMENT_ZONE_43_COLL.findOne(new BasicDBObject("name",row['N']))
def obj = new BasicDBObject()
dbo.each {k,v->
if(k!='_id'){
obj.put(k,v)
}
}
resultMap.coordinates = pointList
bd_resultMap.coordinates = bd_pointList
obj.put("shape", resultMap)
obj.put("operator", "高娟")
obj.put("m_date", new Date())
obj.put("bd_shape",bd_resultMap)
obj.put("create_time", new Date())
obj.put("block", dbo.name+"-地块"+index)
DEVELOPMENT_ZONE_43_COLL.save(obj)
}
// rets << pointList
// bd_rets << bd_pointList
index++
} }
rets << pointList
// if(rets.size()>1) {
// resultMap.type = "MultiPolygon"
// bd_resultMap.type = "MultiPolygon"
// }
// println d.city+" "+d.region+" "+d.town+" "+rets.size()
row['S'] = "已完成"
} }
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+d.town+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def updateQuery = new BasicDBObject("name",row['N'])
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
row['S'] = "ÒÑÍê³É"
} }
} }
}) })
} }
@@ -53,7 +53,7 @@ class ImportIndexStats extends BaseBatch{
public static final def dingjiYear = 2020; public static def dingjiYear = 2020;
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final EsDataSource dataSource = new EsDataSource() private static final EsDataSource dataSource = new EsDataSource()
@@ -76,22 +76,23 @@ class ImportIndexStats extends BaseBatch{
// synch() // synch()
// synchTjjIndex()//同步统计局抓取过来的指数 // synchTjjIndex()//同步统计局抓取过来的指数
// Calendar calendar=Calendar.getInstance(); Calendar calendar=Calendar.getInstance();
// def month= calendar.get(Calendar.MONTH); def month= calendar.get(Calendar.MONTH);
// def year= calendar.get(Calendar.YEAR); def year= calendar.get(Calendar.YEAR);
// def date = createDateString(year, month, 0, ""); def date = createDateString(year, month, 0, "");
// calculatePriceIndex(year, month, date); calculatePriceIndex(year, month, date);
// addLostIndex() // addLostIndex()
// addHistoryIndex()
addHistoryIndex()
} }
def addLostIndex(){ def addLostIndex(){
def citys = CommonDatautils.findAllCity(); def citys = CommonDatautils.findAllCity();
def index = 0
for(def city : citys){ for(def city : citys){
index++
if(cityList.contains(city)){ if(cityList.contains(city)){
println city +" Êý¾ÝÀ´Ô´ÓÚͳ¼Æ¾Ö" println "[${index}] "+ city +" 数据来源于统计局"
continue; continue;
} }
def start = findStartOrEndIndex(city, 1) def start = findStartOrEndIndex(city, 1)
@@ -117,21 +118,48 @@ class ImportIndexStats extends BaseBatch{
if(nextObj && nextObj.chain){ if(nextObj && nextObj.chain){
indexs << nextObj.chain indexs << nextObj.chain
} }
udata.put("chain",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size())) if(indexs){
udata.put("remark","ÍÆµ¼") udata.put("chain",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
udata.put("remark","推导")
}
}
if(!dbo.yeartoyear){
def indexs = []
if(lastMonthObj && lastMonthObj.yeartoyear){
indexs << lastMonthObj.yeartoyear
}
if(nextObj && nextObj.yeartoyear){
indexs << nextObj.yeartoyear
}
if(indexs){
udata.put("yeartoyear",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
udata.put("remark","推导")
}
}
if(!dbo.fixedbase){
def indexs = []
if(lastMonthObj && lastMonthObj.fixedbase){
indexs << lastMonthObj.fixedbase
}
if(nextObj && nextObj.fixedbase){
indexs << nextObj.fixedbase
}
if(indexs){
udata.put("fixedbase",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
udata.put("remark","推导")
}
}
if(udata){
println city +" 补充"
STATS_INDEX_COLL.update(new BasicDBObject("_id", dbo._id), new BasicDBObject('$set', udata),false, true); STATS_INDEX_COLL.update(new BasicDBObject("_id", dbo._id), new BasicDBObject('$set', udata),false, true);
} }
// if(!dbo.yeartoyear){
//
// }
// if(!dbo.fixedbase){
//
// }
} }
} }
} }
println "[${index}] "+ city +" 已处理完"
} }
} }
@@ -154,7 +182,7 @@ class ImportIndexStats extends BaseBatch{
} }
def addHistoryIndex(){ def addHistoryIndex(){
def s = "202101" def s = "202001"
def startDate = null; def startDate = null;
def stop = false; def stop = false;
def runDates = [] def runDates = []
@@ -171,7 +199,7 @@ class ImportIndexStats extends BaseBatch{
runDates << map runDates << map
s = DateUtils.addMonth(date, 1, "yyyyMM") s = DateUtils.addMonth(date, 1, "yyyyMM")
if(year==2024 && month==9){ if(year==2024 && month==8){
stop = true; stop = true;
} }
} }
@@ -182,6 +210,7 @@ class ImportIndexStats extends BaseBatch{
//计算统计局没有发布的城市的价格指数 //计算统计局没有发布的城市的价格指数
def calculatePriceIndex(def year, def month, def date){ def calculatePriceIndex(def year, def month, def date){
// def citys = ["五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]//
def citys = CommonDatautils.findAllCity(); def citys = CommonDatautils.findAllCity();
// def date = createDateString(year, month, 0, ""); // def date = createDateString(year, month, 0, "");
def haveIndexCitys = findPriceIndexCitys(date.toInteger()); def haveIndexCitys = findPriceIndexCitys(date.toInteger());
@@ -193,7 +222,7 @@ class ImportIndexStats extends BaseBatch{
def newCitys = [] def newCitys = []
for(def i=0;i<citys.size();i++) { for(def i=0;i<citys.size();i++) {
def city = citys[i] def city = citys[i]
if (!haveIndexCitys.contains(city)) { if (!haveIndexCitys.contains(city) && !cityList.contains(city)) {
newCitys << city newCitys << city
} }
} }
@@ -204,6 +233,15 @@ class ImportIndexStats extends BaseBatch{
if(id){//已计算 if(id){//已计算
continue; continue;
} }
if(date.toInteger()>=201601){
dingjiYear = 2015
}else if(date.toInteger()>=202101){
dingjiYear = 2020
}else if(date.toInteger()>=202601){
dingjiYear = 2025
}else if(date.toInteger()>=203101){
dingjiYear = 2030
}
fixedThreadPool.execute(new Runnable() { fixedThreadPool.execute(new Runnable() {
void run() { void run() {
try{ try{
@@ -220,7 +258,7 @@ class ImportIndexStats extends BaseBatch{
} }
}); });
} }
latch.await(20, TimeUnit.SECONDS); latch.await(1, TimeUnit.MINUTES);
} }
def calculating(city, year, month, date){ def calculating(city, year, month, date){
@@ -647,6 +685,7 @@ class ImportIndexStats extends BaseBatch{
DBObject q = new BasicDBObject(); DBObject q = new BasicDBObject();
q.put("yearmonth", date); q.put("yearmonth", date);
q.put("cat", "district") q.put("cat", "district")
q.put("type", "城市")
DBObject keys = new BasicDBObject(); DBObject keys = new BasicDBObject();
keys.put("city", 1) keys.put("city", 1)
@@ -666,6 +705,7 @@ class ImportIndexStats extends BaseBatch{
q.put("yearmonth", date); q.put("yearmonth", date);
q.put("city", city); q.put("city", city);
q.put("cat", "district"); q.put("cat", "district");
q.put("type", "城市")
DBObject keys = new BasicDBObject(); DBObject keys = new BasicDBObject();
keys.put("city", 1) keys.put("city", 1)
@@ -684,6 +724,7 @@ class ImportIndexStats extends BaseBatch{
dbo.put("source",result.source); dbo.put("source",result.source);
dbo.put("remark",result.remark); dbo.put("remark",result.remark);
dbo.put("cat",result.cat); dbo.put("cat",result.cat);
dbo.put("type",result.type);
dbo.put("c_date",new Date()); dbo.put("c_date",new Date());
if(result.markup){ if(result.markup){
dbo.put("chain", result.markup.toDouble()); dbo.put("chain", result.markup.toDouble());