Merge branch 'script_gj' into 'master'

Script gj

See merge request !981
This commit is contained in:
2024-10-22 18:21:10 +08:00
27 changed files with 1652 additions and 225 deletions
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年7月深圳二手房成交.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年9月深圳二手房成交.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -13,19 +13,24 @@ class AddThirdPriceSample {
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static final DBCollection NO_PRICE_DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com")
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
static main(args) {
def task = new AddThirdPriceSample()
task.run()
task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据
// task.addTrade() //添加成交
}
def run(){
for(def i=1;i<19;i++){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d->
if(d['A'] && d['C']){
for(def i=0;i<19;i++){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达参考价作为第三方入库.xlsx",{d->
if(d['A'] && d['B']){
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
if(dist){
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407)
def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202408)
// def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
def udata = new BasicDBObject("fccs_esf_price", d['B'].toString().toDouble().toInteger())
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
}else{
println d['A'] +" 没查到"
@@ -37,4 +42,67 @@ class AddThirdPriceSample {
}
}
def addTrade(){
def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"]
def rets = findTrades(citys)
// DEAL_PRICE_COLL.insert(rets)
}
def findTrades(citys){
def q = new BasicDBObject()
q.put("city", new BasicDBObject('$in', citys))
q.put("flag", 1)
def dbcursor = NO_PRICE_DEAL_PRICE_COLL.find(q)
def rets = []
def index = 0
for(def db : dbcursor){
index++
def udata = new BasicDBObject()
udata.put("_id",db._id)
udata.put("url",db.url)
udata.put("area",db.area)
udata.put("avg_price",db.avg_price.toString().toDouble().toInteger())
udata.put("build_structure",db.build_structure)
udata.put("build_type",db.build_type)
udata.put("build_year",db.build_year)
udata.put("city",db.city)
udata.put("direction", db.direction)
udata.put("district_name",db.district_name)
udata.put("elevator",db.elevator)
udata.put("fitment",db.fitment)
udata.put("floor_info",db.floor_info)
udata.put("hall",db.hall.toString().toInteger())
udata.put("height",db.height.toString().toInteger())
udata.put("housing_age",db.housing_age)
udata.put("lift",db.lift.toString().toInteger())
udata.put("lift_household",db.lift_household.toString().toInteger())
udata.put("property_ownership",db.property_ownership)
udata.put("region",db.region)
udata.put("room", db.room.toString().toInteger())
udata.put("room_structure",db.room_structure)
udata.put("source",db.source)
udata.put("total_price",db.total_price.toString().toInteger())
udata.put("trade_date",db.trade_date)
udata.put("trade_property",db.trade_property)
udata.put("house_using",db.house_using)
udata.put("heating_type",db.heating_type)
udata.put("community_id",db.community_id)
udata.put("create_date",db.create_time)
udata.put("m_date", new Date())
udata.put("address",db.address)
udata.put("match_city",db.match_city)
udata.put("match_region",db.match_region)
udata.put("match_name",db.match_name)
udata.put("match_id",db.match_id)
udata.put("price_flag", "国信达成交-推导")
DEAL_PRICE_COLL.save(udata)
println index
// rets << udata
}
return rets
}
}
@@ -40,7 +40,7 @@ class CcbCommonUtils {
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
// public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com")
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
@@ -5,9 +5,14 @@ import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 建行成交数据表优化
*/
@@ -24,6 +29,8 @@ class DealPriceDataProcess {
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new DealPriceDataProcess()
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
@@ -394,43 +401,73 @@ class DealPriceDataProcess {
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
def index = 0
for(def city : citys) {
index ++
if(index<145){
continue
}
println index +" "+city
def cityid = cityCodeDatas.get(city)
def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
println city +" "+rets.size()
if(rets){
set.add(city)
}
for(def m : rets){
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
// sqlInstance236.executeUpdate(sql)
def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
def total = 0
pgsqlInstance.eachRow(querySql, { r ->
total = r.count
})
if(total>0){
println city +" "+m.commid+" (按价格)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
pgsqlInstance.executeUpdate(sql)
}else{
querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
pgsqlInstance.eachRow(querySql, { r ->
total = r.count
})
if(total>0){
println city +" "+m.commid+" (按小区)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
pgsqlInstance.executeUpdate(sql)
}
}
}
processDetele(city, cityid, month, rets)
}
println set.toString()
}
def processDetele(city, cityid, month, rets){
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def m = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
// sqlInstance236.executeUpdate(sql)
def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
def total = 0
pgsqlInstance.eachRow(querySql, { r ->
total = r.count
})
if(total>0){
println city +" "+m.commid+" (按价格)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
pgsqlInstance.executeUpdate(sql)
}else{
querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
pgsqlInstance.eachRow(querySql, { r ->
total = r.count
})
if(total>0){
println city +" "+m.commid+" (按小区)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
pgsqlInstance.executeUpdate(sql)
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def findThirdCount(sqlInstance236, sql){
def count = 0
sqlInstance236.eachRow(sql,{r->
@@ -631,36 +631,36 @@ def findAllData(sqlInstance236, citycode){
//},0)
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
if(d['A'] && d['B']){
def dbo = CommonDatautils.findDistrictById(d['A'])
if(dbo){
def alias = null
if(!dbo.alias){
alias = []
}else{
alias = dbo.alias
}
if(!alias.contains(d['B'])){
alias << d['B']
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", dbo._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{
println d['A']+" "+"别名已存在"
}
}else{
println dbo._id.toString() +" 没找到"
}
}
})
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
// if(d['A'] && d['B']){
// def dbo = CommonDatautils.findDistrictById(d['A'])
// if(dbo){
// def alias = null
// if(!dbo.alias){
// alias = []
// }else{
// alias = dbo.alias
// }
// if(!alias.contains(d['B'])){
// alias << d['B']
// }
// if(alias){
// def udata = new BasicDBObject()
// udata.put("alias", alias)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", dbo._id)
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }else{
// println d['A']+" "+"别名已存在"
// }
//
// }else{
// println dbo._id.toString() +" 没找到"
// }
// }
//})
//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
//for(def c : citys){
@@ -1331,34 +1331,34 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx"
//
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['J']){
//// def region = RegionData.formatRegion(d['A'], d['J'])
//// if(region){
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
//// if(dist){
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['L'] = dist.status-dist.name_cat
//// }
//// }
//// if(d['J'] instanceof String){
// def dist = CommonDatautils.findDistrictById(d['J'])
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['G']){
// def region = RegionData.formatRegion(d['A'], d['J'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
// if(dist){
// d['B'] = dist.region
// d['C'] = dist.name
// d['L'] = dist.status-dist.name_cat
// }
// }
if(d['G'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['G'])
if(dist){
// d['A'] = dist.city
// d['B'] = dist.region
// d['C'] = dist.name
// d['K'] = dist.estate_type2
// d['L'] = dist.status
// d['M'] = dist.name_cat
//
//// d['H'] = dist.name
// }
//// }
// }
//},0)
d['H'] = dist.name
}
}
}
},0)
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
@@ -47,7 +47,8 @@ class TradeStatLog {
// task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
task.exportTrade2()//导出未入库成交
// task.exportTrade2()//导出未入库成交
task.updateDealPriceRegion()//修改成交区域数据
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -597,7 +598,7 @@ class TradeStatLog {
// def endDate = "2024-07-01"
def ids = findAllDealIds();
println ids.size()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\廊坊.xlsx",{d->
if(ids.contains(d['A'])){
d['V'] = "供"
}
@@ -674,7 +675,7 @@ class TradeStatLog {
}
def findAllDealIds(){
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'"
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='廊坊'"
def ids = []
pssqlInstance.eachRow(sql,{r->
if(!ids.contains(r.gujia_bds_mdl_id)){
@@ -801,6 +802,20 @@ class TradeStatLog {
},0)
}
def updateDealPriceRegion(){
ExcelUtils.modify("D:\\数据\\建行\\建行成交\\成交待修改区域数据.xlsx",{d->
if(d['A'] && d['G']){
def region = RegionData.formatRegion(d['D'], d['G'])
if(region){
def udata = new BasicDBObject()
udata.put("region", region)
DEAL_PRICE_COLL.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}
}
},0)
}
def findTradeById(id){
def q = new BasicDBObject()
@@ -24,6 +24,13 @@ class AddressCut extends BaseBatch{
final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
final DBCollection thirdPartyPriceBkColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price_history_bk_new", "192.168.0.235", 27018)
final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018)
final DBCollection dataProcessBkColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process_history_bk", "192.168.0.235", 27018)
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
@@ -39,50 +46,66 @@ class AddressCut extends BaseBatch{
@Override
public void process(Object args) {
<<<<<<< HEAD
// sameAddr()
seaweedStreetUpdate()
def dates = [202408]
for(def date : dates){
// synch(date)
synchDataProcess(date)
println date + "同步完成! "
}
}
def seaweedStreetUpdate(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区地址拆分\\已核实-路2.xlsx",{d->
if(d['A']){
def dbo = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(dbo){
def udate = new BasicDBObject()
def street = null
def address = ""
if(d['E']){
street = d['E']
udate.put("street", street)
address = street
}
def streetNum = null
if(d['F']){
streetNum = d['F']
udate.put("street_number", streetNum)
address = address + streetNum
}
if(dbo.address!=address){
udate.put("address", address)
}
if(udate){
udate.put("m_date", new Date())
DBUtils.seaweed.update(new BasicDBObject('_id', dbo.get("_id")), new BasicDBObject('$set', udate))
}
}
def synch(date){
def q = new BasicDBObject()
q.put("s_date", date)
def dbcursor = thirdPartyPriceColl.find(q).batchSize(5000)
def index = 0
def rets = []
def num = 0
for(def d : dbcursor){
num++
index++
rets << d
if(index%10000==0){
thirdPartyPriceBkColl.insert(rets)
println(num)
rets.clear()
}
})
=======
// sameAddr() //相同地址小区
// roadMatch()
addAnjukeRoad()
}
if(rets.size()>0){
thirdPartyPriceBkColl.insert(rets)
rets.clear()
}
}
def synchDataProcess(date){
def q = new BasicDBObject()
q.put("s_date", date)
def dbcursor = dataProcessColl.find(q).batchSize(5000)
def index = 0
def rets = []
def num = 0
for(def d : dbcursor){
num++
index++
rets << d
if(index%10000==0){
dataProcessBkColl.insert(rets)
println(num)
rets.clear()
}
}
if(rets.size()>0){
dataProcessBkColl.insert(rets)
rets.clear()
}
}
def addAnjukeRoad(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d ->
def roads = findThirdPartyPrice(d['A'], d['B'], d['C'], "链家")
@@ -130,8 +153,6 @@ class AddressCut extends BaseBatch{
}
})
>>>>>>> c1a5ab8c6aa973a91e6be5547ec0925db3fa500f
}
def sameAddr() {
@@ -122,6 +122,8 @@ class CommonDatautils {
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
return house_dataStoreConnection
@@ -820,6 +822,7 @@ class CommonDatautils {
DBObject q = new BasicDBObject();
q.put("status",0)
// q.put("status",new BasicDBObject('$ne', 0))
q.put("cat", cat);
q.put("city", city)
@@ -829,7 +832,7 @@ class CommonDatautils {
keys.put(field, 1);
}
}
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
def rets = []
for(def d : dbsour){
rets << d
@@ -959,13 +962,7 @@ class CommonDatautils {
keys.put(field, 1);
}
}
def cursor = getDataStoreConnection().find(q, keys).batchSize(2000)
if(cursor){
for(def obj : cursor){
return obj;
}
}
return null;
return SEAWEED_COLL.findOne(q, keys)
}
@@ -1132,6 +1129,10 @@ class CommonDatautils {
q.put("city", city);
q.put("cat", cat);
q.put("name", name);
if(region){
q.put("region", region);
}
q.put("status",new BasicDBObject('$ne', 0))
DBObject keys = new BasicDBObject();
if(fields){
@@ -1680,7 +1681,7 @@ class CommonDatautils {
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("category","region");
queryCondition.put("show",1);
// queryCondition.put("show",1);
def dbCursor = REGION_BLOCK_COLL.find(queryCondition)
def regions = [];
for(DBObject obj : dbCursor){
@@ -2385,6 +2386,9 @@ class CommonDatautils {
q.put("city", city);
if(region){
q.put("region", region);
q.put("cat", "区域")
}else{
q.put("cat", "城市")
}
q.put('weekend', weekend);
def dbCursor = REGION_STAT_COLL.find(q).sort(new BasicDBObject().append("weekend", -1)).limit(1);
@@ -3735,9 +3739,9 @@ class CommonDatautils {
if(!region){
if(city){
q.put("cat", "city")
}else{
q.put("cat", "region")
}
}else{
q.put("cat", "region")
}
}
@@ -4095,6 +4099,21 @@ class CommonDatautils {
return retMap;
}
static def findHouseCount(def city, def region, def name){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
.must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("district_name", name)).must(QueryBuilders.termQuery("cat", "sell"))
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(5);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
return houses.size();
}
/**
* 是否是别墅
* @return
@@ -5060,4 +5079,38 @@ class CommonDatautils {
}
static def findStatIndexBySort(type, city, region, sort){
def q = new BasicDBObject()
q.put("cat", "district")
q.put("type", type)
if(region){
q.put("district", region)
}
if(city){
q.put("city", city)
}
def cur = STATS_INDEX_COLL.find(q).sort(new BasicDBObject("yearmonth", sort)).limit(1)
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def obj = 0;
while(cur.hasNext()) {
obj = cur.next()
}
return obj;
}
static def findStatIndexByMonth(type, city, region, month){
def q = new BasicDBObject()
q.put("cat", "district")
q.put("type", type)
if(region){
q.put("district", region)
}
if(city){
q.put("city", city)
}
q.put("yearmonth", month)
return STATS_INDEX_COLL.findOne(q)
}
}
@@ -2,8 +2,10 @@ package com.fangjia.script.gj
import com.fangjia.core.BlockData
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
@@ -531,6 +533,28 @@ class FiledStrand {
return null
}
static def processLngAndLat(val) {
if (val) {
def map = [:]
def lng = DataHandleUtils.getDoubleRound(6, val.x)
def lat = DataHandleUtils.getDoubleRound(6, val.y)
Long gd_lat = lat * 10000000000L
Long gd_lng = lng * 10000000000L
map.lat = gd_lat
map.lng = gd_lng
def baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
Long bd_lng = baidu.x * 10000000000L
Long bd_lat = baidu.y * 10000000000L
map.lat2 = bd_lat
map.lng2 = bd_lng
return map
}
return null
}
static def isRoad(def s){
def pattern = Pattern.compile("[路|街|大道|巷]");
def matcher = pattern.matcher(s)
@@ -2,33 +2,97 @@ package com.fangjia.script.gj
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d->
if(d['A'] && !d['E']){
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district")
// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district")
println d['A'] +" "+d['C']
// def total = findDatas(d['A'], d['B'], d['C'])
// d['D'] = total
def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·",""))
d['E'] = c
}
})
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
//ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d->
// if(d['A'] && !d['E']){
//// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district")
//// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district")
// println d['A'] +" "+d['C']
//// def total = findDatas(d['A'], d['B'], d['C'])
//// d['D'] = total
// def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·",""))
// d['E'] = c
// }
//})
def URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection seaweed_coll = MongoUtils.getCollectionByName2("fangjia", "seaweed", URL)
def q = new BasicDBObject()
q.put("city", "上海市")
q.put("category", "district")
q.put("status", 0)
def ret = []
def list = seaweed_coll.find(q).batchSize(5000).toList()
println "总计:"+list.size()
//for(def l : list){
// def obj = getAmapLocation(l.city, l.region, l.name)
// if(obj){
// ret << [l.city, l.region, l.name, obj.cityname, obj.adname, obj.name,obj.type]
// }
//}
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(list.size())
def total = list.size()
for(def i=0;i<list.size();i++){
final def l = list[i]
def dist = CommonDatautils.findDistrict(l.fangjia_city, l.fangjia_region, l.name, "district")
fixedThreadPool.execute(new Runnable() {
void run() {
try{
addCount.addAndGet(1)
def alias = []
alias << dist.name
if(l.alias){
alias.addAll(l.alias)
}
for(def alia : alias){
def obj = getAmapLocation(l.city, l.region, alia)
if(obj){
ret << [l.city, l.region, l.name, obj.cityname, obj.adname, obj.name,obj.type]
break
}
}
println addCount.get()+"/"+total+" "+ ret.size()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
ExcelUtils.write("D:\\数据\\数据建设\\小区\\高德地图有围栏数据.xlsx", ret, "高德")
def getAmapLocation(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
def total = 0
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name.contains(name) && d.type.contains("住宅")){
total ++
if(d.name.contains(name) && d.type.contains("住宅") && d.shape){
obj = d
}
})
return total;
return obj;
}
@@ -6,21 +6,33 @@ import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.MongoClient
import com.rabbitmq.client.AMQP
class AddCityRegionPrice {
public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl)
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", mongoDbUrl)
static main(args) {
def task = new AddCityRegionPrice()
// task.addRegionPrice()
// task.addCityprice()
task.addCityprice()
// task.addDistrictprice()
// task.addCityRentPrice()
task.addDistrictRentprice()
// task.addRegionRentPrice()
// task.addDistrictRentprice()
// task.updateRegionRentData()
// task.updateDealPriceAreaType()
// task.checkCityImport()
}
def addRegionPrice(){
@@ -54,7 +66,7 @@ class AddCityRegionPrice {
def addCityprice(){
InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch()
def citys = CommonDatautils.findAllCity().reverse()
def citys = ["乐东"]// CommonDatautils.findAllCity().reverse()
def add = 0
for(def c : citys){
def startObj = CommonDatautils.queryCityOrRegionPrice(c, null, 1)
@@ -125,6 +137,35 @@ class AddCityRegionPrice {
}
}
def addRegionRentPrice(){
InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch();
def citys = CommonDatautils.findAllCity()
for(def c : citys){
def regions = CommonDatautils.getCityRegions(c)
for(def r : regions){
// if(r!="南川"){
// continue
// }
def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, r, null, 1)
def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, r, null, -1)
if(startObj && endObj){
def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409")
months.each {m->
// println m
def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd")))
def obj = CommonDatautils.findSqmetreRent(c, r, null, lastDay)
if(!obj){
inserBatch.saveCityCalcCommand(c, r, null, lastDay, "fangjia-priceanalysis-0")
println "[add] "+c+" "+r+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd")
}
}
}
}
}
}
def addDistrictRentprice(){
InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch();
def citys = CommonDatautils.findAllCity()
@@ -158,4 +199,72 @@ class AddCityRegionPrice {
}
}
def updateRegionRentData(){
def q = new BasicDBObject("cat", "region")
def keys = new BasicDBObject()
keys.put("markup", 1)
def dbcursor = SQMETRE_RENT_COLL.find(q, keys).batchSize(5000)
def index = 1
for(def d : dbcursor){
if(!(d.markup instanceof Double)){
def udata = new BasicDBObject()
double markup = 1.0
markup = d.markup * markup
udata.put("markup", markup)
SQMETRE_RENT_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
println index
index++
}
}
}
def updateDealPriceAreaType(){
def index = 1
def citys = ["凉山州","福州","南京","佛山","沈阳","苏州"]
citys.each {c->
def q = new BasicDBObject()
q.put("city", c)
// q.put("price_flag", "国信达成交-推导")
def dbcursor = DEAL_PRICE_COLL.find(q)
for(def d : dbcursor){
// if(!(d.area instanceof Double)){
// def udata = new BasicDBObject()
// double area = 1.0
// area = d.area * area
// udata.put("area", area)
// DEAL_PRICE_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
// println c+" "+index
// index++
// }
if(!(d.total_price instanceof Integer)){
def udata = new BasicDBObject()
udata.put("total_price", d.total_price.toString().toDouble().toInteger())
DEAL_PRICE_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
println c+" "+index
index++
}
}
}
}
def checkCityImport(){
def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
for(def c : citys){
def startObj = CommonDatautils.findStatIndexBySort("城市", c, null, 1)
def endObj = CommonDatautils.findStatIndexBySort("城市", c,null, -1)
if(startObj && endObj){
def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202409")
months.each {m->
def obj = CommonDatautils.findStatIndexByMonth("城市", c,null, m.toInteger())
if(!obj){
println "[add] "+c+" "+m
}
}
}else{
println c +" 一个也没有"
}
}
}
}
@@ -0,0 +1,286 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 添加高德地图,百度地图围栏
*/
@BatchOwner(Watcher.GaoJuan)
class AddDistrictShapeBatch extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameCity()
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
runBatch(new AddDistrictShapeBatch(), args)
}
@Override
public void process(Object args) {
def citys = CommonDatautils.findAllCity()
def index = 1
for(def c : citys){
println c +" "+index
index ++
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
// baiduMatch(c, districts)
gaodeMatch(c, districts)
}
}
def baiduMatch(city,districts){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
final CountDownLatch latch = new CountDownLatch(districts.size())
if(fullCityName){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def baiduId = baiduProcess(d, fullCityName)
if(baiduId){
if(!rets.contains(baiduId)){
rets << baiduId
}else{
println "百度围栏 "+baiduId +" 被:["+d.city+" "+d.region+" "+d.name+" ]再次匹配,查看原始小区是否重复或别名相同"
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
}else{
println city +" 找不到相应的全称"
}
latch.await()
}
def baiduProcess(d, fullCityName){
def alias = []
alias << d.name
if(d.alias){
alias.addAll(d.alias)
}
def distData = mcaDistMap.get(d.city+"#"+d.region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
def fenceMap = null
for(def name : alias){
fenceMap = findBaiduDatas(fullCityName, fullDistName, name)
if(!fenceMap){
continue
}
}
if(fenceMap){
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", d.region)
o.put("name", d.name)
o.put("source", "百度地图")
o.put("c_date", new Date())
o.put("data_id", fenceMap.data_id)
o.put("comm_id", d._id.toString())
def resultMap = [:]
resultMap.coordinates = fenceMap.coordinatePairs
// if(fenceMap.coordinatePairs.size()>1) {
// resultMap.type = "MultiPolygon"
// }else{
resultMap.type = "polygon"
// }
o.put("shape", resultMap)
def q = new BasicDBObject("comm_id", d._id.toString())
seaweed_fence_coll.update(q, new BasicDBObject('$set', o),true,false)
// seaweed_fence_coll.save(o)
println "[save] 围栏 ( "+d.city +" "+d.region +" "+d.name+" )"
return fenceMap.data_id
}
return null
}
def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
q.put("guoke_geo", new BasicDBObject('$ne', null))
def dbcursor = BAIDU_POI_COLL.find(q)
def map = [:]
println dbcursor.count()
for(def d : dbcursor){
def arrs = d.gcj02_guoke_geo.split("\\|")
def coordinatesStr = arrs[2].toString().substring(2, arrs[2].toString().size()-1 )
// 将字符串分割成列表
def coordinatePairs = coordinatesStr.split(',')
def coordinatesList = []
for (int i = 0; i < coordinatePairs.size(); i += 2) {
// 将字符串坐标转换为浮点数
double longitude = coordinatePairs[i] as double
double latitude = coordinatePairs[i + 1] as double
// 将坐标点作为一个元组添加到集合中
coordinatesList << [longitude, latitude]
}
if(coordinatePairs){
map.data_id = d._id
map.coordinatePairs = coordinatesList
}
}
return map;
}
def gaodeMatch(city,districts){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
final CountDownLatch latch = new CountDownLatch(districts.size())
if(fullCityName){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
for(def i=0;i<districts.size();i++){
final def d = districts[i]
def fence = findSeaweedFenceExists(d._id.toString())
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(!fence){
def baiduId = gaodeProcess(d, fullCityName)
if(baiduId){
if(!rets.contains(baiduId)){
rets << baiduId
}else{
println "高德围栏 "+baiduId +" 被:["+d.city+" "+d.region+" "+d.name+" ]再次匹配,查看原始小区是否重复或别名相同"
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
}else{
println city +" 找不到相应的全称"
}
latch.await()
}
def gaodeProcess(d, fullCityName){
def alias = []
alias << d.name
if(d.alias){
alias.addAll(d.alias)
}
def distData = mcaDistMap.get(d.city+"#"+d.region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
def fenceMap = null
for(def name : alias){
fenceMap = findGaoDeDatas(fullCityName, fullDistName, name)
if(!fenceMap){
continue
}
}
if(fenceMap){
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", d.region)
o.put("name", d.name)
o.put("source", "高德地图")
o.put("c_date", new Date())
o.put("data_id", fenceMap.id)
o.put("comm_id", d._id.toString())
def resultMap = [:]
def shape = fenceMap.shape
def points_x = fenceMap.shape.polygon.x
def points_y = fenceMap.shape.polygon.y
def list = []
for(def i=0;i<points_x.size();i++){
list << [points_x[i], points_y[i]]
}
resultMap.coordinates = list
resultMap.type = "polygon"
o.put("shape", resultMap)
def q = new BasicDBObject("comm_id", d._id.toString())
seaweed_fence_coll.update(q, new BasicDBObject('$set', o),true,false)
// seaweed_fence_coll.save(o)
println "[save] 高德围栏 ( "+d.city +" "+d.region +" "+d.name+" )"
return fenceMap.data_id
}
return null
}
def findGaoDeDatas(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
def newName = d.name.replace("-","").replace("·","")
if(newName == name && d.type.contains("住宅") && d.shape){
obj = d
}
})
return obj;
}
def findSeaweedFenceExists(commid){
def q = new BasicDBObject()
q.put("comm_id", commid)
return seaweed_fence_coll.findOne(q)
}
}
@@ -0,0 +1,234 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 小区自动入库(新增小区,每月运行一次)
*/
class AutoAddDistrictBatch extends BaseBatch{
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def fullCityMap = CommonDatautils.queryFullNameCity()
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
static main(args) {
runBatch(new AutoAddDistrictBatch(), args)
}
@Override
void process(Object args) {
//查找第三方匹配没有匹配上的小区
def citys = CommonDatautils.findAllCity()
def list = []
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
for(def c : citys){
def noMatchDatas = findNoMatchResult(c)
println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
def rets = checkDatas(noMatchDatas.keySet())
list.addAll(rets)
}
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\名录集\\第三方待入库数据.xlsx"
}
ExcelUtils.write(fileName, list, "待入库数据")
}
def checkDatas(keysets){
def noMatchCount = keysets.size()
final CountDownLatch latch = new CountDownLatch(noMatchCount)
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
for(def i=0;i<keysets.size();i++){
final def key = keysets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def arrs = key.split("##")
//检测有没有成交
def trade_count = findTrades(arrs[0], arrs[1], arrs[2])
//检测有没有房源
def house_count = CommonDatautils.findHouseCount(arrs[0], arrs[1], arrs[2])
//检查百度地图有没有
def baidu_count = findBaiduCount(arrs[0], arrs[1], arrs[2])
//检查高德地图有没有
def gaode_count = findGaodeCount(arrs[0], arrs[1], arrs[2])
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_count]
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def findBaiduCount(city, region, name){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
if(fullCityName){
def distData = mcaDistMap.get(city+"#"+region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
if(fullDistName){
return findBaiduDatas(fullCityName, fullDistName, name)
}
}
return null
}
def findGaodeCount(city, region, name){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
if(fullCityName){
def distData = mcaDistMap.get(city+"#"+region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
if(fullDistName){
def obj = findGaoDeDatas(fullCityName, fullDistName, name)
if(obj){
return 1
}
}
}
return 0
}
def findGaoDeDatas(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name == name && d.type.contains("住宅") && d.shape){
obj = d
}
})
return obj;
}
def findTrades(city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
return DEAL_PRICE_COLL.count(q)
}
def findNoMatchResult(def c){
def q = new BasicDBObject()
q.put("match_id", new BasicDBObject('$exists',false))
q.put("old_match_id", new BasicDBObject('$exists',false))
q.put("city", c)
def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000)
def rMap = [:]
for(def d : dbcursor){
if(d.estate_type){
if(d.estate_type.contains("自建房")){
continue
}
if(d.estate_type.contains("购物中心")){
continue
}
if(d.estate_type.contains("住宅底商") || d.estate_type.contains("酒店") || d.estate_type.contains("商铺")){
continue
}
if(d.estate_type.contains("办公") || d.estate_type.contains("写字楼")){
continue
}
}else{
continue
}
def key = d.city + "##"+d.district+"##"+d.name
def rets = rMap.get(key)
if(!rMap.containsKey(key)){
rets = []
rMap.put(key, rets)
}
rets << d
}
return rMap
}
def createData(city, region, name, address, lng, lat){
def udata = new BasicDBObject()
udata.put("city", city)
udata.put("region", region)
name = name.replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
udata.put("name", name)
def addr = AddressUtil.formatAddress(city, address)
if(addr){
udata.put("address", addr)
}
udata.put("status", "0")
udata.put("name_cat", "0")
udata.put("cat", "district")
udata.put("username", "高娟")
def estateType2 = "普通住宅"
def type = ""
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
udata.put("estateType2", estateType2)
if(lng){
Long gd_lng = lng*10000000000
udata.put("lng", gd_lng)
}
if(lat){
Long gd_lat = lat * 10000000000
udata.put("lat", gd_lat)
}
FangjiaClientUtils.saveOrUpdateDistrict(udata)
}
def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
def count = BAIDU_POI_COLL.count(q)
return count;
}
}
@@ -46,7 +46,7 @@ class DevProcess {
def task = new DevProcess();
// task.run()
// task.parseCityAndDist()
task.devSplitRoad()
// task.devSplitRoad()
// task.devNameSplitRoad()
// task.addFlag()
// task.matchDev()
@@ -63,7 +63,7 @@ class DevProcess {
// task.createDevMapping()
// task.deleteAddr()
// task.deleteAddr2()
// task.synchDevTo100()
task.synchDevTo100()
// task.addNewDev2()
// task.addAlias()
@@ -20,6 +20,10 @@ class DevUpdateTestBatch {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_43_CONN = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
@@ -30,8 +34,8 @@ class DevUpdateTestBatch {
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
task.addDevAddress(path)
task.addDev(path)
// task.addDevAddress(path)
//
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
@@ -42,7 +46,7 @@ class DevUpdateTestBatch {
def addDev(path){
ExcelUtils.modify(path,{ d->
if(!d['I'] && d['A']){
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
def obj = development_zone_43_CONN.findOne(new BasicDBObject("name", d['A']))
if(!obj){
def o = new BasicDBObject()
o.put("province", d['B'])
@@ -28,7 +28,7 @@ class GXDdistrictMatch {
static main(args) {
def task = new GXDdistrictMatch();
// task.run()
task.addOldMatch()
task.addOldMatch2()
// task.runDb()
// task.updateDistStatus()//恢复name_cat不正常数据
@@ -68,6 +68,32 @@ class GXDdistrictMatch {
}
def addOldMatch2(){
//老匹配
def fileName = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx"//11城重新匹配.xlsx
ExcelUtils.multiThreadingModify(fileName, { d ->
println d['C']
if(!d['AJ']){
def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D'])
if (jsonObject != null && !jsonObject.containsKey("result")) {
d['AJ'] = jsonObject.get("id")
d['AK'] = jsonObject.get("city")
d['AL'] = jsonObject.get("district")
d['AM'] = jsonObject.get("name")
d['AN'] = jsonObject.get("address")
def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
if(dbo){
d['AO'] = dbo.status
d['AP'] = dbo.name_cat
d['AQ'] = dbo.estate_type2
}
}
}
});
}
def runDb(){
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
ExcelUtils.modify(fileName, { d ->
@@ -0,0 +1,65 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
//更新第三方均价表中小区区域
class RegionUpdateBatch {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new RegionUpdateBatch()
task.run()
}
def run(){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def districtMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["city","region", "name"])
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("name", 1)
keys.put("seaweed_id", 1)
def statistics = AVERAGE_STATISTICS.find(new BasicDBObject("city", city), keys)
def total = statistics.size()
final CountDownLatch latch = new CountDownLatch(total)
def totalCount = new AtomicLong()
for(def i=0;i<statistics.size();i++){
final def d = statistics[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def dist = districtMap.get(d.seaweed_id)
if(dist && dist.region!=d.region){
def udata = new BasicDBObject()
udata.put("region", dist.region)
AVERAGE_STATISTICS.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata),false,false)
}
println totalCount.get() + " / "+total + " "+ d.city
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
}
}
@@ -35,7 +35,7 @@ class ThirdPriceMappingBatch extends BaseBatch{
final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018)
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
static main(args) {
runBatch(new ThirdPriceMappingBatch(), args);
@@ -44,9 +44,9 @@ class ThirdPriceMappingBatch extends BaseBatch{
@Override
public void process(Object args) {
// //将新增数据同步过来
// addNemCommunity()
addNemCommunity()
// //接口进行匹配
match()
// match()
// addOldMatchResult()
}
@@ -151,8 +151,9 @@ class ThirdPriceMappingBatch extends BaseBatch{
sd.each {k,v->
udata.put(k,v)
}
udata.removeField("_id")
udata.put("m_date", new Date())
thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata))
thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata),false, false)
}else{
sd.put("m_date", new Date())
thirdSeaweedMappingColl.save(sd)
@@ -0,0 +1,79 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 乡镇街道 判断 中心点是否在围栏或一个围栏是否包含多个中心点
*/
@BatchOwner(Watcher.GaoJuan)
class TownCenterBatch extends BaseBatch{
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
static main(args) {
runBatch(new TownCenterBatch(), args)
}
@Override
void process(Object args) {
def shapes = findHasShapeTown()
def centers = findAllCenters()
checkCenter(shapes, centers)
}
def checkCenter(shapes, centers){
def map = [:]
for(def town : shapes){
def points = InsidePolygon.createPoint4(town.shape.coordinates)
//1.先判断当前镇的中心点是否在围栏中
if(town.location){
def flag = InsidePolygon.isInArea(town.location.lng, town.location.lat, points)
if(!flag){
println town.city + " "+town.district+" "+town.street+" 不在本身围栏中"
}
}
//2.判断是否有其它点在围栏中
for(def center : centers){
if(center._id.toString() != town._id.toString()){
def flag = InsidePolygon.isInArea(center.location.lng, center.location.lat, points)
if(flag){
def list = map.get(town.city + "#"+town.district+"#"+town.street)
if(!list){
list = []
map.put(town.city + "#"+town.district+"#"+town.street)
}
list << center.city + "#"+center.district+"#"+center.street
println center.city + " "+center.district+" "+center.street+" 在 "+town.city + " "+town.district+" "+town.street+" 围栏中"
}
}
}
}
return map
}
def findHasShapeTown(){
def q = new BasicDBObject()
q.put("type", "乡镇街道")
q.put("shape", new BasicDBObject('$exists', true))
return ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000).toList()
}
def findAllCenters(){
def q = new BasicDBObject()
q.put("type", "乡镇街道")
q.put("location", new BasicDBObject('$exists', true))
def keys = new BasicDBObject()
keys.put("location", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("street", 1)
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).toList()
}
}
@@ -1,6 +1,7 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
@@ -900,41 +901,92 @@ class TownMiddleProcess {
}
def updateTownPolineBatch(){
def i = 0
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx",{ row->
if(row['R'] && row['R']=="新增围栏"){
def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D'])
println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q)
def d = TOWNPOLYLINE_COLL.findOne(q)
if(d){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble())
def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble())
pointList << [lon, lat]
i++
if(i>=134){
def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D'])
println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q)
def d = TOWNPOLYLINE_COLL.findOne(q)
if(d){
def resultMap = [:]
def bd_resultMap = [:]
resultMap.type = "polygon"
bd_resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
def bd_rets = []
println row['N'] +" 共有:"+arrs.size()+" 个围栏"
def index = 1
for(def s : arrs){
def points = s.split(";")
def pointList = []
def bd_pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble())
def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble())
pointList << [lon, lat]
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lon, lat, 3)
def bd_lng = baidu.x
def bd_lat = baidu.y
bd_pointList << [bd_lng, bd_lat]
}
if(index<=1){
resultMap.coordinates = pointList
bd_resultMap.coordinates = bd_pointList
def udata = new BasicDBObject()
udata.put("shape", resultMap)
udata.put("operator", "高娟")
udata.put("m_date", new Date())
udata.put("bd_shape",bd_resultMap)
def updateQuery = new BasicDBObject("name",row['N'])
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}else{
def dbo = DEVELOPMENT_ZONE_43_COLL.findOne(new BasicDBObject("name",row['N']))
def obj = new BasicDBObject()
dbo.each {k,v->
if(k!='_id'){
obj.put(k,v)
}
}
resultMap.coordinates = pointList
bd_resultMap.coordinates = bd_pointList
obj.put("shape", resultMap)
obj.put("operator", "高娟")
obj.put("m_date", new Date())
obj.put("bd_shape",bd_resultMap)
obj.put("create_time", new Date())
obj.put("block", dbo.name+"-地块"+index)
DEVELOPMENT_ZONE_43_COLL.save(obj)
}
// rets << pointList
// bd_rets << bd_pointList
index++
}
rets << pointList
// if(rets.size()>1) {
// resultMap.type = "MultiPolygon"
// bd_resultMap.type = "MultiPolygon"
// }
// println d.city+" "+d.region+" "+d.town+" "+rets.size()
row['S'] = "已完成"
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+d.town+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def updateQuery = new BasicDBObject("name",row['N'])
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
row['S'] = "ÒÑÍê³É"
}
}
})
}
@@ -47,7 +47,7 @@ class DistrictFieldAddBatch extends BaseBatch{
//"alias":"alias",
//ownership_type:权属类型 为商品房、经适房、动迁房等
//house_property 产权性质:国有产权 "house_property":"网站没有",
def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
def fieldsMap = ["lng":"location","estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households",
"car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure",
"build_type":"build_type", "land_property":"ownership_type","live_date":"live_date",
@@ -115,7 +115,7 @@ class DistrictFieldAddBatch extends BaseBatch{
println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%"
def precentMap = fieldUpdate(totalCount, updateRets)
println c +" "+precentMap.toString()
//updateDB(c, updateRets)
updateDB(c, updateRets)
println "------------------------------------------------------------------------------"
def pre1 = 0
def pre2 = 0
@@ -212,6 +212,11 @@ class DistrictFieldAddBatch extends BaseBatch{
def rets = []
rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"]
for(def matchDist : matchRets){
// if(!["智富中心","龙山新寓","天怡瀚园","远盛凤林府","京口路小区","梦溪路小区","浩创优尚","溪山丽苑","旭园"].contains(matchDist.name)){
// continue
// }
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
if(!dist){
println c+" "+matchDist.match_id+" 不存在"
@@ -306,6 +311,15 @@ class DistrictFieldAddBatch extends BaseBatch{
if(newVal){
newValues.put(seaweedfield, newVal)
}
if(seaweedfield == "lng"){
def locationMap = FiledStrand.processLngAndLat(esVal)
if(locationMap){
newValues.put("lat", locationMap.lat)
newValues.put("lng", locationMap.lng)
newValues.put("lat2", locationMap.lat2)
newValues.put("lng2", locationMap.lng2)
}
}
}
}
}
@@ -0,0 +1,56 @@
package com.fangjia.script.gj.district
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
/**
* 临时移除 相同名称即在正常状态下有,在删除状态下也有的小区
*/
class RemoveDistrict {
public static void main(String[] args) {
def task = new RemoveDistrict()
task.run()
}
def run(){
def citys = CommonUtils.findAllCity()
def cat = "shop"
def keys = []
citys.each {c->
def districts = CommonDatautils.findAllDistricts(c, cat, ["city", "region", "name", "status"])
districts.each {it->
def key = it.city +"##"+it.region+"##"+it.name
if(!keys.contains(key)){
keys << key
}else{
println it.toString()+" 删除"
DbUtils.seaweed.remove(new BasicDBObject("_id",it._id))
}
// def list = CommonDatautils.findAllStatusDistrict(it.city, it.region, it.name, cat, ["city", "region", "name", "status"])
// if(list && list.size()>1){
// println it.toString() + " "+list.size()
// for(def d : list){
//// if(d.status.toString()=="0"){
//// println d.toString()+" 保留"
//// }else if(d.status.toString()=="1"){
//// println d.toString()+" 删除"
//// DbUtils.seaweed.remove(new BasicDBObject("_id",d._id))
//// }
//
// if(d.status.toString()=="1"){
// println d.toString()+" 保留"
// }else if(d.status.toString()!="1"){
// println d.toString()+" 删除"
// DbUtils.seaweed.remove(new BasicDBObject("_id",d._id))
// }
// }
//
// println "................................................................................................"
// }
}
}
}
}
@@ -0,0 +1,146 @@
package com.fangjia.script.gj.images
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 更新小区图片
*/
@BatchOwner(Watcher.GaoJuan)
class AddDistImagesBatch extends BaseBatch{
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static final DBCollection seaweed_fs_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_images", "mongo.db.fangjia.com")
static main(args) {
runBatch(new AddDistImagesBatch(), args);
}
@Override
public void process(Object args) {
// def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
def citys = CommonUtils.findAllCity()
citys.each {city->
def communitys = findAllEsData2(city, mdate)
println "抓取城市: "+city +" 第三方小区有图片数:"+communitys.size()
//第三方匹配小区
matchCommunitys(communitys)
}
}
def findAllEsData2(city, mdate){
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "安居客小程序"))
.must(date_range).must(date_range)
def rets = []
def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
if(datas) {
for (int k = 0; k < datas.size(); k++) {
def d = datas[k];
if(d.house_type_images_2 || d.outdoor_images) {
def data = new BasicDBObject()
data.put("city", d.city)
data.put("region", d.region)
data.put("name", d.name)
data.put("id", d.id)
data.put("out_images", d.outdoor_images)
data.put("house_type_images", d.house_type_images_2)
rets << data
}
}
}
return rets;
}
def findAllEsData(city, mdate){
def names = new HashSet()
def rets = []
ElasticUtils.executeSqlQuery("select * from community where source='安居客小程序' and city='${city}' and m_date>='${mdate}' order by m_date", { d->
if(d.house_type_images_2 || d.outdoor_images){
def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type
if(!names.contains(s)){
names << s
def data = new BasicDBObject()
data.put("city", d.city)
data.put("region", d.region)
data.put("name", d.name)
data.put("out_images", d.outdoor_images)
data.put("house_type_images", d.house_type_images_2)
rets << data
}
}
})
return rets;
}
def matchCommunitys(communitys){
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(communitys.size())
def rets = new CopyOnWriteArrayList()
def total = communitys.size()
for(def i=0;i<communitys.size();i++){
final def d = communitys[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
addCount.addAndGet(1)
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
def obj = thirdSeaweedMappingColl.findOne(query)
if(obj) {
def vo = new BasicDBObject()
vo.put("city", obj.match_city)
vo.put("district", obj.match_district)
vo.put("name", obj.match_name)
if(d.out_images){
vo.put("out_pics", d.out_images)
}
if(d.house_type_images){
vo.put("floorplan_pics", d.house_type_images)
}
vo.put("m_date", new Date())
rets.add(vo)
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
println addCount.get()+"/"+total +" [匹配库中存在] "
}else{
println addCount.get()+"/"+total +" [没找到相应的匹配数据] "+d._id.toString()
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
return rets
}
}
@@ -25,6 +25,10 @@ class AddPriceTest {
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
//235为房价网内部计算均价的库
def pgsqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
@@ -33,7 +37,7 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"
def filePath = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx"
// addGXDPrice(filePath)
addCcbPrice(filePath)
// addThirdPrice(filePath)
@@ -173,17 +177,22 @@ class AddPriceTest {
def cityId = cityCodeDataMap.get(d['V'])
def map = findAvgPrice(d['U'], cityId)
if(map){
d['F'] = map.avg_price
// d['P'] = map.calc_type
d['H'] = map.avg_price
d['I'] = map.calc_type
// d['AM'] = map.credit_value
}
// def newPriceMap = findNewFjwAvgPrice(d['U'], cityId)
// if(newPriceMap){
// d['Q'] = newPriceMap.avg_price
// }
}
})
}
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1"
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
@@ -202,6 +211,25 @@ class AddPriceTest {
}
def findNewFjwAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance235.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
def addThirdPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (!d['X']) {
@@ -62,7 +62,7 @@ class DataStandardBatch extends BaseBatch{
private static def districtMap = [:];
private static def limit = 5000;
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"]
@@ -76,6 +76,10 @@ class SynchDistrictPrice extends BaseBatch{
data.put("build_type",d.build_type)
data.put("c_date", new Date())
data.put("m_date", new Date())
// if(d.location){
// data.put("lng", d.location.x)
// data.put("lat", d.location.y)
// }
rets << data
}
})
@@ -53,7 +53,7 @@ class ImportIndexStats extends BaseBatch{
public static final def dingjiYear = 2020;
public static def dingjiYear = 2020;
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final EsDataSource dataSource = new EsDataSource()
@@ -76,22 +76,23 @@ class ImportIndexStats extends BaseBatch{
// synch()
// synchTjjIndex()//同步统计局抓取过来的指数
// Calendar calendar=Calendar.getInstance();
// def month= calendar.get(Calendar.MONTH);
// def year= calendar.get(Calendar.YEAR);
// def date = createDateString(year, month, 0, "");
// calculatePriceIndex(year, month, date);
Calendar calendar=Calendar.getInstance();
def month= calendar.get(Calendar.MONTH);
def year= calendar.get(Calendar.YEAR);
def date = createDateString(year, month, 0, "");
calculatePriceIndex(year, month, date);
// addLostIndex()
addHistoryIndex()
// addHistoryIndex()
}
def addLostIndex(){
def citys = CommonDatautils.findAllCity();
def index = 0
for(def city : citys){
index++
if(cityList.contains(city)){
println city +" Êý¾ÝÀ´Ô´ÓÚͳ¼Æ¾Ö"
println "[${index}] "+ city +" 数据来源于统计局"
continue;
}
def start = findStartOrEndIndex(city, 1)
@@ -117,21 +118,48 @@ class ImportIndexStats extends BaseBatch{
if(nextObj && nextObj.chain){
indexs << nextObj.chain
}
udata.put("chain",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
udata.put("remark","ÍÆµ¼")
if(indexs){
udata.put("chain",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
udata.put("remark","推导")
}
}
if(!dbo.yeartoyear){
def indexs = []
if(lastMonthObj && lastMonthObj.yeartoyear){
indexs << lastMonthObj.yeartoyear
}
if(nextObj && nextObj.yeartoyear){
indexs << nextObj.yeartoyear
}
if(indexs){
udata.put("yeartoyear",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
udata.put("remark","推导")
}
}
if(!dbo.fixedbase){
def indexs = []
if(lastMonthObj && lastMonthObj.fixedbase){
indexs << lastMonthObj.fixedbase
}
if(nextObj && nextObj.fixedbase){
indexs << nextObj.fixedbase
}
if(indexs){
udata.put("fixedbase",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
udata.put("remark","推导")
}
}
if(udata){
println city +" 补充"
STATS_INDEX_COLL.update(new BasicDBObject("_id", dbo._id), new BasicDBObject('$set', udata),false, true);
}
// if(!dbo.yeartoyear){
//
// }
// if(!dbo.fixedbase){
//
// }
}
}
}
println "[${index}] "+ city +" 已处理完"
}
}
@@ -154,7 +182,7 @@ class ImportIndexStats extends BaseBatch{
}
def addHistoryIndex(){
def s = "202101"
def s = "202001"
def startDate = null;
def stop = false;
def runDates = []
@@ -171,7 +199,7 @@ class ImportIndexStats extends BaseBatch{
runDates << map
s = DateUtils.addMonth(date, 1, "yyyyMM")
if(year==2024 && month==9){
if(year==2024 && month==8){
stop = true;
}
}
@@ -182,6 +210,7 @@ class ImportIndexStats extends BaseBatch{
//计算统计局没有发布的城市的价格指数
def calculatePriceIndex(def year, def month, def date){
// def citys = ["五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]//
def citys = CommonDatautils.findAllCity();
// def date = createDateString(year, month, 0, "");
def haveIndexCitys = findPriceIndexCitys(date.toInteger());
@@ -193,7 +222,7 @@ class ImportIndexStats extends BaseBatch{
def newCitys = []
for(def i=0;i<citys.size();i++) {
def city = citys[i]
if (!haveIndexCitys.contains(city)) {
if (!haveIndexCitys.contains(city) && !cityList.contains(city)) {
newCitys << city
}
}
@@ -204,6 +233,15 @@ class ImportIndexStats extends BaseBatch{
if(id){//已计算
continue;
}
if(date.toInteger()>=201601){
dingjiYear = 2015
}else if(date.toInteger()>=202101){
dingjiYear = 2020
}else if(date.toInteger()>=202601){
dingjiYear = 2025
}else if(date.toInteger()>=203101){
dingjiYear = 2030
}
fixedThreadPool.execute(new Runnable() {
void run() {
try{
@@ -220,7 +258,7 @@ class ImportIndexStats extends BaseBatch{
}
});
}
latch.await(20, TimeUnit.SECONDS);
latch.await(1, TimeUnit.MINUTES);
}
def calculating(city, year, month, date){
@@ -647,6 +685,7 @@ class ImportIndexStats extends BaseBatch{
DBObject q = new BasicDBObject();
q.put("yearmonth", date);
q.put("cat", "district")
q.put("type", "城市")
DBObject keys = new BasicDBObject();
keys.put("city", 1)
@@ -666,6 +705,7 @@ class ImportIndexStats extends BaseBatch{
q.put("yearmonth", date);
q.put("city", city);
q.put("cat", "district");
q.put("type", "城市")
DBObject keys = new BasicDBObject();
keys.put("city", 1)
@@ -684,6 +724,7 @@ class ImportIndexStats extends BaseBatch{
dbo.put("source",result.source);
dbo.put("remark",result.remark);
dbo.put("cat",result.cat);
dbo.put("type",result.type);
dbo.put("c_date",new Date());
if(result.markup){
dbo.put("chain", result.markup.toDouble());