This commit is contained in:
2023-11-24 17:21:36 +08:00
parent bdfa62a42f
commit 40e738ccf8
10 changed files with 302 additions and 82 deletions
@@ -95,6 +95,9 @@ class BigdataCommunityBatch extends BaseBatch{
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
// delPgData()
// saveToPgSql2(datas)
//更新静态数据表
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
def addDeleteDist(dists, ccbMaps, rets){
@@ -339,4 +342,12 @@ class BigdataCommunityBatch extends BaseBatch{
return rows;
}
def updateVersion(def time){
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
for(def tableName : tables){
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
sqlInstance.executeUpdate(sql)
}
}
}
@@ -65,7 +65,6 @@ class BigdataDealBatch extends BaseBatch{
println "curDate:"+curDate
def allCitys = CommonDatautils.findAllCity()
for(def city : allCitys){
addDealPriceMatch(city);
}
@@ -79,7 +78,7 @@ class BigdataDealBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def dealIds = findHistoryDealIds(city)
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
println "历史id加载量:"+dealIds.size()
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
@@ -110,7 +109,7 @@ class BigdataDealBatch extends BaseBatch{
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size()
delPgData()
// delPgData()
saveToPgSql(rows)
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
@@ -131,6 +130,8 @@ class BigdataDealBatch extends BaseBatch{
def createData(city, def rets, cityCodeDatas, distCodeMap, source, dealIds){
def result = Collections.synchronizedList(new ArrayList());
dealIds = Collections.synchronizedList(dealIds);
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
@@ -158,12 +159,14 @@ class BigdataDealBatch extends BaseBatch{
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, ccbPriceMap)
if(ret){
result << ret
dealIds << trade._id.toString()
println totalCount.get() +" 友达:"+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
}
}else{
def ret = converData(trade, cityCodeDatas, distCodeMap)
if(ret){
result << ret
dealIds << trade._id.toString()
println totalCount.get() +" 第三方:"+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
}
}
@@ -178,6 +181,7 @@ class BigdataDealBatch extends BaseBatch{
}
println "查询完成......."
latch.await(5, TimeUnit.MINUTES);
return result
}
@@ -526,6 +530,7 @@ class BigdataDealBatch extends BaseBatch{
def getCityData(def city, def statMap, def ppeDistsMap){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
println city +" 小区量:"+dists.size()
def tradesList = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
@@ -646,7 +651,7 @@ class BigdataDealBatch extends BaseBatch{
}
def findHistoryDealIds(def city){
def sql = "SELECT gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${city}' "
def sql = "SELECT gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${city}'"
def ids = []
try{
pgsqlInstance.eachRow(sql,{
@@ -30,6 +30,9 @@ class CcbCommonUtils {
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
public static final DBCollection SEAWEED_MONGO_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
@@ -225,7 +228,7 @@ class CcbCommonUtils {
def lists = []
println dist.name +" "+total
for(def d : dbcursor){
def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
@@ -432,7 +435,7 @@ class CcbCommonUtils {
q.put("cat", cat);
q.put("city", city);
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
// q.put("name","金地玺湾")
// q.put("name","汤堡小区")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -443,7 +446,7 @@ class CcbCommonUtils {
}
}
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000)
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
println "[${city}] 原始:"+dbsour.count()
def rets = []
@@ -467,7 +470,7 @@ class CcbCommonUtils {
public static def findAllDistrictByCityAndRegion(def city, def region, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
q.put("region", region)
@@ -482,7 +485,7 @@ class CcbCommonUtils {
}
}
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000)
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
println "原始:"+dbsour.count()
def rets = []
for(def d : dbsour){
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-10-27", "2023-11-18")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-10-27", "2023-11-25")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -84,8 +84,7 @@ class GuiDingPriceBatch extends BaseBatch{
saveToPgSql(rows)
//
//更新静态数据表
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
@@ -244,14 +243,7 @@ class GuiDingPriceBatch extends BaseBatch{
println "end: "+new Date()
}
def updateVersion(def time){
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
for(def tableName : tables){
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
pgsqlInstance.executeUpdate(sql)
}
}
@@ -1287,51 +1287,49 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
//}
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\9.25人工指导价.xlsx"
//
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\指导价(保亭-文昌-詹州).xlsx"
//
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['A']){
//// def region = RegionData.formatRegion(d['B'], d['C'])
//// if(region){
//// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
//// if(dist){
//// d['G'] = dist.region
//// d['H'] = dist._id.toString()
//// }
//// }
// if(!d['K']){
// def dist = CommonDatautils.findDistrictById(d['E'])
//// if(d['A']){
////// def region = RegionData.formatRegion(d['B'], d['C'])
////// if(region){
////// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
////// if(dist){
////// d['G'] = dist.region
////// d['H'] = dist._id.toString()
////// }
////// }
// if(!d['G']){
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// d['K'] = dist.region
//// d['I'] = dist.name
//// d['J'] = dist.status
//// def val = d['F'].toString().toDouble().toInteger() + RandomUtil.random(-100,500)
//// d['K'] = val.toInteger()
// d['G'] = dist.region
// d['H'] = dist.name
// d['I'] = dist.status
// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
// d['J'] = val.toInteger()
// }
// }
// }
//// }
//},0)
//def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20231124--批量修正价格.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
// if(count && count==1){
// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
// d['E'] = dist.city
// }
//
//// def dist = CommonDatautils.findDistrictById(d['A'])
//// if(dist){
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['J'] = dist.status
//// }
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
d['C'] = dist.region
}
//// def newRegion = RegionData.formatRegion(d['A'], d['B'])
//// d['F'] = newRegion
//},0)
},0)
//ExcelUtils.modify(fileName, {d->
// if(d['A']){
@@ -1415,11 +1413,28 @@ def findSampleRate(pgsqlInstance, cityName, yearmonth){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\23号以后新增人工指导价.xlsx", {d->
def obj = CommonDatautils.findDistrict(d['A'],d['B'],d['C'], "district")
d['H'] = obj._id.toString()
})
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\房源\\仙桃.xlsx", {d->
// def dist = CommonDatautils.findDistrict(d['A'],d['B'],d['C'], "district")
// def flag = null;
// if(!dist){
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'],d['B'],d['C'],"district")
// flag = "映射别名匹配"
// if(!dist) {
// def rets = CommonDatautils.findDistrictByAlias(d['A'],d['B'],d['C'], "district")
// if(rets && rets.size()==1){
// flag = "别名匹配"
// dist = rets[0]
// }
// }
// }
// if(dist){
// d['D'] = dist.status
// d['E'] = dist.estate_type2
// d['F'] = flag
// d['G'] = dist.name
// }
//
//})
@@ -93,6 +93,10 @@ class CommonDatautils {
public static final DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
public static final def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
public static final DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
public static final DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
@@ -269,7 +273,7 @@ class CommonDatautils {
}
}
DSCursor cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
List<DBObject> rets = []
for(def obj : cursor){
rets << obj
@@ -811,7 +815,7 @@ class CommonDatautils {
keys.put(field, 1);
}
}
def dbsour = DbUtils.seaweed.find(q, keys).batchSize(2000)
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
rets << d
@@ -823,7 +827,7 @@ class CommonDatautils {
public static def findAllDistricMaptByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
@@ -833,7 +837,7 @@ class CommonDatautils {
keys.put(field, 1);
}
}
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
def retMap = [:]
for(def d : dbsour){
retMap.put(d._id.toString(), d)
@@ -845,7 +849,7 @@ class CommonDatautils {
public static def findAllDistrictByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","万达华府一期")
@@ -856,7 +860,7 @@ class CommonDatautils {
keys.put(field, 1);
}
}
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
rets << d
@@ -4233,5 +4237,66 @@ class CommonDatautils {
return townMap
}
static def queryFullNameCity(){
def q = new BasicDBObject()
def dbcursor = administrative_division_100_CONN.find(q)
def retMap = [:]
for(def d : dbcursor){
def fjwcity = null
if(d.level == "城市"){
fjwcity = RegionData.formatCity(d.city)
if(!fjwcity){
if(d.city == "市辖区"){
fjwcity = RegionData.formatCity(d.province)
if(fjwcity){
d.city = d.province
}
}
}
}else if(d.level == "区域"){
if(d.administer == "省直辖县级行政区划"){
fjwcity = RegionData.formatCity(d.city)
if(fjwcity){
d.city = d.district
}
}
}
if(fjwcity){
retMap.put(fjwcity, d)
}else{
println d.toString() +" 格式化不成功!"
}
}
return retMap;
}
static def queryDistNameCity(){
def q = new BasicDBObject()
q.put("level","区域")
def dbcursor = administrative_division_100_CONN.find(q)
def retMap = [:]
for(def d : dbcursor){
if(d.city == "省直辖县级行政区划"){
continue
}else if(d.city == "市辖区"){
def fjwcity = RegionData.formatCity(d.province)
if(fjwcity){
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
retMap.put(fjwcity+"#"+fjwdist, d)
}
}else{
def fjwcity = RegionData.formatCity(d.city)
if(fjwcity){
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
retMap.put(fjwcity+"#"+fjwdist, d)
}
}
}
return retMap;
}
}
@@ -42,7 +42,7 @@ class ApiTest {
}
def addrStructure(path){
ExcelUtils.multiThreadingModify(path,{d->
ExcelUtils.modify(path,{d->
if(d['B']){
JSONObject jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']);
if(jsonObject) {
@@ -0,0 +1,87 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class PostCodeSynchBatch extends BaseBatch{
public static final DBCollection post_code_coll = MongoUtils.getCollectionByName("crawl", "post_code", "192.168.0.206", 27020)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection POST_CODE_CONN = MongoUtils.getCollectionByName2("fangjia", "postcode", MONGO100_URL)
def fullCityMap = null
def mcaDistMap = null
static main(args) {
runBatch(new PostCodeSynchBatch(), args);
}
@Override
public void process(Object args) {
fullCityMap = CommonDatautils.queryFullNameCity()
mcaDistMap = CommonDatautils.queryDistNameCity()
def datas = findAllDatas();
for(def d : datas){
def map = [:]
map._id = d._id
def area_code = null
map.province = d.province
def short_city = RegionData.formatCity(d.city)
if(short_city){
def data = fullCityMap.get(short_city)
if(data){
map.city = data.city
map.city_code = data.mca_city_code
map.province_code = data.mca_province_code
}
def region = d.district.replace(d.province, "").replace(d.city, "")
def short_district = RegionData.formatRegion(short_city, region)
def distData = mcaDistMap.get(short_city+"#"+short_district)
if(distData){
map.district = distData.district
map.district_code = distData.mca_district_code
}else{
println d.city +" "+d.district+" 民政部中没有找到相应的区域全称"
}
}else{
println d.city +" 没有标准化出来"
}
map.city_zip_code = d.city_code
map.district_zip_code = d.district_code
map.address = d.addr
map.addr_zip_code = d.addr_code
updateData(map)
}
}
def updateData(map){
def udata = new BasicDBObject()
map.each{k,v->
if(k!="_id"){
udata.put(k,v)
}
}
POST_CODE_CONN.update(new BasicDBObject('_id', map._id), new BasicDBObject('$set', udata), true, false)
}
def findAllDatas(){
def q = new BasicDBObject()
def dbcursor = post_code_coll.find(q)
return dbcursor.toList();
}
}
@@ -6,6 +6,7 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
@@ -38,6 +39,8 @@ class SeaweedDataSynch extends BaseBatch{
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
def fullCityMap = null
def mcaDistMap = null
static main(args) {
runBatch(new SeaweedDataSynch(), args);
@@ -45,6 +48,8 @@ class SeaweedDataSynch extends BaseBatch{
@Override
public void process(Object args) {
run()
println "全国district, shop, office同步完成"
//民政部编码上传
@@ -56,8 +61,12 @@ class SeaweedDataSynch extends BaseBatch{
def run(){
fullCityMap = CommonDatautils.queryFullNameCity()
mcaDistMap = CommonDatautils.queryDistNameCity()
priceMap = findAllDistrictSellPrice()
def citys = CommonDatautils.findAllCity()
def citys = CommonDatautils.findAllCity().reverse()
def index = 0
for(def c : citys){
index++
@@ -116,9 +125,33 @@ class SeaweedDataSynch extends BaseBatch{
def createData(dist){
def map = [:]
map._id = dist._id
map.province = provinceMap.get(dist.city)
map.city = dist.city
map.region = dist.region
def area_code = null
def province = provinceMap.get(dist.city)
if(province){
map.province = ProvinceData.getFullProvince(province)
}
map.fangjia_province = province
def data = fullCityMap.get(dist.city)
if(data){
map.city = data.city
area_code = data.mca_city_code
}
map.fangjia_city = dist.city
def distData = mcaDistMap.get(dist.city+"#"+dist.region)
if(distData){
map.region = distData.district
area_code = distData.mca_district_code
}else{
// map.region = dist.region
println dist.city +" "+dist.region+" 民政部中没有找到相应的区域全称"
}
map.area_code = area_code
map.fangjia_region = dist.region
map.name = dist.name
map.address = dist.address
map.block = dist.block
@@ -150,22 +183,24 @@ class SeaweedDataSynch extends BaseBatch{
map.location = loc
}
//高德转百度
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
if(baiduMap){
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng)
location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat)
map.bd_location = location
}
if(map.location){
//高德转百度
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
if(baiduMap){
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng)
location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat)
map.bd_location = location
}
//高德转WGS84
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng)
gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat)
map.gps_location = gpslocation
//高德转WGS84
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng)
gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat)
map.gps_location = gpslocation
}
}
map.build_type = dist.build_type
@@ -209,10 +244,13 @@ class SeaweedDataSynch extends BaseBatch{
if(dist.main_type_area && dist.main_type_area.toString().isNumber()){
map.main_house_area = DataHandleUtils.getDoubleRound(2, dist.main_type_area.toString().toFloat())
}
def dataMap = priceMap.get(dist._id.toString())
if(dataMap){
map.avg_price = dataMap.avg_price
if(priceMap){
def dataMap = priceMap.get(dist._id.toString())
if(dataMap){
map.avg_price = dataMap.avg_price
}
}
// map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
return map
}
@@ -93,6 +93,10 @@ public class ProvinceData {
put("宁夏", "宁夏回族自治区");
put("海南", "海南省");
put("台湾", "台湾省");
put("上海", "上海市");
put("北京", "北京市");
put("天津", "天津市");
put("重庆", "重庆市");
}};
static {