update
This commit is contained in:
+11
@@ -95,6 +95,9 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
|
||||
// delPgData()
|
||||
// saveToPgSql2(datas)
|
||||
|
||||
//更新静态数据表
|
||||
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
}
|
||||
|
||||
def addDeleteDist(dists, ccbMaps, rets){
|
||||
@@ -339,4 +342,12 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
return rows;
|
||||
}
|
||||
|
||||
def updateVersion(def time){
|
||||
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
|
||||
for(def tableName : tables){
|
||||
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
|
||||
sqlInstance.executeUpdate(sql)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+9
-4
@@ -65,7 +65,6 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
println "curDate:"+curDate
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
@@ -79,7 +78,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def dealIds = findHistoryDealIds(city)
|
||||
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
|
||||
println "历史id加载量:"+dealIds.size()
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
@@ -110,7 +109,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成 " + rows.size()
|
||||
|
||||
delPgData()
|
||||
// delPgData()
|
||||
saveToPgSql(rows)
|
||||
|
||||
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
|
||||
@@ -131,6 +130,8 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def createData(city, def rets, cityCodeDatas, distCodeMap, source, dealIds){
|
||||
def result = Collections.synchronizedList(new ArrayList());
|
||||
dealIds = Collections.synchronizedList(dealIds);
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
|
||||
@@ -158,12 +159,14 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, ccbPriceMap)
|
||||
if(ret){
|
||||
result << ret
|
||||
dealIds << trade._id.toString()
|
||||
println totalCount.get() +" 友达:"+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
|
||||
}
|
||||
}else{
|
||||
def ret = converData(trade, cityCodeDatas, distCodeMap)
|
||||
if(ret){
|
||||
result << ret
|
||||
dealIds << trade._id.toString()
|
||||
println totalCount.get() +" 第三方:"+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
|
||||
}
|
||||
}
|
||||
@@ -178,6 +181,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
}
|
||||
println "查询完成......."
|
||||
latch.await(5, TimeUnit.MINUTES);
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -526,6 +530,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def getCityData(def city, def statMap, def ppeDistsMap){
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
|
||||
println city +" 小区量:"+dists.size()
|
||||
def tradesList = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
@@ -646,7 +651,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def findHistoryDealIds(def city){
|
||||
def sql = "SELECT gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${city}' "
|
||||
def sql = "SELECT gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${city}'"
|
||||
def ids = []
|
||||
try{
|
||||
pgsqlInstance.eachRow(sql,{
|
||||
|
||||
+8
-5
@@ -30,6 +30,9 @@ class CcbCommonUtils {
|
||||
|
||||
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection SEAWEED_MONGO_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
@@ -225,7 +228,7 @@ class CcbCommonUtils {
|
||||
def lists = []
|
||||
println dist.name +" "+total
|
||||
for(def d : dbcursor){
|
||||
def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
d.dist_id = dist._id.toString()
|
||||
@@ -432,7 +435,7 @@ class CcbCommonUtils {
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
|
||||
// q.put("name","金地玺湾")
|
||||
// q.put("name","汤堡小区")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
|
||||
@@ -443,7 +446,7 @@ class CcbCommonUtils {
|
||||
}
|
||||
}
|
||||
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000)
|
||||
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
|
||||
println "[${city}] 原始:"+dbsour.count()
|
||||
def rets = []
|
||||
|
||||
@@ -467,7 +470,7 @@ class CcbCommonUtils {
|
||||
public static def findAllDistrictByCityAndRegion(def city, def region, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible",0)
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
q.put("region", region)
|
||||
@@ -482,7 +485,7 @@ class CcbCommonUtils {
|
||||
}
|
||||
}
|
||||
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000)
|
||||
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
|
||||
println "原始:"+dbsour.count()
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
|
||||
+2
-10
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-10-27", "2023-11-18")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-10-27", "2023-11-25")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
@@ -84,8 +84,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
saveToPgSql(rows)
|
||||
//
|
||||
//更新静态数据表
|
||||
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -244,14 +243,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
def updateVersion(def time){
|
||||
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
|
||||
for(def tableName : tables){
|
||||
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
|
||||
pgsqlInstance.executeUpdate(sql)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+49
-34
@@ -1287,51 +1287,49 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
//}
|
||||
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\9.25人工指导价.xlsx"
|
||||
|
||||
//
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\指导价(保亭-文昌-詹州).xlsx"
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['A']){
|
||||
//// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
//// if(region){
|
||||
//// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
//// if(dist){
|
||||
//// d['G'] = dist.region
|
||||
//// d['H'] = dist._id.toString()
|
||||
//// }
|
||||
//// }
|
||||
// if(!d['K']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['E'])
|
||||
//// if(d['A']){
|
||||
////// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
////// if(region){
|
||||
////// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
////// if(dist){
|
||||
////// d['G'] = dist.region
|
||||
////// d['H'] = dist._id.toString()
|
||||
////// }
|
||||
////// }
|
||||
// if(!d['G']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// d['K'] = dist.region
|
||||
//// d['I'] = dist.name
|
||||
//// d['J'] = dist.status
|
||||
//// def val = d['F'].toString().toDouble().toInteger() + RandomUtil.random(-100,500)
|
||||
//// d['K'] = val.toInteger()
|
||||
// d['G'] = dist.region
|
||||
// d['H'] = dist.name
|
||||
// d['I'] = dist.status
|
||||
// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
|
||||
// d['J'] = val.toInteger()
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//// }
|
||||
//},0)
|
||||
|
||||
|
||||
|
||||
//def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20231124--批量修正价格.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
|
||||
// if(count && count==1){
|
||||
// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
|
||||
// d['E'] = dist.city
|
||||
// }
|
||||
//
|
||||
//// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
//// if(dist){
|
||||
//// d['B'] = dist.region
|
||||
//// d['C'] = dist.name
|
||||
//// d['J'] = dist.status
|
||||
//// }
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
d['C'] = dist.region
|
||||
}
|
||||
//// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
//// d['F'] = newRegion
|
||||
//},0)
|
||||
},0)
|
||||
|
||||
//ExcelUtils.modify(fileName, {d->
|
||||
// if(d['A']){
|
||||
@@ -1415,11 +1413,28 @@ def findSampleRate(pgsqlInstance, cityName, yearmonth){
|
||||
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\23号以后新增人工指导价.xlsx", {d->
|
||||
def obj = CommonDatautils.findDistrict(d['A'],d['B'],d['C'], "district")
|
||||
d['H'] = obj._id.toString()
|
||||
|
||||
})
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\房源\\仙桃.xlsx", {d->
|
||||
// def dist = CommonDatautils.findDistrict(d['A'],d['B'],d['C'], "district")
|
||||
// def flag = null;
|
||||
// if(!dist){
|
||||
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'],d['B'],d['C'],"district")
|
||||
// flag = "映射别名匹配"
|
||||
// if(!dist) {
|
||||
// def rets = CommonDatautils.findDistrictByAlias(d['A'],d['B'],d['C'], "district")
|
||||
// if(rets && rets.size()==1){
|
||||
// flag = "别名匹配"
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// if(dist){
|
||||
// d['D'] = dist.status
|
||||
// d['E'] = dist.estate_type2
|
||||
// d['F'] = flag
|
||||
// d['G'] = dist.name
|
||||
// }
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
|
||||
|
||||
+71
-6
@@ -93,6 +93,10 @@ class CommonDatautils {
|
||||
|
||||
public static final DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
|
||||
public static final def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
public static final DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
public static final DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
|
||||
|
||||
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
@@ -269,7 +273,7 @@ class CommonDatautils {
|
||||
}
|
||||
}
|
||||
|
||||
DSCursor cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
|
||||
def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
|
||||
List<DBObject> rets = []
|
||||
for(def obj : cursor){
|
||||
rets << obj
|
||||
@@ -811,7 +815,7 @@ class CommonDatautils {
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = DbUtils.seaweed.find(q, keys).batchSize(2000)
|
||||
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
rets << d
|
||||
@@ -823,7 +827,7 @@ class CommonDatautils {
|
||||
public static def findAllDistricMaptByCity(def city, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible",0)
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
|
||||
@@ -833,7 +837,7 @@ class CommonDatautils {
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
|
||||
def retMap = [:]
|
||||
for(def d : dbsour){
|
||||
retMap.put(d._id.toString(), d)
|
||||
@@ -845,7 +849,7 @@ class CommonDatautils {
|
||||
public static def findAllDistrictByCity(def city, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible",0)
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","万达华府一期")
|
||||
@@ -856,7 +860,7 @@ class CommonDatautils {
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
rets << d
|
||||
@@ -4233,5 +4237,66 @@ class CommonDatautils {
|
||||
return townMap
|
||||
}
|
||||
|
||||
static def queryFullNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = administrative_division_100_CONN.find(q)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def fjwcity = null
|
||||
if(d.level == "城市"){
|
||||
fjwcity = RegionData.formatCity(d.city)
|
||||
if(!fjwcity){
|
||||
if(d.city == "市辖区"){
|
||||
fjwcity = RegionData.formatCity(d.province)
|
||||
if(fjwcity){
|
||||
d.city = d.province
|
||||
}
|
||||
}
|
||||
}
|
||||
}else if(d.level == "区域"){
|
||||
if(d.administer == "省直辖县级行政区划"){
|
||||
fjwcity = RegionData.formatCity(d.city)
|
||||
if(fjwcity){
|
||||
d.city = d.district
|
||||
}
|
||||
}
|
||||
}
|
||||
if(fjwcity){
|
||||
retMap.put(fjwcity, d)
|
||||
}else{
|
||||
println d.toString() +" 格式化不成功!"
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def queryDistNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("level","区域")
|
||||
def dbcursor = administrative_division_100_CONN.find(q)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.city == "省直辖县级行政区划"){
|
||||
continue
|
||||
}else if(d.city == "市辖区"){
|
||||
def fjwcity = RegionData.formatCity(d.province)
|
||||
if(fjwcity){
|
||||
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
|
||||
retMap.put(fjwcity+"#"+fjwdist, d)
|
||||
}
|
||||
}else{
|
||||
def fjwcity = RegionData.formatCity(d.city)
|
||||
if(fjwcity){
|
||||
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
|
||||
retMap.put(fjwcity+"#"+fjwdist, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -42,7 +42,7 @@ class ApiTest {
|
||||
}
|
||||
|
||||
def addrStructure(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
ExcelUtils.modify(path,{d->
|
||||
if(d['B']){
|
||||
JSONObject jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']);
|
||||
if(jsonObject) {
|
||||
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class PostCodeSynchBatch extends BaseBatch{
|
||||
|
||||
public static final DBCollection post_code_coll = MongoUtils.getCollectionByName("crawl", "post_code", "192.168.0.206", 27020)
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection POST_CODE_CONN = MongoUtils.getCollectionByName2("fangjia", "postcode", MONGO100_URL)
|
||||
|
||||
def fullCityMap = null
|
||||
def mcaDistMap = null
|
||||
|
||||
static main(args) {
|
||||
runBatch(new PostCodeSynchBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
mcaDistMap = CommonDatautils.queryDistNameCity()
|
||||
|
||||
def datas = findAllDatas();
|
||||
for(def d : datas){
|
||||
def map = [:]
|
||||
map._id = d._id
|
||||
def area_code = null
|
||||
map.province = d.province
|
||||
|
||||
|
||||
def short_city = RegionData.formatCity(d.city)
|
||||
if(short_city){
|
||||
def data = fullCityMap.get(short_city)
|
||||
if(data){
|
||||
map.city = data.city
|
||||
map.city_code = data.mca_city_code
|
||||
map.province_code = data.mca_province_code
|
||||
}
|
||||
|
||||
def region = d.district.replace(d.province, "").replace(d.city, "")
|
||||
def short_district = RegionData.formatRegion(short_city, region)
|
||||
|
||||
def distData = mcaDistMap.get(short_city+"#"+short_district)
|
||||
if(distData){
|
||||
map.district = distData.district
|
||||
map.district_code = distData.mca_district_code
|
||||
}else{
|
||||
println d.city +" "+d.district+" 民政部中没有找到相应的区域全称"
|
||||
}
|
||||
|
||||
}else{
|
||||
println d.city +" 没有标准化出来"
|
||||
}
|
||||
map.city_zip_code = d.city_code
|
||||
map.district_zip_code = d.district_code
|
||||
map.address = d.addr
|
||||
map.addr_zip_code = d.addr_code
|
||||
|
||||
updateData(map)
|
||||
}
|
||||
}
|
||||
|
||||
def updateData(map){
|
||||
def udata = new BasicDBObject()
|
||||
map.each{k,v->
|
||||
if(k!="_id"){
|
||||
udata.put(k,v)
|
||||
}
|
||||
}
|
||||
POST_CODE_CONN.update(new BasicDBObject('_id', map._id), new BasicDBObject('$set', udata), true, false)
|
||||
}
|
||||
|
||||
|
||||
def findAllDatas(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = post_code_coll.find(q)
|
||||
return dbcursor.toList();
|
||||
}
|
||||
}
|
||||
+60
-22
@@ -6,6 +6,7 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
@@ -38,6 +39,8 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
|
||||
|
||||
|
||||
def fullCityMap = null
|
||||
def mcaDistMap = null
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SeaweedDataSynch(), args);
|
||||
@@ -45,6 +48,8 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
|
||||
run()
|
||||
println "全国district, shop, office同步完成"
|
||||
//民政部编码上传
|
||||
@@ -56,8 +61,12 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
|
||||
def run(){
|
||||
|
||||
fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
mcaDistMap = CommonDatautils.queryDistNameCity()
|
||||
|
||||
priceMap = findAllDistrictSellPrice()
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
index++
|
||||
@@ -116,9 +125,33 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
def createData(dist){
|
||||
def map = [:]
|
||||
map._id = dist._id
|
||||
map.province = provinceMap.get(dist.city)
|
||||
map.city = dist.city
|
||||
map.region = dist.region
|
||||
def area_code = null
|
||||
def province = provinceMap.get(dist.city)
|
||||
if(province){
|
||||
map.province = ProvinceData.getFullProvince(province)
|
||||
}
|
||||
map.fangjia_province = province
|
||||
|
||||
def data = fullCityMap.get(dist.city)
|
||||
if(data){
|
||||
map.city = data.city
|
||||
area_code = data.mca_city_code
|
||||
}
|
||||
map.fangjia_city = dist.city
|
||||
|
||||
def distData = mcaDistMap.get(dist.city+"#"+dist.region)
|
||||
if(distData){
|
||||
map.region = distData.district
|
||||
area_code = distData.mca_district_code
|
||||
}else{
|
||||
// map.region = dist.region
|
||||
println dist.city +" "+dist.region+" 民政部中没有找到相应的区域全称"
|
||||
}
|
||||
map.area_code = area_code
|
||||
|
||||
map.fangjia_region = dist.region
|
||||
|
||||
|
||||
map.name = dist.name
|
||||
map.address = dist.address
|
||||
map.block = dist.block
|
||||
@@ -150,22 +183,24 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
map.location = loc
|
||||
}
|
||||
|
||||
//高德转百度
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng)
|
||||
location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat)
|
||||
map.bd_location = location
|
||||
}
|
||||
if(map.location){
|
||||
//高德转百度
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng)
|
||||
location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat)
|
||||
map.bd_location = location
|
||||
}
|
||||
|
||||
//高德转WGS84
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng)
|
||||
gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat)
|
||||
map.gps_location = gpslocation
|
||||
//高德转WGS84
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng)
|
||||
gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat)
|
||||
map.gps_location = gpslocation
|
||||
}
|
||||
}
|
||||
|
||||
map.build_type = dist.build_type
|
||||
@@ -209,10 +244,13 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
if(dist.main_type_area && dist.main_type_area.toString().isNumber()){
|
||||
map.main_house_area = DataHandleUtils.getDoubleRound(2, dist.main_type_area.toString().toFloat())
|
||||
}
|
||||
def dataMap = priceMap.get(dist._id.toString())
|
||||
if(dataMap){
|
||||
map.avg_price = dataMap.avg_price
|
||||
if(priceMap){
|
||||
def dataMap = priceMap.get(dist._id.toString())
|
||||
if(dataMap){
|
||||
map.avg_price = dataMap.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
// map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||
return map
|
||||
}
|
||||
|
||||
@@ -93,6 +93,10 @@ public class ProvinceData {
|
||||
put("宁夏", "宁夏回族自治区");
|
||||
put("海南", "海南省");
|
||||
put("台湾", "台湾省");
|
||||
put("上海", "上海市");
|
||||
put("北京", "北京市");
|
||||
put("天津", "天津市");
|
||||
put("重庆", "重庆市");
|
||||
}};
|
||||
|
||||
static {
|
||||
|
||||
Reference in New Issue
Block a user