diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index c5e18a29f..84782eb55 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -95,6 +95,9 @@ class BigdataCommunityBatch extends BaseBatch{ // def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat") // delPgData() // saveToPgSql2(datas) + + //更新静态数据表 + updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") } def addDeleteDist(dists, ccbMaps, rets){ @@ -339,4 +342,12 @@ class BigdataCommunityBatch extends BaseBatch{ return rows; } + def updateVersion(def time){ + def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"] + for(def tableName : tables){ + def sql = "update base."+tableName+" set p9_start_date='"+time+"'" + sqlInstance.executeUpdate(sql) + } + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 05a4bbf0a..4d473e36a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -65,7 +65,6 @@ class BigdataDealBatch extends BaseBatch{ println "curDate:"+curDate def allCitys = CommonDatautils.findAllCity() - for(def city : allCitys){ addDealPriceMatch(city); } @@ -79,7 +78,7 @@ class BigdataDealBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def ppeDists = [:] - def dealIds = findHistoryDealIds(city) + def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city)) println "历史id加载量:"+dealIds.size() def statMap = [:]//getDistrictStatByCity(city, weekend) if(city=="上海"){ @@ -110,7 +109,7 @@ class BigdataDealBatch extends BaseBatch{ FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); println "txt 生成 " + rows.size() - delPgData() +// delPgData() saveToPgSql(rows) // def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat") @@ -131,6 +130,8 @@ class BigdataDealBatch extends BaseBatch{ def createData(city, def rets, cityCodeDatas, distCodeMap, source, dealIds){ def result = Collections.synchronizedList(new ArrayList()); + dealIds = Collections.synchronizedList(dealIds); + def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(rets.size()) @@ -158,12 +159,14 @@ class BigdataDealBatch extends BaseBatch{ def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, ccbPriceMap) if(ret){ result << ret + dealIds << trade._id.toString() println totalCount.get() +" 友达:"+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString() } }else{ def ret = converData(trade, cityCodeDatas, distCodeMap) if(ret){ result << ret + dealIds << trade._id.toString() println totalCount.get() +" 第三方:"+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString() } } @@ -178,6 +181,7 @@ class BigdataDealBatch extends BaseBatch{ } println "查询完成......." latch.await(5, TimeUnit.MINUTES); + return result } @@ -526,6 +530,7 @@ class BigdataDealBatch extends BaseBatch{ def getCityData(def city, def statMap, def ppeDistsMap){ def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"]) + println city +" 小区量:"+dists.size() def tradesList = Collections.synchronizedList(new ArrayList()); def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) @@ -646,7 +651,7 @@ class BigdataDealBatch extends BaseBatch{ } def findHistoryDealIds(def city){ - def sql = "SELECT gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${city}' " + def sql = "SELECT gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${city}'" def ids = [] try{ pgsqlInstance.eachRow(sql,{ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 3929090a1..65f84ab77 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -30,6 +30,9 @@ class CcbCommonUtils { public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com") + + public static final DBCollection SEAWEED_MONGO_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com") + public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com") @@ -225,7 +228,7 @@ class CcbCommonUtils { def lists = [] println dist.name +" "+total for(def d : dbcursor){ - def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") if(!rets.contains(key)){ rets << key d.dist_id = dist._id.toString() @@ -432,7 +435,7 @@ class CcbCommonUtils { q.put("cat", cat); q.put("city", city); // q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 -// q.put("name","金地玺湾") +// q.put("name","汤堡小区") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) @@ -443,7 +446,7 @@ class CcbCommonUtils { } } // def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) - def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000) + def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000) println "[${city}] 原始:"+dbsour.count() def rets = [] @@ -467,7 +470,7 @@ class CcbCommonUtils { public static def findAllDistrictByCityAndRegion(def city, def region, def cat, def fields){ DBObject q = new BasicDBObject(); - q.put("visible",0) + q.put("status",0) q.put("cat", cat); q.put("city", city); q.put("region", region) @@ -482,7 +485,7 @@ class CcbCommonUtils { } } // def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) - def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000) + def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000) println "原始:"+dbsour.count() def rets = [] for(def d : dbsour){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 6e94090c9..8d844fb36 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-10-27", "2023-11-18") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-10-27", "2023-11-25") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -84,8 +84,7 @@ class GuiDingPriceBatch extends BaseBatch{ saveToPgSql(rows) // - //更新静态数据表 - updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") + } @@ -244,14 +243,7 @@ class GuiDingPriceBatch extends BaseBatch{ println "end: "+new Date() } - def updateVersion(def time){ - def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"] - for(def tableName : tables){ - def sql = "update base."+tableName+" set p9_start_date='"+time+"'" - pgsqlInstance.executeUpdate(sql) - } - } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 9adec0396..61707d54d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1287,51 +1287,49 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ //} - -//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\9.25人工指导价.xlsx" - +// +//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\指导价(保亭-文昌-詹州).xlsx" +// //ExcelUtils.multiThreadingModify(fileName, {d-> -// if(d['A']){ -//// def region = RegionData.formatRegion(d['B'], d['C']) -//// if(region){ -//// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") -//// if(dist){ -//// d['G'] = dist.region -//// d['H'] = dist._id.toString() -//// } -//// } -// if(!d['K']){ -// def dist = CommonDatautils.findDistrictById(d['E']) +//// if(d['A']){ +////// def region = RegionData.formatRegion(d['B'], d['C']) +////// if(region){ +////// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") +////// if(dist){ +////// d['G'] = dist.region +////// d['H'] = dist._id.toString() +////// } +////// } +// if(!d['G']){ +// def dist = CommonDatautils.findDistrictById(d['A']) // if(dist){ -// d['K'] = dist.region -//// d['I'] = dist.name -//// d['J'] = dist.status -//// def val = d['F'].toString().toDouble().toInteger() + RandomUtil.random(-100,500) -//// d['K'] = val.toInteger() +// d['G'] = dist.region +// d['H'] = dist.name +// d['I'] = dist.status +// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200) +// d['J'] = val.toInteger() // } // } -// } +//// } //},0) -//def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx" -//ExcelUtils.multiThreadingModify(fileName, { d -> +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20231124--批量修正价格.xlsx" +ExcelUtils.multiThreadingModify(fileName, { d -> // def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district") // if(count && count==1){ // def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district") // d['E'] = dist.city // } // -//// def dist = CommonDatautils.findDistrictById(d['A']) -//// if(dist){ -//// d['B'] = dist.region -//// d['C'] = dist.name -//// d['J'] = dist.status -//// } + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + d['C'] = dist.region + } //// def newRegion = RegionData.formatRegion(d['A'], d['B']) //// d['F'] = newRegion -//},0) +},0) //ExcelUtils.modify(fileName, {d-> // if(d['A']){ @@ -1415,11 +1413,28 @@ def findSampleRate(pgsqlInstance, cityName, yearmonth){ -ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\23号以后新增人工指导价.xlsx", {d-> - def obj = CommonDatautils.findDistrict(d['A'],d['B'],d['C'], "district") - d['H'] = obj._id.toString() - -}) +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\房源\\仙桃.xlsx", {d-> +// def dist = CommonDatautils.findDistrict(d['A'],d['B'],d['C'], "district") +// def flag = null; +// if(!dist){ +// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'],d['B'],d['C'],"district") +// flag = "映射别名匹配" +// if(!dist) { +// def rets = CommonDatautils.findDistrictByAlias(d['A'],d['B'],d['C'], "district") +// if(rets && rets.size()==1){ +// flag = "别名匹配" +// dist = rets[0] +// } +// } +// } +// if(dist){ +// d['D'] = dist.status +// d['E'] = dist.estate_type2 +// d['F'] = flag +// d['G'] = dist.name +// } +// +//}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index c40d48b46..0de37e6f5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -93,6 +93,10 @@ class CommonDatautils { public static final DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline") + public static final def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + public static final DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL) + public static final DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL) + static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() @@ -269,7 +273,7 @@ class CommonDatautils { } } - DSCursor cursor = DbUtils.seaweed.find(q, keys).batchSize(2000) + def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000) List rets = [] for(def obj : cursor){ rets << obj @@ -811,7 +815,7 @@ class CommonDatautils { keys.put(field, 1); } } - def dbsour = DbUtils.seaweed.find(q, keys).batchSize(2000) + def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000) def rets = [] for(def d : dbsour){ rets << d @@ -823,7 +827,7 @@ class CommonDatautils { public static def findAllDistricMaptByCity(def city, def cat, def fields){ DBObject q = new BasicDBObject(); - q.put("visible",0) + q.put("status",0) q.put("cat", cat); q.put("city", city); @@ -833,7 +837,7 @@ class CommonDatautils { keys.put(field, 1); } } - def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) + def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000) def retMap = [:] for(def d : dbsour){ retMap.put(d._id.toString(), d) @@ -845,7 +849,7 @@ class CommonDatautils { public static def findAllDistrictByCity(def city, def cat, def fields){ DBObject q = new BasicDBObject(); - q.put("visible",0) + q.put("status",0) q.put("cat", cat); q.put("city", city); // q.put("name","万达华府一期") @@ -856,7 +860,7 @@ class CommonDatautils { keys.put(field, 1); } } - def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) + def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000) def rets = [] for(def d : dbsour){ rets << d @@ -4233,5 +4237,66 @@ class CommonDatautils { return townMap } + static def queryFullNameCity(){ + def q = new BasicDBObject() + def dbcursor = administrative_division_100_CONN.find(q) + def retMap = [:] + for(def d : dbcursor){ + def fjwcity = null + if(d.level == "城市"){ + fjwcity = RegionData.formatCity(d.city) + if(!fjwcity){ + if(d.city == "市辖区"){ + fjwcity = RegionData.formatCity(d.province) + if(fjwcity){ + d.city = d.province + } + } + } + }else if(d.level == "区域"){ + if(d.administer == "省直辖县级行政区划"){ + fjwcity = RegionData.formatCity(d.city) + if(fjwcity){ + d.city = d.district + } + } + } + if(fjwcity){ + retMap.put(fjwcity, d) + }else{ + println d.toString() +" 格式化不成功!" + } + } + return retMap; + } + + + + static def queryDistNameCity(){ + def q = new BasicDBObject() + q.put("level","区域") + def dbcursor = administrative_division_100_CONN.find(q) + def retMap = [:] + for(def d : dbcursor){ + if(d.city == "省直辖县级行政区划"){ + continue + }else if(d.city == "市辖区"){ + def fjwcity = RegionData.formatCity(d.province) + if(fjwcity){ + def fjwdist = RegionData.formatRegion(fjwcity, d.district) + retMap.put(fjwcity+"#"+fjwdist, d) + } + }else{ + def fjwcity = RegionData.formatCity(d.city) + if(fjwcity){ + def fjwdist = RegionData.formatRegion(fjwcity, d.district) + retMap.put(fjwcity+"#"+fjwdist, d) + } + } + } + return retMap; + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ApiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ApiTest.groovy index bd28d23a2..775c4fa05 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ApiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ApiTest.groovy @@ -42,7 +42,7 @@ class ApiTest { } def addrStructure(path){ - ExcelUtils.multiThreadingModify(path,{d-> + ExcelUtils.modify(path,{d-> if(d['B']){ JSONObject jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']); if(jsonObject) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PostCodeSynchBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PostCodeSynchBatch.groovy new file mode 100644 index 000000000..a67322dcb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PostCodeSynchBatch.groovy @@ -0,0 +1,87 @@ +package com.fangjia.script.gj.xiamen + +import com.fangjia.core.RegionData +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.ProvinceData +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class PostCodeSynchBatch extends BaseBatch{ + + public static final DBCollection post_code_coll = MongoUtils.getCollectionByName("crawl", "post_code", "192.168.0.206", 27020) + + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection POST_CODE_CONN = MongoUtils.getCollectionByName2("fangjia", "postcode", MONGO100_URL) + + def fullCityMap = null + def mcaDistMap = null + + static main(args) { + runBatch(new PostCodeSynchBatch(), args); + } + + @Override + public void process(Object args) { + + fullCityMap = CommonDatautils.queryFullNameCity() + mcaDistMap = CommonDatautils.queryDistNameCity() + + def datas = findAllDatas(); + for(def d : datas){ + def map = [:] + map._id = d._id + def area_code = null + map.province = d.province + + + def short_city = RegionData.formatCity(d.city) + if(short_city){ + def data = fullCityMap.get(short_city) + if(data){ + map.city = data.city + map.city_code = data.mca_city_code + map.province_code = data.mca_province_code + } + + def region = d.district.replace(d.province, "").replace(d.city, "") + def short_district = RegionData.formatRegion(short_city, region) + + def distData = mcaDistMap.get(short_city+"#"+short_district) + if(distData){ + map.district = distData.district + map.district_code = distData.mca_district_code + }else{ + println d.city +" "+d.district+" 民政部中没有找到相应的区域全称" + } + + }else{ + println d.city +" 没有标准化出来" + } + map.city_zip_code = d.city_code + map.district_zip_code = d.district_code + map.address = d.addr + map.addr_zip_code = d.addr_code + + updateData(map) + } + } + + def updateData(map){ + def udata = new BasicDBObject() + map.each{k,v-> + if(k!="_id"){ + udata.put(k,v) + } + } + POST_CODE_CONN.update(new BasicDBObject('_id', map._id), new BasicDBObject('$set', udata), true, false) + } + + + def findAllDatas(){ + def q = new BasicDBObject() + def dbcursor = post_code_coll.find(q) + return dbcursor.toList(); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy index da19a39b6..1e0e7891d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -6,6 +6,7 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.DateUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.ProvinceData import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.AMQP @@ -38,6 +39,8 @@ class SeaweedDataSynch extends BaseBatch{ public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020) + def fullCityMap = null + def mcaDistMap = null static main(args) { runBatch(new SeaweedDataSynch(), args); @@ -45,6 +48,8 @@ class SeaweedDataSynch extends BaseBatch{ @Override public void process(Object args) { + + run() println "全国district, shop, office同步完成" //民政部编码上传 @@ -56,8 +61,12 @@ class SeaweedDataSynch extends BaseBatch{ def run(){ + + fullCityMap = CommonDatautils.queryFullNameCity() + mcaDistMap = CommonDatautils.queryDistNameCity() + priceMap = findAllDistrictSellPrice() - def citys = CommonDatautils.findAllCity() + def citys = CommonDatautils.findAllCity().reverse() def index = 0 for(def c : citys){ index++ @@ -116,9 +125,33 @@ class SeaweedDataSynch extends BaseBatch{ def createData(dist){ def map = [:] map._id = dist._id - map.province = provinceMap.get(dist.city) - map.city = dist.city - map.region = dist.region + def area_code = null + def province = provinceMap.get(dist.city) + if(province){ + map.province = ProvinceData.getFullProvince(province) + } + map.fangjia_province = province + + def data = fullCityMap.get(dist.city) + if(data){ + map.city = data.city + area_code = data.mca_city_code + } + map.fangjia_city = dist.city + + def distData = mcaDistMap.get(dist.city+"#"+dist.region) + if(distData){ + map.region = distData.district + area_code = distData.mca_district_code + }else{ +// map.region = dist.region + println dist.city +" "+dist.region+" 民政部中没有找到相应的区域全称" + } + map.area_code = area_code + + map.fangjia_region = dist.region + + map.name = dist.name map.address = dist.address map.block = dist.block @@ -150,22 +183,24 @@ class SeaweedDataSynch extends BaseBatch{ map.location = loc } - //高德转百度 - def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon) - if(baiduMap){ - def location = [:] - location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng) - location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat) - map.bd_location = location - } + if(map.location){ + //高德转百度 + def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon) + if(baiduMap){ + def location = [:] + location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng) + location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat) + map.bd_location = location + } - //高德转WGS84 - def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon) - if(gpsLoactionMap){ - def gpslocation = [:] - gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng) - gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat) - map.gps_location = gpslocation + //高德转WGS84 + def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon) + if(gpsLoactionMap){ + def gpslocation = [:] + gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng) + gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat) + map.gps_location = gpslocation + } } map.build_type = dist.build_type @@ -209,10 +244,13 @@ class SeaweedDataSynch extends BaseBatch{ if(dist.main_type_area && dist.main_type_area.toString().isNumber()){ map.main_house_area = DataHandleUtils.getDoubleRound(2, dist.main_type_area.toString().toFloat()) } - def dataMap = priceMap.get(dist._id.toString()) - if(dataMap){ - map.avg_price = dataMap.avg_price + if(priceMap){ + def dataMap = priceMap.get(dist._id.toString()) + if(dataMap){ + map.avg_price = dataMap.avg_price + } } + // map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T") return map } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ProvinceData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ProvinceData.groovy index 48243f8e3..d694a37e5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ProvinceData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ProvinceData.groovy @@ -93,6 +93,10 @@ public class ProvinceData { put("宁夏", "宁夏回族自治区"); put("海南", "海南省"); put("台湾", "台湾省"); + put("上海", "上海市"); + put("北京", "北京市"); + put("天津", "天津市"); + put("重庆", "重庆市"); }}; static {