房源去重更改
This commit is contained in:
+6
@@ -62,6 +62,12 @@ class BigdataDealBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd")
|
||||
def objectId = dateToObjectId(date)
|
||||
println objectId
|
||||
|
||||
//664a22000000000000000000 5-20
|
||||
//664b73800000000000000000 5-21
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
//
|
||||
// println "curDate:"+curDate
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-20")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-24")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
|
||||
+8
-8
@@ -32,7 +32,7 @@ class DevProcess {
|
||||
// task.devSplitRoad()
|
||||
// task.devNameSplitRoad()
|
||||
// task.addFlag()
|
||||
// task.matchDev()
|
||||
task.matchDev()
|
||||
|
||||
// task.calAddrDistince()
|
||||
// task.updateCityAndDist()
|
||||
@@ -44,7 +44,7 @@ class DevProcess {
|
||||
// task.addNewDev()
|
||||
// task.createDevMapping()
|
||||
// task.deleteAddr()
|
||||
task.deleteAddr2()
|
||||
// task.deleteAddr2()
|
||||
// task.synchDevTo43()
|
||||
}
|
||||
|
||||
@@ -292,17 +292,17 @@ class DevProcess {
|
||||
}
|
||||
|
||||
def matchDev(){
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d->
|
||||
if(d['H']){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d->
|
||||
if(!d['F']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['H'])
|
||||
q.put("name", d['D'].replace("河北", "").replace("浙江",""))
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['S'] = obj._id.toString()
|
||||
d['T'] = obj.name
|
||||
d['F'] = obj._id.toString()
|
||||
d['G'] = obj.name
|
||||
}
|
||||
}
|
||||
},1);
|
||||
},0);
|
||||
}
|
||||
|
||||
def calAddrDistince(){
|
||||
|
||||
+1
@@ -180,6 +180,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
m.house_structure_code = null//getHouseStructureCode();
|
||||
m.source = "FJW"
|
||||
retMap.put(m.houseId, converData(m))
|
||||
houseIdsfilter.put(m.houseId)
|
||||
}else{
|
||||
println city +" "+m.houseId+" 已存在"
|
||||
}
|
||||
|
||||
+21
-7
@@ -33,17 +33,31 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\苏州价格.xlsx"
|
||||
addGXDPrice(filePath)
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
// exportPkPrice()
|
||||
// addRegion()
|
||||
addTradePrice(filePath)
|
||||
|
||||
|
||||
}
|
||||
|
||||
def addTradePrice(filePath){
|
||||
ExcelUtils.multiThreadingModify(filePath, { d ->
|
||||
if(d['Y']){
|
||||
def dist = CommonDatautils.findDistrictById(d['Y'])
|
||||
def tradePrice = CommonDatautils.findTrades(dist, '2024-02-01', '2024-05-01')
|
||||
if (tradePrice) {
|
||||
d['W'] = tradePrice.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
def addRegion(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
@@ -190,11 +204,11 @@ class AddPriceTest {
|
||||
|
||||
def addThirdPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (d['C']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['C'])
|
||||
if (!d['X']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['Y'])
|
||||
if(dist){
|
||||
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
|
||||
d['O'] = thirdPrice
|
||||
d['X'] = thirdPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -206,8 +220,8 @@ class AddPriceTest {
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404))
|
||||
|
||||
// q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404))
|
||||
q.put("s_date", 202404)
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def price = null
|
||||
for(def db : dbcursor){
|
||||
|
||||
+152
-3
@@ -39,7 +39,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class GaoDePoiDataProcess extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
|
||||
public static def version = new Date().getTime()
|
||||
|
||||
@@ -58,6 +58,9 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
|
||||
def fullCityMap = null
|
||||
|
||||
DBCollection CHUN_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MAP_POI_URL)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new GaoDePoiDataProcess(), args)
|
||||
@@ -69,12 +72,152 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
// synchMapPoi();
|
||||
|
||||
|
||||
synchCompany()
|
||||
// synchCompany()
|
||||
// poiCatProcess()
|
||||
// addressAndCatProcess()
|
||||
// synchNetwork()
|
||||
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
|
||||
// sysnchPoi()
|
||||
// sysnchVillage()
|
||||
|
||||
addTownship()
|
||||
}
|
||||
|
||||
def addTownship(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\湖州小区.xlsx",{d->
|
||||
if(d['J']){
|
||||
// def gaodeMap = getTownshipInfo(d['F']+","+d['E'])
|
||||
// if(gaodeMap && gaodeMap.township){
|
||||
// d['H'] = gaodeMap.city
|
||||
// d['I'] = gaodeMap.district
|
||||
// d['J'] = gaodeMap.township
|
||||
// d['K'] = gaodeMap.format_addr
|
||||
// }
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['H'])
|
||||
q.put("district", d['I'])
|
||||
q.put("street",d['G'])
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL_100.findOne(q)
|
||||
if(obj){
|
||||
d['O'] = "是"
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def sysnchVillage(){
|
||||
def pois = findAllVillagePoiMap()
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(pois.size())
|
||||
|
||||
|
||||
|
||||
for(def i=0;i<pois.size();i++){
|
||||
def p = pois[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def obj = MAPPOI_CONN.findOne(new BasicDBObject("city", p.city).append("district", p.district).append("name", p.name))
|
||||
if(!obj){
|
||||
def prov = ProvinceData.getProvince(p.city.replace("市",""))
|
||||
def townMap = findTownRange(p.city)
|
||||
createVillageData(prov, p, townMap)
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
def createVillageData(province, map, townMap){
|
||||
def fjw_province = province
|
||||
def fjw_city = RegionData.formatCity(map.city)
|
||||
def fjw_region = RegionData.formatRegion(fjw_city, map.district)
|
||||
|
||||
map.province = ProvinceData.getFullProvince(province)
|
||||
map.fjw_province = fjw_province
|
||||
map.fjw_city = fjw_city
|
||||
|
||||
if(map.cat){
|
||||
map.type = "地名地址信息"
|
||||
map.secondary_type = "普通地名"
|
||||
map.small_type = "村庄级地名"
|
||||
map.cat = "地名地址信息;普通地名;村庄级地名"
|
||||
|
||||
}
|
||||
map.location = ["lon": map.gcj02_x,"lat":map.gcj02_y]
|
||||
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
|
||||
//高德转百度
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(6, baiduMap.lng)
|
||||
location.lat = DataHandleUtils.getDoubleRound(6, baiduMap.lat)
|
||||
map.bd_location = location
|
||||
}
|
||||
|
||||
//高德转WGS84
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lng)
|
||||
gpslocation.lat = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lat)
|
||||
map.gps_location = gpslocation
|
||||
}
|
||||
|
||||
if(!map.township){
|
||||
|
||||
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
map.township = gaodeMap.township
|
||||
}
|
||||
|
||||
if(!fjw_region){
|
||||
if(gaodeMap && gaodeMap.district){
|
||||
fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
|
||||
}
|
||||
}
|
||||
|
||||
if(!fjw_region){
|
||||
println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap
|
||||
}
|
||||
|
||||
map.fjw_district = fjw_region
|
||||
}
|
||||
map.fjw_district = fjw_region
|
||||
map.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ")
|
||||
map.version = version
|
||||
|
||||
if(map.secondary_type && TAGS.contains(map.secondary_type)){
|
||||
map.status = 0
|
||||
}else{
|
||||
map.status = 1
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def findAllVillagePoiMap(){
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("name", 1)
|
||||
keys.put("addr", 1)
|
||||
keys.put("gcj02_x", 1)
|
||||
keys.put("gcj02_y", 1)
|
||||
def list = CHUN_POI_COLL.find(new BasicDBObject("std_tag","行政地标;村庄"), keys).limit(10).toList()
|
||||
return list;
|
||||
}
|
||||
|
||||
def sysnchPoi(){
|
||||
@@ -158,7 +301,6 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
|
||||
def prov = ProvinceData.getProvince(city.replace("市",""))
|
||||
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
@@ -409,6 +551,8 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def city = null
|
||||
def format_addr = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
@@ -420,6 +564,9 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
city = addressComponent.city
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -433,8 +580,10 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user