Merge branch 'script_gj' into 'master'

房源去重更改

See merge request !847
This commit is contained in:
2024-05-23 18:24:27 +08:00
6 changed files with 189 additions and 19 deletions
@@ -62,6 +62,12 @@ class BigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd")
def objectId = dateToObjectId(date)
println objectId
//664a22000000000000000000 5-20
//664b73800000000000000000 5-21
def citys = CcbCommonUtils.getRunningCitys(args)
//
// println "curDate:"+curDate
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-20")
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-24")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -32,7 +32,7 @@ class DevProcess {
// task.devSplitRoad()
// task.devNameSplitRoad()
// task.addFlag()
// task.matchDev()
task.matchDev()
// task.calAddrDistince()
// task.updateCityAndDist()
@@ -44,7 +44,7 @@ class DevProcess {
// task.addNewDev()
// task.createDevMapping()
// task.deleteAddr()
task.deleteAddr2()
// task.deleteAddr2()
// task.synchDevTo43()
}
@@ -292,17 +292,17 @@ class DevProcess {
}
def matchDev(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d->
if(d['H']){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d->
if(!d['F']){
def q = new BasicDBObject()
q.put("name", d['H'])
q.put("name", d['D'].replace("河北", "").replace("浙江",""))
def obj = development_zone_CONN.findOne(q)
if(obj){
d['S'] = obj._id.toString()
d['T'] = obj.name
d['F'] = obj._id.toString()
d['G'] = obj.name
}
}
},1);
},0);
}
def calAddrDistince(){
@@ -180,6 +180,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
m.house_structure_code = null//getHouseStructureCode();
m.source = "FJW"
retMap.put(m.houseId, converData(m))
houseIdsfilter.put(m.houseId)
}else{
println city +" "+m.houseId+" 已存在"
}
@@ -33,17 +33,31 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\苏州价格.xlsx"
addGXDPrice(filePath)
def filePath = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达.xlsx"
// addGXDPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
// exportPkPrice()
// addRegion()
addTradePrice(filePath)
}
def addTradePrice(filePath){
ExcelUtils.multiThreadingModify(filePath, { d ->
if(d['Y']){
def dist = CommonDatautils.findDistrictById(d['Y'])
def tradePrice = CommonDatautils.findTrades(dist, '2024-02-01', '2024-05-01')
if (tradePrice) {
d['W'] = tradePrice.toInteger()
}
}
});
}
def addRegion(){
ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d->
def dist = CommonDatautils.findDistrictById(d['A'])
@@ -190,11 +204,11 @@ class AddPriceTest {
def addThirdPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['C']) {
def dist = CommonDatautils.findDistrictById(d['C'])
if (!d['X']) {
def dist = CommonDatautils.findDistrictById(d['Y'])
if(dist){
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
d['O'] = thirdPrice
d['X'] = thirdPrice
}
}
})
@@ -206,8 +220,8 @@ class AddPriceTest {
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404))
// q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404))
q.put("s_date", 202404)
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def price = null
for(def db : dbcursor){
@@ -39,7 +39,7 @@ import java.util.concurrent.atomic.AtomicLong
@BatchOwner(Watcher.GaoJuan)
class GaoDePoiDataProcess extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
public static def version = new Date().getTime()
@@ -58,6 +58,9 @@ class GaoDePoiDataProcess extends BaseBatch{
def fullCityMap = null
DBCollection CHUN_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MAP_POI_URL)
static main(args) {
runBatch(new GaoDePoiDataProcess(), args)
@@ -69,12 +72,152 @@ class GaoDePoiDataProcess extends BaseBatch{
// synchMapPoi();
synchCompany()
// synchCompany()
// poiCatProcess()
// addressAndCatProcess()
// synchNetwork()
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
// sysnchPoi()
// sysnchVillage()
addTownship()
}
def addTownship(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\湖州小区.xlsx",{d->
if(d['J']){
// def gaodeMap = getTownshipInfo(d['F']+","+d['E'])
// if(gaodeMap && gaodeMap.township){
// d['H'] = gaodeMap.city
// d['I'] = gaodeMap.district
// d['J'] = gaodeMap.township
// d['K'] = gaodeMap.format_addr
// }
def q = new BasicDBObject()
q.put("city", d['H'])
q.put("district", d['I'])
q.put("street",d['G'])
def obj = ADMINISTRATIVE_MAPPING_COLL_100.findOne(q)
if(obj){
d['O'] = "是"
}
}
})
}
def sysnchVillage(){
def pois = findAllVillagePoiMap()
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(pois.size())
for(def i=0;i<pois.size();i++){
def p = pois[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
def obj = MAPPOI_CONN.findOne(new BasicDBObject("city", p.city).append("district", p.district).append("name", p.name))
if(!obj){
def prov = ProvinceData.getProvince(p.city.replace("市",""))
def townMap = findTownRange(p.city)
createVillageData(prov, p, townMap)
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
}
def createVillageData(province, map, townMap){
def fjw_province = province
def fjw_city = RegionData.formatCity(map.city)
def fjw_region = RegionData.formatRegion(fjw_city, map.district)
map.province = ProvinceData.getFullProvince(province)
map.fjw_province = fjw_province
map.fjw_city = fjw_city
if(map.cat){
map.type = "地名地址信息"
map.secondary_type = "普通地名"
map.small_type = "村庄级地名"
map.cat = "地名地址信息;普通地名;村庄级地名"
}
map.location = ["lon": map.gcj02_x,"lat":map.gcj02_y]
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
//高德转百度
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
if(baiduMap){
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(6, baiduMap.lng)
location.lat = DataHandleUtils.getDoubleRound(6, baiduMap.lat)
map.bd_location = location
}
//高德转WGS84
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lng)
gpslocation.lat = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lat)
map.gps_location = gpslocation
}
if(!map.township){
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
if(gaodeMap && gaodeMap.township){
map.township = gaodeMap.township
}
if(!fjw_region){
if(gaodeMap && gaodeMap.district){
fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
}
}
if(!fjw_region){
println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap
}
map.fjw_district = fjw_region
}
map.fjw_district = fjw_region
map.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ")
map.version = version
if(map.secondary_type && TAGS.contains(map.secondary_type)){
map.status = 0
}else{
map.status = 1
}
return map
}
def findAllVillagePoiMap(){
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("district", 1)
keys.put("name", 1)
keys.put("addr", 1)
keys.put("gcj02_x", 1)
keys.put("gcj02_y", 1)
def list = CHUN_POI_COLL.find(new BasicDBObject("std_tag","行政地标;村庄"), keys).limit(10).toList()
return list;
}
def sysnchPoi(){
@@ -158,7 +301,6 @@ class GaoDePoiDataProcess extends BaseBatch{
final CountDownLatch latch = new CountDownLatch(results.size())
def prov = ProvinceData.getProvince(city.replace("市",""))
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
@@ -409,6 +551,8 @@ class GaoDePoiDataProcess extends BaseBatch{
def getTownshipInfo(def glocation){
def township = null
def district = null
def city = null
def format_addr = null
def jsonObject = null
def stop = false
def count = 0
@@ -420,6 +564,9 @@ class GaoDePoiDataProcess extends BaseBatch{
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
city = addressComponent.city
format_addr = jsonObject.regeocode.formatted_address
}
}
}
@@ -433,8 +580,10 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def map = [:]
map.city = city
map.district = district
map.township = township
map.format_addr = format_addr
return map;
}