This commit is contained in:
2024-05-15 16:41:40 +08:00
parent 1874268b33
commit 5ef49f615d
23 changed files with 988 additions and 66 deletions
@@ -23,7 +23,7 @@ class AddThirdPartyPrice {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def c : citys){
println c +" "+cityCodeDatas.get(c)
def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(c)
def rets = createDate(dists, distCodeMap)
println dists.size() +" "+rets.size()
@@ -47,7 +47,7 @@ class BigdataBuildingBatch extends BaseBatch{
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"], null);
def distMap = [:]
for(def d : districts){
def alias = d.alias;
@@ -58,7 +58,7 @@ class BigdataCommunityBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
def ccbMaps = null
@@ -529,7 +529,7 @@ class BigdataDealBatch extends BaseBatch{
def getCityData(def city, def statMap, def ppeDistsMap){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], null)
println city +" 小区量:"+dists.size()
def tradesList = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
@@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], null);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
//测试通过匹配接口配房源效果
@@ -428,7 +428,7 @@ class CcbCommonUtils {
return rows
}
public static def findAllDistrictByCity(def city, def cat, def fields){
public static def findAllDistrictByCity(def city, def cat, def fields, def kind){
DBObject q = new BasicDBObject();
q.put("status",0)
@@ -445,14 +445,13 @@ class CcbCommonUtils {
keys.put(field, 1);
}
}
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
println "[${city}] 原始:"+dbsour.count()
def rets = []
def ids = []
for(def d : dbsour){
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
if(kind){
if(d.address){
d.address = d.address.replaceAll("\r|\n|\\s", "");
}
@@ -460,6 +459,16 @@ class CcbCommonUtils {
ids << d._id.toString()
rets << d
}
}else{
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
if(d.address){
d.address = d.address.replaceAll("\r|\n|\\s", "");
}
if(!ids.contains(d._id.toString())){
ids << d._id.toString()
rets << d
}
}
}
}
println "[${city}] 小区量:"+rets.size()
@@ -41,7 +41,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null);
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2024-03-27", "2024-04-26")
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-10")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -207,7 +207,7 @@ class GuiDingPriceBatch extends BaseBatch{
}
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null);
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
@@ -31,7 +31,7 @@ class StagingDistrictBatch extends BaseBatch{
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
for(def city : citys){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"], null)
def fenQiMap = getFenQiMap(dists);
def rets = createCityData(dists, fenQiMap)
rows.addAll(rets)
@@ -1289,10 +1289,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240425--人工指导价.xls"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240508--人工指导价.xls"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['C']){
if(d['J']){
// def region = RegionData.formatRegion(d['B'], d['C'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
@@ -1301,12 +1301,15 @@ ExcelUtils.multiThreadingModify(fileName, {d->
// d['H'] = dist._id.toString()
// }
// }
if(!d['L']){
def dist = CommonDatautils.findDistrictById(d['C'])
if(!d['A']){
def dist = CommonDatautils.findDistrictById(d['J'])
if(dist){
d['L'] = dist.region
d['M'] = dist.name
d['N'] = dist.status
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
d['K'] = dist.estate_type2
d['L'] = dist.status
d['M'] = dist.name_cat
}
}
}
@@ -71,7 +71,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
def statMap = getDistrictStatByCity(city, weekend)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null)
def thirdPriceMap = findThirdPartyPrice(city, statMap)
//def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)//老第三方均价表(现不用)
@@ -207,7 +207,7 @@ class BigdataCitypriceBatch extends BaseBatch{
def allLastMonthMap = [:]
def allLastYearMap = [:]
for(def city : citys) {
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"], null);
allDistricts.addAll(districts)
def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
@@ -1720,7 +1720,34 @@ class CommonDatautils {
}
return regionMap;
}
public static def getAllCityFullRegions(){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("category","region");
//queryCondition.put("show",1);
def dbCursor = REGION_BLOCK_COLL.find(queryCondition)
def regionMap = [:];
for(DBObject obj : dbCursor){
def city = obj.get("city");
def region = obj.get("name");
def pattern = obj.get("pattern");
if(region.indexOf("周边")==-1 && region.indexOf("其他")==-1) {
def r_map = regionMap.get(city);
if(!r_map){
r_map = [:]
regionMap.put(city, r_map);
}
if(!pattern){
pattern = [];
pattern << region;
}
r_map.put(region, ["pattern":pattern, "full_region":obj.get("org_name")]);
}
}
return regionMap;
}
public static def getRegionsByCity(def city){
@@ -1,25 +1,44 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
class DevProcess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
static main(args) {
def task = new DevProcess();
// task.run()
task.parseCityAndDist()
// task.parseCityAndDist()
// task.devSplitRoad()
// task.addFlag()
// task.matchDev()
// task.calAddrDistince()
// task.updateCityAndDist()
// task.addDevAddres1() //按添加地址中包含开发区名称的地址
// task.addDevAddres2()//添加相差2公里范围内的地址
// task.stat()
// task.addNewDev()
// task.createDevMapping()
task.deleteAddr()
}
def parseCityAndDist(){
@@ -131,10 +150,336 @@ class DevProcess {
}else{
println udata.name +" ÒÑ´æÔÚ"
}
}
}
}
def devSplitRoad(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d->
/*if(d['C'] && !d['V']){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C'])
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['V'] = jsonObject.get("province")
}
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['W'] = jsonObject.get("city")
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['X'] = jsonObject.get("district")
}
if(jsonObject && street && streetNumber && street!="null"){
d['Y'] = jsonObject.get("street")
if(streetNumber!='null'){
d['Z'] = jsonObject.get("streetNumber")
}
}
}*/
if(d['D'] && !d['M'] && d['P'] && d['P'].toString().toDouble().toInteger()==1){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['D'])
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['J'] = jsonObject.get("province")
}
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['K'] = jsonObject.get("city")
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['L'] = jsonObject.get("district")
}
if(jsonObject && street && streetNumber && street!="null"){
d['M'] = jsonObject.get("street")
if(streetNumber!='null'){
d['N'] = jsonObject.get("streetNumber")
}
}
}
},0)
}
def addFlag(){
ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d->
if(d['H'] && d['D']){
if(d['D'].contains(d['H'])){
d['O'] = "是"
}else{
def newName = null
if(d['H'].toString().startsWith(d['I'])){
newName = d['H'].toString().substring(d['I'].toString().length())
}else if(d['H'].toString().startsWith(d['I']+"省")){
newName = d['H'].toString().substring(d['I'].toString().length()+1)
}
if(newName && d['D'].contains(newName)){
d['O'] = "是2"
}
}
}
},1);
}
def matchDev(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d->
if(d['H']){
def q = new BasicDBObject()
q.put("name", d['H'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['S'] = obj._id.toString()
d['T'] = obj.name
}
}
},1);
}
def calAddrDistince(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d->
if(d['M'] && d['N'] && d['Y'] && d['Z']){
def addr1 = getFullAddress(d['J'], d['K'],d['L'],d['M'] , d['N'])
def addr2 = getFullAddress(d['V'], d['W'], d['X'], d['Y'] , d['Z'])
def json1 = GaoDeApiUtils.geo(d['K'], addr1)
def json2 = GaoDeApiUtils.geo(d['W'], addr2)
def arrs1 = getLocation(json1)
def arrs2 = getLocation(json2)
if(arrs1 && arrs2){
def distance = GeoUtils.calculateDistance(arrs1[0].toString().toDouble(), arrs1[1].toString().toDouble(), arrs2[0].toString().toDouble() , arrs2[1].toString().toDouble() )
d['AA'] = distance
}
}
},0);
}
def getFullAddress(province, city, region, street, streetName){
def addr = ""
if(addr){
addr = province
}
if(city){
addr = addr + city
}
if(region){
addr = addr + region
}
addr = addr + street + streetName
return addr
}
def getLocation(json1){
def arrs1 = null
if(json1 && json1.status=="1") {
def ret = json1.geocodes
if (ret) {
def str = ret.get(0).location
if(str){
arrs1 = str.split(",")
}
}
}
return arrs1
}
def updateCityAndDist(){
ExcelUtils.modify("D:\\数据\\数据建设\\等级开发区 - 汇总.xlsx",{d->
if(d['L']){
def udata = new BasicDBObject()
if(d['L']=="采用拆分结果"){
if(d["I"]){
udata.put("province", d["I"])
}
if(d["J"]){
udata.put("city", d["J"])
}
if(d["K"]){
udata.put("district", d["K"])
}
}else if(d['L']=="采用地图结果"){
if(d["H"]){
udata.put("province", d["H"])
}
if(d["E"]){
udata.put("city", d["E"])
}
if(d["F"]){
udata.put("district", d["F"])
}
}
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['C']))
if(obj && (!obj.city && !obj.district)){
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
}
}
})
}
def addDevAddres1(){
def rets = []
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d->
if(d['O'] && d['P']=="正常" && d['U']){
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U']))
if(obj){
d['AB'] = "存在"
def addr_alias = obj.addr_alias
if(!addr_alias){
addr_alias = []
}
def old_count = addr_alias.size()
if(!addr_alias.contains(d['M']+d['N'])){
addr_alias << d['M']+d['N']
}
if(old_count != addr_alias.size()){
def udata = new BasicDBObject()
udata.put("addr_alias", addr_alias)
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
}
}else{
rets << d
println "匹配:"+d['U']+" 原始:"+d['H']
}
}
},0)
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
}
def addDevAddres2(){
def rets = []
ExcelUtils.modify("D:\\数据\\数据建设\\待新增国家级以上开发区.xlsx",{d->
if((d['R'] && d['R'].toString()=="正常") || (d['P'] && d['P'].toString()=="正常")){
def obj = null
if(d['U']){
try{
obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U']))
}catch(Exception e){
obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U']))
}
}
if(!obj){
try{
obj = development_zone_CONN.findOne(new BasicDBObject("name", d['H']))
}catch(Exception e){
obj = development_zone_CONN.findOne(new BasicDBObject("name", d['H']))
}
}
if(obj){
d['AB'] = "存在"
def addr_alias = obj.addr_alias
if(!addr_alias){
addr_alias = []
}
def old_count = addr_alias.size()
if(d['R'] && d['R']=="正常" && !addr_alias.contains(d['Y']+d['Z']) && d['X'] && d['F'] && d['X']==d['F']){
addr_alias << d['Y']+d['Z']
}
if(d['P'] && d['P']=='正常' && !addr_alias.contains(d['M']+d['N']) && d['F'] && d['L'] && d['L']==d['F']){
addr_alias << d['M']+d['N']
}
if(old_count != addr_alias.size()){
def udata = new BasicDBObject()
udata.put("addr_alias", addr_alias)
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
}
}else{
// rets << d
println "匹配:"+d['U']+" 原始:"+d['H']
}
}
}, 0)
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
}
def stat(){
def dbcursor = development_zone_CONN.find(new BasicDBObject())
def total = 0
for(def d : dbcursor){
if(d.addr_alias){
total = total + d.addr_alias.size()
println d.addr_alias.size()
}
}
println total
}
def addNewDev(){
ExcelUtils.modify("D:\\数据\\数据建设\\待新增国家级以上开发区.xlsx",{d->
def udata = new BasicDBObject()
udata.address = d['M']+d['N']
udata.create_time = new Date()
if(d['G']){
if(d['G'].contains("国家级")){
udata.level = "国家级"
}else if(d['G'].contains("省级")){
udata.level = "省级"
}
}
udata.name = d['H']
udata.province = d['J']
udata.city = d['K']
udata.district = d['L']
udata.type = d['G']
def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name))
if(!obj){
development_zone_CONN.save(udata)
}else{
println udata.name +" 已存在"
}
})
}
def createDevMapping(){
def dbcursor = development_zone_CONN.find(new BasicDBObject())
for(def d : dbcursor){
def formatAddress = null
if(d.address){
formatAddress = AddressUtil.formatAddress(d.city, d.address)
}
if(formatAddress && !formatAddress.endsWith("号") && !formatAddress.endsWith("弄")){
formatAddress = null
}
def addr_alias = []
if(formatAddress){
addr_alias << formatAddress
}
if(d.addr_alias){
addr_alias = addr_alias + d.addr_alias
addr_alias.unique()
}
if(addr_alias){
addr_alias.each{addr->
def udata = new BasicDBObject()
udata.put("province", d.province)
udata.put("city", d.city)
udata.put("district", d.district)
udata.put("name",d.name)
udata.put("type", "地址")
if(d.township){
udata.put("street", d.township)
}
udata.put("address", addr)
DEV_MAPPING_CONN.save(udata)
}
}
}
}
def deleteAddr(){
ExcelUtils.modify("D:\\数据\\数据建设\\删除开发区地址.xlsx",{d->
DEV_MAPPING_CONN.remove(new BasicDBObject("_id", new ObjectId(d['A'])))
});
}
}
@@ -0,0 +1,45 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
class DistrictAliasProcess {
static main(args) {
def task = new DistrictAliasProcess();
task.run()
}
def run(){
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\名称在映射别名中的小区(2).xlsx",{ d->
if(d['H']=="是"){
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}else if(d['H']=="否"){
def aliasDist = d['F']
def arrs = aliasDist.split("-")
def obj = CommonDatautils.findDistrict(d['B'], arrs[0], arrs[1],"district")
if(obj){
def alias = obj.alias
def mapping_alias = obj.mapping_alias
def udata = new BasicDBObject()
if(alias && alias.contains(d['D'])){
alias.remove(d['D'])
udata.put("alias", alias)
}
if(mapping_alias && mapping_alias.contains(d['D'])){
mapping_alias.remove(d['D'])
udata.put("mapping_alias", mapping_alias)
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
}
}
})
}
}
@@ -30,7 +30,7 @@ class TownMiddleProcess {
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
def fullCityMap //= CommonDatautils.queryFullNameCity()
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
def task = new TownMiddleProcess();
@@ -48,9 +48,10 @@ class TownMiddleProcess {
//修改乡镇街道别名
// updateTownAlias()
//中间表同步到100
// synchTown()
synchTown()
//添加地级市标志
addCityFlag()
// addCityFlag()
// addPinyin()
}
@@ -171,16 +172,33 @@ class TownMiddleProcess {
}
}
//添加别名
def regionMap = CommonDatautils.getAllCityRegions()
def regionMap = CommonDatautils.getAllCityFullRegions()
lastRets.each {o->
if(o.district.contains("¿ª·¢Çø")){
println "ssss"
}
def city = RegionData.formatCity(o.city)
def district = RegionData.formatRegion(city, o.district)
if(city && district){
def cMap = regionMap.get(city)
if(cMap){
def regions = cMap.get(district)
o.put("alias", regions)
def regionInfoMap = cMap.get(district)
if(regionInfoMap){
def regions = regionInfoMap.get("pattern")
o.put("alias", regions)
o.put("mzb_district_name", regionInfoMap.full_region)
if(!o.mzb_district_code && regionInfoMap.full_region){
def mzb_val_str = mzbMaps.get(o.province+"|@|"+o.city+"|@|"+regionInfoMap.full_region)
if(mzb_val_str){
def mzb_vals = mzb_val_str.split("\\|@\\|")
o.put("mzb_province_code", mzb_vals[0])
o.put("mzb_city_code", mzb_vals[1])
o.put("mzb_district_code", mzb_vals[2])
}
}
}
}
}else{
println o.province+" "+o.city+" "+ o.district +" 房价网无此城市"
}
@@ -719,4 +737,18 @@ class TownMiddleProcess {
}
},1);
}
def addPinyin(){
def q = new BasicDBObject("type", "ÏçÕò½ÖµÀ")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
for(def d : dbcursor){
if(d.street){
def street_pinyin = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.street, " ")
def udata = new BasicDBObject()
udata.put("street_pinyin", street_pinyin)
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
}
}
}
}
@@ -354,7 +354,7 @@ class FangjiaClientUtils {
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/structureAndComplete?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true"
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -0,0 +1,137 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 房价网内部模型估值项目
* 小区建筑类型数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class FjwBigdataCommbuildtypeBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
//235为房价网内部计算均价的库
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
runBatch(new FjwBigdataCommbuildtypeBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def rets = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def distIds = PgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city))
def rows = createDatas(cityCodeDatas.get(city), distIds)
rets.addAll(rows)
println city +" "+rows.size()
}
delPgData()
saveToPgSql(rets)
}
def createDatas(cityCode, dists){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def id = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def dist = CommonDatautils.findDistrictFieldById(id, ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
if(dist){
def ret = createData(cityCode, dist)
if(ret){
rets.addAll(ret)
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createData(cityCode, dist){
def types = CcbCodeUtils.parseBuildType(dist.build_type)
def rets = []
for(def type : types){
def ret = []
ret << cityCode
ret << dist._id.toString()
ret << type
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
rets << ret
}
return rets;
}
def delPgData(){
sqlInstance.execute("delete from base.bigdata_commbuildtype")
println "base.bigdata_commbuildtype 数据已删除"
}
def saveToPgSql(rows){
def sqls = []
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
def sql = "insert into base.bigdata_commbuildtype values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
println "start: "+new Date()
def index =0
sqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
index++
println index
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14])
}
}
println "end: "+new Date()
}
}
@@ -0,0 +1,324 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
/**
* 房价网内部估值项目使用
* 小区数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class FjwBigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
//235为房价网内部计算均价的库
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
runBatch(new FjwBigdataCommunityBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
def today = new Date().getDate()
println "starting 。。。。。。。。。"
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], "all")
def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
}
if(today > 10){
delPgData()
saveToPgSql(rows)
}
}
def addDeleteDist(dists, ccbMaps, rets){
def fangjiaDists = []
for(def d : dists){
fangjiaDists << d._id.toString()
}
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
def ccbDists = ccbMaps.keySet()
for(def ccbId : ccbDists){
if(!fangjiaDists.contains(ccbId)){
def datas = ccbMaps.get(ccbId)
datas[datas.size()-1] = 1
rets << datas
}
}
println "添加删除状态小区:"+rets.size()
return rets
}
def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){
//def rets = Collections.synchronizedList(new ArrayList());
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = createData(cityCodeDataMap, distCodeMap, dist)
if(ret){
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createData(def cityCodeDataMap, def distCodeMap, def dist){
def ret = []
try{
def data = distCodeMap.get(dist.region)
if(!data){
println dist.city +" "+dist.region +" "+dist.name+" 没有找到对应的区域"
return ret;
}
ret << cityCodeDataMap.get(dist.city)
if(data){
ret << data.district_code
}else{
ret << ""
}
ret << dist._id.toString()
if(data){
ret << data.district
}else{
ret << ""
}
ret << dist.name
ret << dist.address
if(dist.lat){
def lat = dist.lat/10000000000d
def lng = dist.lng/10000000000d
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
if(loactionMap){
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
}else{
ret << ""
ret << ""
}
}else{
ret << ""
ret << ""
}
ret << dist.all_area
ret << dist.build_area
ret << dist.virescence_percent
ret << dist.cubage_percent
ret << dist.car_site_count
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
ret << dist.household_count
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd").toInteger()
ret << dist.status
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "pa101"
ret << ""
if(dist.address){
if(dist.street){
ret << dist.street
if(dist.street_number){
ret << dist.street_number
}else{
ret << ""
}
}else{
ret << ""
ret << ""
// def jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
// if(jsonObj && jsonObj.result && jsonObj.result.street){
// if(jsonObj.result.street){
// ret << jsonObj.result.street
// }else{
// ret << ""
// }
//
// if(jsonObj.result.streetNumber){
// ret << jsonObj.result.streetNumber
// }else{
// ret << ""
// }
// }else{
// ret << ""
// ret << ""
// }
}
}else{
ret << ""
ret << ""
}
ret << 0
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << ""
ret << ""
ret << ""
ret << 0
}catch(Exception e){
println dist.city + " "+dist.name+" error"
e.printStackTrace()
}
return ret;
}
def delPgData(){
sqlInstance.execute("delete from base.bigdata_community")
println "base.bigdata_community 数据已删除"
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.bigdata_community values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
sqlInstance.withBatch(50000, sql){ ps->
for(def r : rows) {
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
ps.toString()
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
def saveToPgSql2(rows){
def newRows = []
for(def r : rows) {
def newR = []
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
// r[i] = null
newR << null
}else{
newR << r[i]
}
}
newR << null
newR << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
newR << null
newR << '2999-12-31'
newR << null
newR << null
newR << null
newRows << newR
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
def index = 0
sqlInstance.withBatch(50000, sql){ ps->
for(def r : newRows) {
index++
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
index
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
def readTxt(def fileName){
File file = new File(fileName);
def rows = [];
file.eachLine{
def row = it.split("\\|@\\|")
rows << row
}
return rows;
}
def updateVersion(def time){
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
for(def tableName : tables){
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
sqlInstance.executeUpdate(sql)
}
}
}
@@ -13,7 +13,7 @@ class MsbPocTest {
task.run();
}
def run(){
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(小区、城市均价).xlsx"
def inputfile = "D:\\ĘýžÝ\\˝¨ĐĐ\\ˇż˛úšŔÖľ\\202311šăÖÝ.xlsx"
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
// standard(inputfile)
// match(inputfile)
@@ -24,7 +24,7 @@ class MsbPocTest {
// cityPrice()
// mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx")
addCCBPrice("C:\\Users\\gaojuan\\Desktop\\合肥科技农商行测试.xlsx")
addCCBPrice(inputfile)
}
def cityPrice(){
@@ -111,23 +111,18 @@ class MsbPocTest {
def match(file){
ExcelUtils.multiThreadingModify(file, { row ->
if(!row['J'] && row['F'] && row['P']=="没匹配" ){
def city = row['C']
def region = row['D']
def address = row['G'] //row['G']//row['F']
if(!row['N']){
def city = row['A']
def region = row['B']
def address = row['C'] //row['G']//row['F']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false)
if(jsonObject != null && jsonObject.credit>0) {
row['J'] = jsonObject.get("id")
row['K'] = jsonObject.get("city")
row['L'] = jsonObject.get("district")
row['M'] = jsonObject.get("name")
def type = jsonObject.get("propertyType")
if(type && type!='null'){
row['N'] = type
}
row['O'] = jsonObject.get("credit")
if(jsonObject != null) {
row['N'] = jsonObject.get("id")
row['O'] = jsonObject.get("city")
row['P'] = jsonObject.get("district")
row['Q'] = jsonObject.get("name")
}
}
})
@@ -174,19 +169,22 @@ class MsbPocTest {
def addCCBPrice(path){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['M']){
def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
if(dist){
def d = CommonDatautils.findDistrictById(dist._id.toString())
if(!row['H']){
// def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
// if(dist){
if(row['D'] instanceof String){
def d = CommonDatautils.findDistrictById(row['D'])
if(d){
def cityId = cityCodeDataMap.get(d.city)
def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId)
def map = CcbCommonUtils.findAvgPrice(d._id.toString(), cityId)
if(map){
row['M'] = map.avg_price
row['H'] = map.avg_price
row['I'] = map.calc_type
}
}
}
// }
}
})
}
@@ -33,7 +33,7 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\新疆3月待优化小区 - 副本.xlsx"
def filePath = "D:\\数据\\建行\\房产估值\\指导价\\补充154价格.xlsx"
// addGXDPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
@@ -256,14 +256,16 @@ class AddPriceTest {
def addFjwPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(!d['P']){
def dist = CommonDatautils.findDistrictById(d['G'])
if(dist){
if(!d['F']){
if(d['E'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['E'])
if(dist){
// d['N'] = dist.region
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){
d['O'] = dbo.avg_price
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){
d['F'] = dbo.avg_price
// d['J'] = dbo.avg_price_2b
}
}
}
}
@@ -27,8 +27,8 @@ class QiliangTest {
// task.run()
def data = [
'address': '浙江省湖州市德清县莫干山开发区中小工业园区',
'type': '2'
'address': '上海市奉贤区星火开发区莲塘路251号8幢',
'type': '1' //1:对公 0:对私
]
def s = task.getProxyResult(data);
print(s);
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 4
def nThread = Runtime.getRuntime().availableProcessors() * 5
println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0)