+1
-1
@@ -23,7 +23,7 @@ class AddThirdPartyPrice {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
for(def c : citys){
|
||||
println c +" "+cityCodeDatas.get(c)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(c)
|
||||
def rets = createDate(dists, distCodeMap)
|
||||
println dists.size() +" "+rets.size()
|
||||
|
||||
+1
-1
@@ -47,7 +47,7 @@ class BigdataBuildingBatch extends BaseBatch{
|
||||
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"], null);
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
def alias = d.alias;
|
||||
|
||||
+1
-1
@@ -58,7 +58,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
|
||||
|
||||
|
||||
def ccbMaps = null
|
||||
|
||||
+1
-1
@@ -529,7 +529,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def getCityData(def city, def statMap, def ppeDistsMap){
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], null)
|
||||
println city +" 小区量:"+dists.size()
|
||||
def tradesList = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
+1
-1
@@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], null);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
|
||||
//测试通过匹配接口配房源效果
|
||||
|
||||
+12
-3
@@ -428,7 +428,7 @@ class CcbCommonUtils {
|
||||
return rows
|
||||
}
|
||||
|
||||
public static def findAllDistrictByCity(def city, def cat, def fields){
|
||||
public static def findAllDistrictByCity(def city, def cat, def fields, def kind){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status",0)
|
||||
@@ -445,14 +445,13 @@ class CcbCommonUtils {
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
|
||||
println "[${city}] 原始:"+dbsour.count()
|
||||
def rets = []
|
||||
|
||||
def ids = []
|
||||
for(def d : dbsour){
|
||||
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
|
||||
if(kind){
|
||||
if(d.address){
|
||||
d.address = d.address.replaceAll("\r|\n|\\s", "");
|
||||
}
|
||||
@@ -460,6 +459,16 @@ class CcbCommonUtils {
|
||||
ids << d._id.toString()
|
||||
rets << d
|
||||
}
|
||||
}else{
|
||||
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
|
||||
if(d.address){
|
||||
d.address = d.address.replaceAll("\r|\n|\\s", "");
|
||||
}
|
||||
if(!ids.contains(d._id.toString())){
|
||||
ids << d._id.toString()
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
println "[${city}] 小区量:"+rets.size()
|
||||
|
||||
+1
-1
@@ -41,7 +41,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
|
||||
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null);
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
distMap.put(d.region+"##"+d.name, d)
|
||||
|
||||
+2
-2
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-03-27", "2024-04-26")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-10")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
@@ -207,7 +207,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null);
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
distMap.put(d.region+"##"+d.name, d)
|
||||
|
||||
+1
-1
@@ -31,7 +31,7 @@ class StagingDistrictBatch extends BaseBatch{
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
for(def city : citys){
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"], null)
|
||||
def fenQiMap = getFenQiMap(dists);
|
||||
def rets = createCityData(dists, fenQiMap)
|
||||
rows.addAll(rets)
|
||||
|
||||
+10
-7
@@ -1289,10 +1289,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240425--人工指导价.xls"
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240508--人工指导价.xls"
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['C']){
|
||||
if(d['J']){
|
||||
// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
@@ -1301,12 +1301,15 @@ ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// d['H'] = dist._id.toString()
|
||||
// }
|
||||
// }
|
||||
if(!d['L']){
|
||||
def dist = CommonDatautils.findDistrictById(d['C'])
|
||||
if(!d['A']){
|
||||
def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
if(dist){
|
||||
d['L'] = dist.region
|
||||
d['M'] = dist.name
|
||||
d['N'] = dist.status
|
||||
d['A'] = dist.city
|
||||
d['B'] = dist.region
|
||||
d['C'] = dist.name
|
||||
d['K'] = dist.estate_type2
|
||||
d['L'] = dist.status
|
||||
d['M'] = dist.name_cat
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -71,7 +71,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
def statMap = getDistrictStatByCity(city, weekend)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null)
|
||||
def thirdPriceMap = findThirdPartyPrice(city, statMap)
|
||||
//def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)//老第三方均价表(现不用)
|
||||
|
||||
|
||||
+1
-1
@@ -207,7 +207,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def allLastMonthMap = [:]
|
||||
def allLastYearMap = [:]
|
||||
for(def city : citys) {
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"], null);
|
||||
allDistricts.addAll(districts)
|
||||
|
||||
def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
|
||||
|
||||
+28
-1
@@ -1720,7 +1720,34 @@ class CommonDatautils {
|
||||
}
|
||||
return regionMap;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static def getAllCityFullRegions(){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("category","region");
|
||||
//queryCondition.put("show",1);
|
||||
def dbCursor = REGION_BLOCK_COLL.find(queryCondition)
|
||||
def regionMap = [:];
|
||||
for(DBObject obj : dbCursor){
|
||||
def city = obj.get("city");
|
||||
def region = obj.get("name");
|
||||
def pattern = obj.get("pattern");
|
||||
if(region.indexOf("周边")==-1 && region.indexOf("其他")==-1) {
|
||||
def r_map = regionMap.get(city);
|
||||
if(!r_map){
|
||||
r_map = [:]
|
||||
regionMap.put(city, r_map);
|
||||
}
|
||||
|
||||
if(!pattern){
|
||||
pattern = [];
|
||||
pattern << region;
|
||||
}
|
||||
r_map.put(region, ["pattern":pattern, "full_region":obj.get("org_name")]);
|
||||
}
|
||||
}
|
||||
return regionMap;
|
||||
}
|
||||
|
||||
|
||||
public static def getRegionsByCity(def city){
|
||||
|
||||
+349
-4
@@ -1,25 +1,44 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class DevProcess {
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
|
||||
DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
static main(args) {
|
||||
def task = new DevProcess();
|
||||
// task.run()
|
||||
task.parseCityAndDist()
|
||||
// task.parseCityAndDist()
|
||||
// task.devSplitRoad()
|
||||
// task.addFlag()
|
||||
// task.matchDev()
|
||||
|
||||
// task.calAddrDistince()
|
||||
// task.updateCityAndDist()
|
||||
// task.addDevAddres1() //按添加地址中包含开发区名称的地址
|
||||
// task.addDevAddres2()//添加相差2公里范围内的地址
|
||||
|
||||
// task.stat()
|
||||
// task.addNewDev()
|
||||
// task.createDevMapping()
|
||||
task.deleteAddr()
|
||||
}
|
||||
|
||||
def parseCityAndDist(){
|
||||
@@ -131,10 +150,336 @@ class DevProcess {
|
||||
}else{
|
||||
println udata.name +" ÒÑ´æÔÚ"
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def devSplitRoad(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d->
|
||||
/*if(d['C'] && !d['V']){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C'])
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
d['V'] = jsonObject.get("province")
|
||||
}
|
||||
|
||||
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
d['W'] = jsonObject.get("city")
|
||||
}
|
||||
|
||||
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
|
||||
d['X'] = jsonObject.get("district")
|
||||
}
|
||||
|
||||
if(jsonObject && street && streetNumber && street!="null"){
|
||||
d['Y'] = jsonObject.get("street")
|
||||
if(streetNumber!='null'){
|
||||
d['Z'] = jsonObject.get("streetNumber")
|
||||
}
|
||||
}
|
||||
}*/
|
||||
if(d['D'] && !d['M'] && d['P'] && d['P'].toString().toDouble().toInteger()==1){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['D'])
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
d['J'] = jsonObject.get("province")
|
||||
}
|
||||
|
||||
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
d['K'] = jsonObject.get("city")
|
||||
}
|
||||
|
||||
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
|
||||
d['L'] = jsonObject.get("district")
|
||||
}
|
||||
|
||||
if(jsonObject && street && streetNumber && street!="null"){
|
||||
d['M'] = jsonObject.get("street")
|
||||
if(streetNumber!='null'){
|
||||
d['N'] = jsonObject.get("streetNumber")
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def addFlag(){
|
||||
ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d->
|
||||
if(d['H'] && d['D']){
|
||||
if(d['D'].contains(d['H'])){
|
||||
d['O'] = "是"
|
||||
}else{
|
||||
def newName = null
|
||||
if(d['H'].toString().startsWith(d['I'])){
|
||||
newName = d['H'].toString().substring(d['I'].toString().length())
|
||||
}else if(d['H'].toString().startsWith(d['I']+"省")){
|
||||
newName = d['H'].toString().substring(d['I'].toString().length()+1)
|
||||
}
|
||||
if(newName && d['D'].contains(newName)){
|
||||
d['O'] = "是2"
|
||||
}
|
||||
}
|
||||
}
|
||||
},1);
|
||||
}
|
||||
|
||||
def matchDev(){
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d->
|
||||
if(d['H']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['H'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['S'] = obj._id.toString()
|
||||
d['T'] = obj.name
|
||||
}
|
||||
}
|
||||
},1);
|
||||
}
|
||||
|
||||
def calAddrDistince(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d->
|
||||
if(d['M'] && d['N'] && d['Y'] && d['Z']){
|
||||
def addr1 = getFullAddress(d['J'], d['K'],d['L'],d['M'] , d['N'])
|
||||
def addr2 = getFullAddress(d['V'], d['W'], d['X'], d['Y'] , d['Z'])
|
||||
|
||||
def json1 = GaoDeApiUtils.geo(d['K'], addr1)
|
||||
def json2 = GaoDeApiUtils.geo(d['W'], addr2)
|
||||
def arrs1 = getLocation(json1)
|
||||
def arrs2 = getLocation(json2)
|
||||
if(arrs1 && arrs2){
|
||||
def distance = GeoUtils.calculateDistance(arrs1[0].toString().toDouble(), arrs1[1].toString().toDouble(), arrs2[0].toString().toDouble() , arrs2[1].toString().toDouble() )
|
||||
d['AA'] = distance
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def getFullAddress(province, city, region, street, streetName){
|
||||
def addr = ""
|
||||
if(addr){
|
||||
addr = province
|
||||
}
|
||||
if(city){
|
||||
addr = addr + city
|
||||
}
|
||||
if(region){
|
||||
addr = addr + region
|
||||
}
|
||||
addr = addr + street + streetName
|
||||
return addr
|
||||
}
|
||||
def getLocation(json1){
|
||||
def arrs1 = null
|
||||
if(json1 && json1.status=="1") {
|
||||
def ret = json1.geocodes
|
||||
if (ret) {
|
||||
def str = ret.get(0).location
|
||||
if(str){
|
||||
arrs1 = str.split(",")
|
||||
}
|
||||
}
|
||||
}
|
||||
return arrs1
|
||||
}
|
||||
|
||||
def updateCityAndDist(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\等级开发区 - 汇总.xlsx",{d->
|
||||
if(d['L']){
|
||||
def udata = new BasicDBObject()
|
||||
if(d['L']=="采用拆分结果"){
|
||||
if(d["I"]){
|
||||
udata.put("province", d["I"])
|
||||
}
|
||||
if(d["J"]){
|
||||
udata.put("city", d["J"])
|
||||
}
|
||||
if(d["K"]){
|
||||
udata.put("district", d["K"])
|
||||
}
|
||||
}else if(d['L']=="采用地图结果"){
|
||||
if(d["H"]){
|
||||
udata.put("province", d["H"])
|
||||
}
|
||||
if(d["E"]){
|
||||
udata.put("city", d["E"])
|
||||
}
|
||||
if(d["F"]){
|
||||
udata.put("district", d["F"])
|
||||
}
|
||||
}
|
||||
|
||||
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['C']))
|
||||
if(obj && (!obj.city && !obj.district)){
|
||||
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addDevAddres1(){
|
||||
def rets = []
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d->
|
||||
if(d['O'] && d['P']=="正常" && d['U']){
|
||||
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U']))
|
||||
if(obj){
|
||||
d['AB'] = "存在"
|
||||
def addr_alias = obj.addr_alias
|
||||
if(!addr_alias){
|
||||
addr_alias = []
|
||||
}
|
||||
def old_count = addr_alias.size()
|
||||
if(!addr_alias.contains(d['M']+d['N'])){
|
||||
addr_alias << d['M']+d['N']
|
||||
}
|
||||
if(old_count != addr_alias.size()){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("addr_alias", addr_alias)
|
||||
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}else{
|
||||
rets << d
|
||||
println "匹配:"+d['U']+" 原始:"+d['H']
|
||||
}
|
||||
}
|
||||
},0)
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
|
||||
}
|
||||
|
||||
def addDevAddres2(){
|
||||
|
||||
def rets = []
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\待新增国家级以上开发区.xlsx",{d->
|
||||
if((d['R'] && d['R'].toString()=="正常") || (d['P'] && d['P'].toString()=="正常")){
|
||||
def obj = null
|
||||
if(d['U']){
|
||||
try{
|
||||
obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U']))
|
||||
}catch(Exception e){
|
||||
obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U']))
|
||||
}
|
||||
}
|
||||
|
||||
if(!obj){
|
||||
try{
|
||||
obj = development_zone_CONN.findOne(new BasicDBObject("name", d['H']))
|
||||
}catch(Exception e){
|
||||
obj = development_zone_CONN.findOne(new BasicDBObject("name", d['H']))
|
||||
}
|
||||
}
|
||||
if(obj){
|
||||
d['AB'] = "存在"
|
||||
def addr_alias = obj.addr_alias
|
||||
if(!addr_alias){
|
||||
addr_alias = []
|
||||
}
|
||||
def old_count = addr_alias.size()
|
||||
|
||||
if(d['R'] && d['R']=="正常" && !addr_alias.contains(d['Y']+d['Z']) && d['X'] && d['F'] && d['X']==d['F']){
|
||||
addr_alias << d['Y']+d['Z']
|
||||
}
|
||||
|
||||
if(d['P'] && d['P']=='正常' && !addr_alias.contains(d['M']+d['N']) && d['F'] && d['L'] && d['L']==d['F']){
|
||||
addr_alias << d['M']+d['N']
|
||||
}
|
||||
|
||||
if(old_count != addr_alias.size()){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("addr_alias", addr_alias)
|
||||
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}else{
|
||||
// rets << d
|
||||
println "匹配:"+d['U']+" 原始:"+d['H']
|
||||
}
|
||||
}
|
||||
}, 0)
|
||||
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def dbcursor = development_zone_CONN.find(new BasicDBObject())
|
||||
def total = 0
|
||||
for(def d : dbcursor){
|
||||
if(d.addr_alias){
|
||||
total = total + d.addr_alias.size()
|
||||
println d.addr_alias.size()
|
||||
}
|
||||
}
|
||||
println total
|
||||
}
|
||||
|
||||
def addNewDev(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\待新增国家级以上开发区.xlsx",{d->
|
||||
def udata = new BasicDBObject()
|
||||
udata.address = d['M']+d['N']
|
||||
udata.create_time = new Date()
|
||||
if(d['G']){
|
||||
if(d['G'].contains("国家级")){
|
||||
udata.level = "国家级"
|
||||
}else if(d['G'].contains("省级")){
|
||||
udata.level = "省级"
|
||||
}
|
||||
}
|
||||
|
||||
udata.name = d['H']
|
||||
udata.province = d['J']
|
||||
udata.city = d['K']
|
||||
udata.district = d['L']
|
||||
udata.type = d['G']
|
||||
|
||||
|
||||
def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name))
|
||||
if(!obj){
|
||||
development_zone_CONN.save(udata)
|
||||
}else{
|
||||
println udata.name +" 已存在"
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def createDevMapping(){
|
||||
def dbcursor = development_zone_CONN.find(new BasicDBObject())
|
||||
for(def d : dbcursor){
|
||||
def formatAddress = null
|
||||
if(d.address){
|
||||
formatAddress = AddressUtil.formatAddress(d.city, d.address)
|
||||
}
|
||||
if(formatAddress && !formatAddress.endsWith("号") && !formatAddress.endsWith("弄")){
|
||||
formatAddress = null
|
||||
}
|
||||
def addr_alias = []
|
||||
if(formatAddress){
|
||||
addr_alias << formatAddress
|
||||
}
|
||||
if(d.addr_alias){
|
||||
addr_alias = addr_alias + d.addr_alias
|
||||
addr_alias.unique()
|
||||
}
|
||||
if(addr_alias){
|
||||
addr_alias.each{addr->
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province", d.province)
|
||||
udata.put("city", d.city)
|
||||
udata.put("district", d.district)
|
||||
udata.put("name",d.name)
|
||||
udata.put("type", "地址")
|
||||
if(d.township){
|
||||
udata.put("street", d.township)
|
||||
}
|
||||
udata.put("address", addr)
|
||||
DEV_MAPPING_CONN.save(udata)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def deleteAddr(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\删除开发区地址.xlsx",{d->
|
||||
DEV_MAPPING_CONN.remove(new BasicDBObject("_id", new ObjectId(d['A'])))
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class DistrictAliasProcess {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictAliasProcess();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\名称在映射别名中的小区(2).xlsx",{ d->
|
||||
if(d['H']=="是"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
|
||||
}else if(d['H']=="否"){
|
||||
def aliasDist = d['F']
|
||||
def arrs = aliasDist.split("-")
|
||||
def obj = CommonDatautils.findDistrict(d['B'], arrs[0], arrs[1],"district")
|
||||
if(obj){
|
||||
def alias = obj.alias
|
||||
def mapping_alias = obj.mapping_alias
|
||||
def udata = new BasicDBObject()
|
||||
if(alias && alias.contains(d['D'])){
|
||||
alias.remove(d['D'])
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
if(mapping_alias && mapping_alias.contains(d['D'])){
|
||||
mapping_alias.remove(d['D'])
|
||||
udata.put("mapping_alias", mapping_alias)
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+38
-6
@@ -30,7 +30,7 @@ class TownMiddleProcess {
|
||||
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
|
||||
def fullCityMap //= CommonDatautils.queryFullNameCity()
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
|
||||
static main(args) {
|
||||
def task = new TownMiddleProcess();
|
||||
@@ -48,9 +48,10 @@ class TownMiddleProcess {
|
||||
//修改乡镇街道别名
|
||||
// updateTownAlias()
|
||||
//中间表同步到100
|
||||
// synchTown()
|
||||
synchTown()
|
||||
//添加地级市标志
|
||||
addCityFlag()
|
||||
// addCityFlag()
|
||||
// addPinyin()
|
||||
}
|
||||
|
||||
|
||||
@@ -171,16 +172,33 @@ class TownMiddleProcess {
|
||||
}
|
||||
}
|
||||
//添加别名
|
||||
def regionMap = CommonDatautils.getAllCityRegions()
|
||||
def regionMap = CommonDatautils.getAllCityFullRegions()
|
||||
lastRets.each {o->
|
||||
if(o.district.contains("¿ª·¢Çø")){
|
||||
println "ssss"
|
||||
}
|
||||
def city = RegionData.formatCity(o.city)
|
||||
def district = RegionData.formatRegion(city, o.district)
|
||||
if(city && district){
|
||||
def cMap = regionMap.get(city)
|
||||
if(cMap){
|
||||
def regions = cMap.get(district)
|
||||
o.put("alias", regions)
|
||||
def regionInfoMap = cMap.get(district)
|
||||
if(regionInfoMap){
|
||||
def regions = regionInfoMap.get("pattern")
|
||||
o.put("alias", regions)
|
||||
o.put("mzb_district_name", regionInfoMap.full_region)
|
||||
if(!o.mzb_district_code && regionInfoMap.full_region){
|
||||
def mzb_val_str = mzbMaps.get(o.province+"|@|"+o.city+"|@|"+regionInfoMap.full_region)
|
||||
if(mzb_val_str){
|
||||
def mzb_vals = mzb_val_str.split("\\|@\\|")
|
||||
o.put("mzb_province_code", mzb_vals[0])
|
||||
o.put("mzb_city_code", mzb_vals[1])
|
||||
o.put("mzb_district_code", mzb_vals[2])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}else{
|
||||
println o.province+" "+o.city+" "+ o.district +" 房价网无此城市"
|
||||
}
|
||||
@@ -719,4 +737,18 @@ class TownMiddleProcess {
|
||||
}
|
||||
},1);
|
||||
}
|
||||
|
||||
def addPinyin(){
|
||||
def q = new BasicDBObject("type", "ÏçÕò½ÖµÀ")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
|
||||
for(def d : dbcursor){
|
||||
if(d.street){
|
||||
def street_pinyin = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.street, " ")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("street_pinyin", street_pinyin)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -354,7 +354,7 @@ class FangjiaClientUtils {
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaAddressService/structureAndComplete?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true"
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true"
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 房价网内部模型估值项目
|
||||
* 小区建筑类型数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwBigdataCommbuildtypeBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
//235为房价网内部计算均价的库
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwBigdataCommbuildtypeBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rets = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
for(def city : citys){
|
||||
def distIds = PgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city))
|
||||
|
||||
def rows = createDatas(cityCodeDatas.get(city), distIds)
|
||||
rets.addAll(rows)
|
||||
println city +" "+rows.size()
|
||||
}
|
||||
|
||||
delPgData()
|
||||
saveToPgSql(rets)
|
||||
}
|
||||
|
||||
def createDatas(cityCode, dists){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def id = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def dist = CommonDatautils.findDistrictFieldById(id, ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
|
||||
if(dist){
|
||||
def ret = createData(cityCode, dist)
|
||||
if(ret){
|
||||
rets.addAll(ret)
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(cityCode, dist){
|
||||
|
||||
def types = CcbCodeUtils.parseBuildType(dist.build_type)
|
||||
def rets = []
|
||||
for(def type : types){
|
||||
def ret = []
|
||||
ret << cityCode
|
||||
ret << dist._id.toString()
|
||||
ret << type
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
|
||||
rets << ret
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def delPgData(){
|
||||
sqlInstance.execute("delete from base.bigdata_commbuildtype")
|
||||
println "base.bigdata_commbuildtype 数据已删除"
|
||||
}
|
||||
|
||||
|
||||
def saveToPgSql(rows){
|
||||
def sqls = []
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
def sql = "insert into base.bigdata_commbuildtype values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
println "start: "+new Date()
|
||||
def index =0
|
||||
sqlInstance.withBatch(20000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
index++
|
||||
println index
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14])
|
||||
}
|
||||
}
|
||||
println "end: "+new Date()
|
||||
}
|
||||
}
|
||||
+324
@@ -0,0 +1,324 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 房价网内部估值项目使用
|
||||
* 小区数据同步
|
||||
*/
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwBigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
//235为房价网内部计算均价的库
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwBigdataCommunityBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
|
||||
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
def today = new Date().getDate()
|
||||
|
||||
println "starting 。。。。。。。。。"
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], "all")
|
||||
def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rows.addAll(rets)
|
||||
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
|
||||
}
|
||||
|
||||
if(today > 10){
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
}
|
||||
|
||||
def addDeleteDist(dists, ccbMaps, rets){
|
||||
def fangjiaDists = []
|
||||
for(def d : dists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
|
||||
def ccbDists = ccbMaps.keySet()
|
||||
for(def ccbId : ccbDists){
|
||||
if(!fangjiaDists.contains(ccbId)){
|
||||
def datas = ccbMaps.get(ccbId)
|
||||
datas[datas.size()-1] = 1
|
||||
rets << datas
|
||||
}
|
||||
}
|
||||
println "添加删除状态小区:"+rets.size()
|
||||
return rets
|
||||
}
|
||||
|
||||
def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){
|
||||
//def rets = Collections.synchronizedList(new ArrayList());
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = createData(cityCodeDataMap, distCodeMap, dist)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(def cityCodeDataMap, def distCodeMap, def dist){
|
||||
def ret = []
|
||||
try{
|
||||
def data = distCodeMap.get(dist.region)
|
||||
if(!data){
|
||||
println dist.city +" "+dist.region +" "+dist.name+" 没有找到对应的区域"
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret << cityCodeDataMap.get(dist.city)
|
||||
if(data){
|
||||
ret << data.district_code
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist._id.toString()
|
||||
if(data){
|
||||
ret << data.district
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist.name
|
||||
ret << dist.address
|
||||
if(dist.lat){
|
||||
def lat = dist.lat/10000000000d
|
||||
def lng = dist.lng/10000000000d
|
||||
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
|
||||
if(loactionMap){
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist.all_area
|
||||
ret << dist.build_area
|
||||
ret << dist.virescence_percent
|
||||
ret << dist.cubage_percent
|
||||
ret << dist.car_site_count
|
||||
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
|
||||
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
|
||||
ret << dist.household_count
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd").toInteger()
|
||||
ret << dist.status
|
||||
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "pa101"
|
||||
ret << ""
|
||||
if(dist.address){
|
||||
if(dist.street){
|
||||
ret << dist.street
|
||||
if(dist.street_number){
|
||||
ret << dist.street_number
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
// def jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
|
||||
// if(jsonObj && jsonObj.result && jsonObj.result.street){
|
||||
// if(jsonObj.result.street){
|
||||
// ret << jsonObj.result.street
|
||||
// }else{
|
||||
// ret << ""
|
||||
// }
|
||||
//
|
||||
// if(jsonObj.result.streetNumber){
|
||||
// ret << jsonObj.result.streetNumber
|
||||
// }else{
|
||||
// ret << ""
|
||||
// }
|
||||
// }else{
|
||||
// ret << ""
|
||||
// ret << ""
|
||||
// }
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << 0
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << 0
|
||||
}catch(Exception e){
|
||||
println dist.city + " "+dist.name+" error"
|
||||
e.printStackTrace()
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
def delPgData(){
|
||||
sqlInstance.execute("delete from base.bigdata_community")
|
||||
println "base.bigdata_community 数据已删除"
|
||||
}
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.bigdata_community values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
sqlInstance.withBatch(50000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
try{
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
|
||||
ps.toString()
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
def saveToPgSql2(rows){
|
||||
def newRows = []
|
||||
for(def r : rows) {
|
||||
def newR = []
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
// r[i] = null
|
||||
newR << null
|
||||
}else{
|
||||
newR << r[i]
|
||||
}
|
||||
}
|
||||
newR << null
|
||||
newR << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
newR << null
|
||||
newR << '2999-12-31'
|
||||
newR << null
|
||||
newR << null
|
||||
newR << null
|
||||
|
||||
newRows << newR
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
def index = 0
|
||||
sqlInstance.withBatch(50000, sql){ ps->
|
||||
for(def r : newRows) {
|
||||
index++
|
||||
try{
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
|
||||
index
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def readTxt(def fileName){
|
||||
File file = new File(fileName);
|
||||
def rows = [];
|
||||
|
||||
file.eachLine{
|
||||
def row = it.split("\\|@\\|")
|
||||
rows << row
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
def updateVersion(def time){
|
||||
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
|
||||
for(def tableName : tables){
|
||||
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
|
||||
sqlInstance.executeUpdate(sql)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+21
-23
@@ -13,7 +13,7 @@ class MsbPocTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(小区、城市均价).xlsx"
|
||||
def inputfile = "D:\\ĘýžÝ\\˝¨ĐĐ\\ˇż˛úšŔÖľ\\202311šăÖÝ.xlsx"
|
||||
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
|
||||
// standard(inputfile)
|
||||
// match(inputfile)
|
||||
@@ -24,7 +24,7 @@ class MsbPocTest {
|
||||
// cityPrice()
|
||||
// mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx")
|
||||
|
||||
addCCBPrice("C:\\Users\\gaojuan\\Desktop\\合肥科技农商行测试.xlsx")
|
||||
addCCBPrice(inputfile)
|
||||
}
|
||||
|
||||
def cityPrice(){
|
||||
@@ -111,23 +111,18 @@ class MsbPocTest {
|
||||
|
||||
def match(file){
|
||||
ExcelUtils.multiThreadingModify(file, { row ->
|
||||
if(!row['J'] && row['F'] && row['P']=="没匹配" ){
|
||||
def city = row['C']
|
||||
def region = row['D']
|
||||
def address = row['G'] //row['G']//row['F']
|
||||
if(!row['N']){
|
||||
def city = row['A']
|
||||
def region = row['B']
|
||||
def address = row['C'] //row['G']//row['F']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false)
|
||||
if(jsonObject != null && jsonObject.credit>0) {
|
||||
row['J'] = jsonObject.get("id")
|
||||
row['K'] = jsonObject.get("city")
|
||||
row['L'] = jsonObject.get("district")
|
||||
row['M'] = jsonObject.get("name")
|
||||
def type = jsonObject.get("propertyType")
|
||||
if(type && type!='null'){
|
||||
row['N'] = type
|
||||
}
|
||||
row['O'] = jsonObject.get("credit")
|
||||
if(jsonObject != null) {
|
||||
row['N'] = jsonObject.get("id")
|
||||
row['O'] = jsonObject.get("city")
|
||||
row['P'] = jsonObject.get("district")
|
||||
row['Q'] = jsonObject.get("name")
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -174,19 +169,22 @@ class MsbPocTest {
|
||||
def addCCBPrice(path){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['M']){
|
||||
def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
|
||||
if(dist){
|
||||
def d = CommonDatautils.findDistrictById(dist._id.toString())
|
||||
if(!row['H']){
|
||||
// def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
|
||||
// if(dist){
|
||||
if(row['D'] instanceof String){
|
||||
def d = CommonDatautils.findDistrictById(row['D'])
|
||||
if(d){
|
||||
def cityId = cityCodeDataMap.get(d.city)
|
||||
def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId)
|
||||
def map = CcbCommonUtils.findAvgPrice(d._id.toString(), cityId)
|
||||
if(map){
|
||||
row['M'] = map.avg_price
|
||||
row['H'] = map.avg_price
|
||||
row['I'] = map.calc_type
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+9
-7
@@ -33,7 +33,7 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\新疆3月待优化小区 - 副本.xlsx"
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\指导价\\补充154价格.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
@@ -256,14 +256,16 @@ class AddPriceTest {
|
||||
|
||||
def addFjwPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(!d['P']){
|
||||
def dist = CommonDatautils.findDistrictById(d['G'])
|
||||
if(dist){
|
||||
if(!d['F']){
|
||||
if(d['E'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['E'])
|
||||
if(dist){
|
||||
// d['N'] = dist.region
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(dbo){
|
||||
d['O'] = dbo.avg_price
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(dbo){
|
||||
d['F'] = dbo.avg_price
|
||||
// d['J'] = dbo.avg_price_2b
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -27,8 +27,8 @@ class QiliangTest {
|
||||
// task.run()
|
||||
|
||||
def data = [
|
||||
'address': '浙江省湖州市德清县莫干山开发区中小工业园区',
|
||||
'type': '2'
|
||||
'address': '上海市奉贤区星火开发区莲塘路251号8幢',
|
||||
'type': '1' //1:对公 0:对私
|
||||
]
|
||||
def s = task.getProxyResult(data);
|
||||
print(s);
|
||||
|
||||
@@ -414,7 +414,7 @@ public class ExcelUtils{
|
||||
|
||||
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
|
||||
def body = data.subList(1, data.size())
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 4
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
println nThread
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
final AtomicInteger counter = new AtomicInteger(0)
|
||||
|
||||
Reference in New Issue
Block a user