Merge branch 'script_gj' into 'master'

Script gj

See merge request !840
This commit is contained in:
2024-05-19 22:26:28 +08:00
15 changed files with 1122 additions and 160 deletions
@@ -9,7 +9,7 @@ import com.mongodb.BasicDBObject
import org.apache.commons.lang.StringUtils
import java.util.regex.Pattern
// 分期分区与主小区映射关系表
@BatchOwner(Watcher.WangWenYan)
class FindStageDistrictBatch extends BaseBatch {
@@ -678,11 +678,7 @@ class CcbCommonUtils {
}
public static def deleteGuidedPrice(def dealdate){
def sql = "delete from guided_price_gj where dealdate=to_date('${dealdate}','yyyy-mm-dd')"
sqlInstance.executeUpdate(sql)
def sql2 = "delete from base.t0831_guided_price_gj_h where gujia_drc_dt='${dealdate}'"
def sql2 = "delete from base.guided_price where gujia_drc_dt='${dealdate}'"
pgsqlInstance.executeUpdate(sql2)
}
@@ -1289,7 +1289,7 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240508--人工指导价.xls"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\5月国信达价格录入.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['J']){
@@ -1890,6 +1890,25 @@ class CommonDatautils {
def dbo = null;
while(cur.hasNext()) {
dbo = cur.next()
if(dbo.value_type == 1){
dbo.cal_type = "指导价"
}else if(dbo.value_type == 2){
dbo.cal_type = "自身样本"
}else if(dbo.value_type == 3){
dbo.cal_type = "第三方"
}else if(dbo.value_type == 4){
dbo.cal_type = "交易数据"
}else if(dbo.value_type == 5){
dbo.cal_type = "周边楼盘"
}else if(dbo.value_type == 6){
dbo.cal_type = "推导"
}else if(dbo.value_type == 9){
dbo.cal_type = "神经网络"
}else if(dbo.value_type == 10){
dbo.cal_type = "无方式"
}else{
dbo.cal_type = ""
}
}
return dbo;
}
@@ -22,8 +22,8 @@ class FzDistrictMatch {
// task.addPrice()
// task.addThirdPrice()
// task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx")
task.longyanPrice()
task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx")
// task.longyanPrice()
}
def longyanPrice(){
@@ -29,7 +29,7 @@ class DevProcess {
def task = new DevProcess();
// task.run()
// task.parseCityAndDist()
// task.devSplitRoad()
task.devSplitRoad()
// task.addFlag()
// task.matchDev()
@@ -42,7 +42,8 @@ class DevProcess {
// task.addNewDev()
// task.createDevMapping()
// task.deleteAddr()
task.synchDevTo43()
// task.synchDevTo43()
}
def parseCityAndDist(){
@@ -159,54 +160,69 @@ class DevProcess {
}
def devSplitRoad(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d->
/*if(d['C'] && !d['V']){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C'])
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['V'] = jsonObject.get("province")
}
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['W'] = jsonObject.get("city")
}
def fileName = "/home/goojia/第二批公司.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\数据建设\\第二批公司.xlsx"
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['X'] = jsonObject.get("district")
}
def i = 5;
while(i<13){
ExcelUtils.multiThreadingModify(fileName,{d->
/*if(d['C'] && !d['V']){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C'])
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['V'] = jsonObject.get("province")
}
if(jsonObject && street && streetNumber && street!="null"){
d['Y'] = jsonObject.get("street")
if(streetNumber!='null'){
d['Z'] = jsonObject.get("streetNumber")
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['W'] = jsonObject.get("city")
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['X'] = jsonObject.get("district")
}
if(jsonObject && street && streetNumber && street!="null"){
d['Y'] = jsonObject.get("street")
if(streetNumber!='null'){
d['Z'] = jsonObject.get("streetNumber")
}
}
}*/
if(d['C'] && d['B']!='河北省'){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(null, d['C'])
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['J'] = jsonObject.get("province")
}
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['K'] = jsonObject.get("city")
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['L'] = jsonObject.get("district")
}
if(jsonObject && street && streetNumber && street!="null"){
d['M'] = jsonObject.get("street")
if(streetNumber!='null'){
d['N'] = jsonObject.get("streetNumber")
}
}
if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){
d['O'] = jsonObject.get("dev")
}
}
}*/
if(d['D'] && !d['M'] && d['P'] && d['P'].toString().toDouble().toInteger()==1){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['D'])
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['J'] = jsonObject.get("province")
}
},i)
i++
}
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['K'] = jsonObject.get("city")
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['L'] = jsonObject.get("district")
}
if(jsonObject && street && streetNumber && street!="null"){
d['M'] = jsonObject.get("street")
if(streetNumber!='null'){
d['N'] = jsonObject.get("streetNumber")
}
}
}
},0)
}
def addFlag(){
@@ -14,7 +14,7 @@ class DistrictAliasProcess {
}
def run(){
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\名称在映射别名中的小区(2).xlsx",{ d->
ExcelUtils.modify("D:\\数据\\数据建设\\小区\\名称在映射别名中-第二批.xlsx",{ d->
if(d['H']=="是"){
def udata = new BasicDBObject()
udata.put("status", 1)
@@ -10,6 +10,7 @@ import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.client.AggregateIterable
import org.bson.Document
import org.bson.types.ObjectId
/**
* 乡镇街道中间表
@@ -35,6 +36,7 @@ class TownMiddleProcess {
static main(args) {
def task = new TownMiddleProcess();
task.run()
}
def run(){
//创建省份
@@ -52,6 +54,13 @@ class TownMiddleProcess {
//添加地级市标志
// addCityFlag()
// addPinyin()
//打标记,将开发区与民政部相同的打上1
// addTjjDevFlag()
//打标记,用于标识乡镇街道是否输出及格式化输出街道
// addStreetFlag()
// addDistrictDiffFlag()
}
@@ -751,4 +760,44 @@ class TownMiddleProcess {
}
}
def addTjjDevFlag(){
def q = new BasicDBObject("type", "区域")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
for(def d : dbcursor){
if(d.district && d.mzb_district_name && d.district==d.mzb_district_name){
def udata = new BasicDBObject()
udata.put("district_diff", 1) //民政部,统计局 名称相同 district_diff打1
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
}
}
}
def addStreetFlag(){
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道-结果.xlsx",{ row->
if(row['I']){
if(row['I']=="标0"){
def udata = new BasicDBObject()
udata.put("street_output", 0) //0:不输出 1:输出
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
}else if(row['I']=="标1"){
def udata = new BasicDBObject()
udata.put("street_output", 1) //0:不输出 1:输出
if(row['G']){
udata.put("street", row['G']) //街道中去除区域
}
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
}
}
});
}
def addDistrictDiffFlag(){
ExcelUtils.modify("D:\\数据\\数据建设\\区域-结果.xlsx",{ row->
def flag = row['F'].toString().toDouble().toInteger()
def udata = new BasicDBObject()
udata.put("district_diff", flag)
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
});
}
}
@@ -354,13 +354,13 @@ class FangjiaClientUtils {
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true"
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
println jsonObj
// println jsonObj
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
@@ -48,11 +48,9 @@ class FjwBigdataDealBatch extends BaseBatch{
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
runBatch(new FjwBigdataDealBatch(), args);
@@ -61,59 +59,50 @@ class FjwBigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = ["乌鲁木齐"] //CcbCommonUtils.getRunningCitys(args)
// println "curDate:"+curDate
// def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
// addDealPriceMatch(city);
// }
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
println "历史id加载量:"+dealIds.size()
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
if(trades){
def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
}
println "curDate:"+curDate
def allCitys = CommonDatautils.findAllCity()
for(def city : allCitys){
addDealPriceMatch(city);
}
// def rows = []
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// def index = 0
// for(def city : citys){
// index ++
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def ppeDists = [:]
//
// def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
// println "历史id加载量:"+dealIds.size()
// def statMap = [:]//getDistrictStatByCity(city, weekend)
// if(city=="上海"){
// def youDataTrade = getYouDaData(statMap)
// def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
// rows.addAll(youdaDatas)
// println city +" "+youdaDatas.size()
// }
//
// def trades = getCityData(city, statMap, ppeDists)
// if(trades){
// def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
// rows.addAll(rets)
// println "【${index}】"+city +" 成交共:"+rets.size()
//
// def cityDatas = []
// cityDatas << headers
// cityDatas.addAll(rows)
// }
//
// }
//
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
// if(CommonDatautils.isWindow()){
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
// }
//
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
// println "txt 生成 " + rows.size()
//
//// delPgData()
// saveToPgSql(rows)
//
//// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
//// saveToPgSql2(datas)
saveToPgSql(rows)
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
// saveToPgSql2(datas)
}
def readTxt(def fileName){
@@ -186,19 +175,7 @@ class FjwBigdataDealBatch extends BaseBatch{
}
def getCommunityPrices(def city){
def sql = "select commid, avg_price from community_base_price_gj where year_month=${endDate} and cityname='${city}'";
def map = [:]
try{
sqlInstance.eachRow(sql,{
map.put(it.commid, it.avg_price)
});
}catch(Exception e){
getPpeGTE15Datas(city)
}
return map
}
def findThirdPartyPrice(def city) {
def q = new BasicDBObject()
@@ -529,7 +506,7 @@ class FjwBigdataDealBatch extends BaseBatch{
def getCityData(def city, def statMap, def ppeDistsMap){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], null)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], "all")
println city +" 小区量:"+dists.size()
def tradesList = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
@@ -537,9 +514,13 @@ class FjwBigdataDealBatch extends BaseBatch{
// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
@@ -651,7 +632,7 @@ class FjwBigdataDealBatch extends BaseBatch{
}
def findHistoryDealIds(def city){
def sql = "SELECT gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${city}'"
def sql = "SELECT gujia_bds_mdl_id from base.bigdata_deal where urbn_nm='${city}'"
def ids = []
try{
pgsqlInstance.eachRow(sql,{
@@ -664,20 +645,6 @@ class FjwBigdataDealBatch extends BaseBatch{
return ids
}
def getPpeGTE15Datas(def city){
def sql = "select unit_price,commid from avm_evaluation_item_gj where data_type='成交' and pe>=15 and pe<=20 " +
"and cityname='"+city+"'";
def map = [:]
try{
sqlInstance.eachRow(sql,{
map.put(it.commid, it.unit_price)
});
}catch(Exception e){
getPpeGTE15Datas(city)
}
return map
}
def addDealPriceMatch(city){
@@ -743,8 +710,8 @@ class FjwBigdataDealBatch extends BaseBatch{
def delPgData(){
def month = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
pgsqlInstance.execute("delete from base.t0831_bigdata_deal_gj_h where p9_start_date='${month}'")
println "base.t0831_bigdata_deal_gj_h 数据已删除"
pgsqlInstance.execute("delete from base.bigdata_deal where p9_start_date='${month}'")
println "base.bigdata_deal 数据已删除"
}
def saveToPgSql(rows){
@@ -764,7 +731,7 @@ class FjwBigdataDealBatch extends BaseBatch{
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
def sql = "insert into base.bigdata_deal values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
@@ -0,0 +1,226 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import org.bson.types.ObjectId
/**
* 建行模型估值项目
* 人工指导价同步
*/
@BatchOwner(Watcher.GaoJuan)
class FjwGuiDingPriceBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol();
def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15";
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
runBatch(new FjwGuiDingPriceBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
FjwPgsqlCommUtils.deleteGuidedPrice(curDate)
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-03-01", "2022-04-01")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
rows.addAll(rets)
println city + " " + rets.size()
}
}
saveToPgSql(rows)
}
def findFangDatas(def districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println " 小区量:"+districts.size()
def result = Collections.synchronizedList(new ArrayList());
for(def i=0;i<districts.size();i++){
final def dist = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def obj = findMatchThirdPrice(dist)
if(obj && obj.avg_price){
result << ["dist_id":dist._id.toString(),"city":dist.city, "region":dist.region, "name":dist.name, "avg_price":obj.avg_price,"row_id":obj._id.toString(), "m_date":obj.m_date]
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name +" "+obj.avg_price
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return result
}
def findPersonDatas(def districts, def dataMap){
def result = []
for(def i=0;i<districts.size();i++) {
def dist = districts[i];
def price = dataMap.get(dist._id.toString())
if (price) {
result << ["dist_id": dist._id.toString(), "city": dist.city, "region": dist.region, "name": dist.name, "avg_price": price.toInteger(), "row_id": new ObjectId(), "m_date": new Date()]
println dist.city + " " + dist.region + " " + dist.name + " " + price
}
}
return result;
}
def findMatchThirdPrice(dist){
def q = new BasicDBObject()
q.put("match_city", dist.city)
q.put("match_region", dist.region)
q.put("match_name", dist.name)
q.put("s_date", sDate)
q.put("source","搜房")
def obj = dataProcessColl.findOne(q)
return obj
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
def rets = []
if(prices){
for(def ps : prices){
def dist = distMap.get(ps.region+"##"+ps.name)
if(dist){
def ret = []
def areaData = distCodeMap.get(dist.region)
if(areaData){
ret << ps._id.toString()
ret << dist._id.toString()
ret << dist.name
ret << cityCodeDatas.get(dist.city)
ret << CcbCommonUtils.getCcbCity(dist.city)
ret << areaData.district_code
ret << areaData.district
ret << curDate//DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << ps.guiding_value
ret << "房价网"
ret << ""
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
}
return rets;
}
def createDatas(cityCodeDatas, distCodeMap, datas){
def rets = []
if(datas) {
for (def ps : datas) {
def ret = []
def areaData = distCodeMap.get(ps.region)
if (areaData) {
ret << ps.row_id
ret << ps.dist_id
ret << ps.name
ret << cityCodeDatas.get(ps.city)
ret << ps.city
ret << areaData.district_code
ret << areaData.district
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << ps.avg_price
ret << "房价网"
ret << ""
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
return rets;
}
def createPersonDatas(cityCodeDatas, distCodeMap, dataMap){
}
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], "all");
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
}
return distMap;
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.guided_price values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(300, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20])
}
}
println "end: "+new Date()
}
}
@@ -21,5 +21,19 @@ class FjwPgsqlCommUtils {
return set
}
public static def findDistGovPrice(def city){
def ids = []
sqlInstance.eachRow("SELECT gujia_bds_id from base.community_guide_price where urbn_nm='${city}'",{it->
ids << it.gujia_bds_id
});
return ids;
}
public static def deleteGuidedPrice(def dealdate){
def sql2 = "delete from base.guided_price where gujia_drc_dt='${dealdate}'"
sqlInstance.executeUpdate(sql2)
}
}
@@ -0,0 +1,674 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.text.SimpleDateFormat
import java.util.concurrent.ConcurrentHashMap
/**
* 建行模型估值项目
* 第三方均价数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class FjwThirdPartyPriceBatch extends BaseBatch {
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//202304//
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
runBatch(new FjwThirdPartyPriceBatch(), args);
}
@Override
public void process(Object args) {
def citys = ["乌鲁木齐"]//CcbCommonUtils.getRunningCitys(args)
def stop = false
def month = -1
while(!stop){
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
println weekend.toString() +" "+sDate
runDatas(citys)
month = month -1
sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
if(sDate == 201912){
stop = true;
}
}
}
def runDatas(citys){
def rows = []
def index = 0
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
index++
def gocIdsPrices = []
if(city == "深圳"){
gocIdsPrices = FjwPgsqlCommUtils.findDistGovPrice(city)
}
def statMap = getDistrictStatByCity(city, weekend)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], "all")
def thirdPriceMap = findThirdPartyPrice(city, statMap)
def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocIdsPrices)
rows.addAll(rets)
println index+" "+city+" "+rets.size() +" "+rows.size()
}
saveToPgSql(rows)
}
def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){
def rets = []
for(def d : dists){
def third = thirdPriceMap.get(d.region+"##"+d.name)
if(third){
def data = distCodeMap.get(d.region)
if(data){
def ret = []
ret << third.id.toString()
ret << d._id.toString()
ret << d.name
ret << cityCodeDatas.get(d.city)
ret << d.city
ret << data.district_code
ret << data.district
ret << sDate
if(third.zhugezhaofang_esf_price){
ret << third.zhugezhaofang_esf_price
}else{
ret << ""
}
if(third.anjuke_esf_price){
ret << third.anjuke_esf_price
}else{
ret << ""
}
if(third.soufang_esf_price){
ret << third.soufang_esf_price
}else{
ret << ""
}
if(third.lianjia_esf_price){
ret << third.lianjia_esf_price
}else{
ret << ""
}
if(third.cityhouse_esf_price){
ret << third.cityhouse_esf_price
}else{
ret << ""
}
if(third.maitian_esf_price){
ret << third.maitian_esf_price
}else{
ret << ""
}
if(third.beike_esf_price){
ret << third.beike_esf_price
}else{
ret << ""
}
if(third.get("365house_esf_price")){
ret << third.get("365house_esf_price")
}else{
ret << ""
}
if(third.zhongyuan_esf_price){
ret << third.zhongyuan_esf_price
}else{
ret << ""
}
if(third.tuboshi_esf_price){
ret << third.tuboshi_esf_price
}else{
ret << ""
}
if(third.qfang_esf_price){
ret << third.qfang_esf_price
}else{
ret << ""
}
if(third.fangtan_esf_price){
ret << third.fangtan_esf_price
}else{
ret << ""
}
if(third.fangduoduo_esf_price){
ret << third.fangduoduo_esf_price
}else{
ret << ""
}
if(third.get("5i5j_esf_price")){
ret << third.get("5i5j_esf_price")
}else{
ret << ""
}
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
return rets;
}
def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocPrices){
def allRows = []
for(def d : dists){
if(gocPrices && gocPrices.contains(d._id.toString())){
continue;
}
def third = thirdPriceMap.get(d.region+"##"+d.name)
if(third){
def data = distCodeMap.get(d.region)
if(data){
def dataMap = [:]
dataMap.COMMID = d._id.toString()
dataMap.COMMNAME = d.name
dataMap.CITYID = cityCodeDatas.get(d.city)
dataMap.CITYNAME = d.city
dataMap.DISTID = data.district_code
dataMap.DISTNAME = data.district
dataMap.YEAR_MONTH = sDate
def price_map = [:]
dataMap.price_map = price_map
if(third.zhugezhaofang_esf_price){
price_map.zhuge_avg_price = third.zhugezhaofang_esf_price
}
if(third.anjuke_esf_price){
price_map.anjuke_avg_price = third.anjuke_esf_price
}
if(third.soufang_esf_price){
price_map.fang_avg_price = third.soufang_esf_price
}
if(third.lianjia_esf_price){
price_map.lianjia_avg_price = third.lianjia_esf_price
}
if(third.cityhouse_esf_price){
price_map.cityhouse_avg_price = third.cityhouse_esf_price
}
if(third.maitian_esf_price){
price_map.maitian_avg_price = third.maitian_esf_price
}
if(third.beike_esf_price){
price_map.ke_avg_price = third.beike_esf_price
}
if(third.get("365house_esf_price")){
price_map.taofang_avg_price = third.get("365house_esf_price")
}
if(third.zhongyuan_esf_price){
price_map.zhongyuan_avg_price = third.zhongyuan_esf_price
}
if(third.tuboshi_esf_price){
price_map.tbs_avg_price = third.tuboshi_esf_price
}
if(third.qfang_esf_price){
price_map.qfang_avg_price = third.qfang_esf_price
}
if(third.fangtan_esf_price){
price_map.fangtan_avg_price = third.fangtan_esf_price
}
if(third.fangduoduo_esf_price){
price_map.fangduoduo_avg_price = third.fangduoduo_esf_price
}
if(third.get("5i5j_esf_price")){
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
}
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
allRows << dataMap
}
}
}
def results = []
for(def m : allRows){
def priceMap = m.price_map
priceMap.each{k,v->
def ret = []
ret << new ObjectId().toString()
ret << m.CITYID
ret << m.DISTID
ret << m.COMMID
ret << m.COMMNAME
def web = ""
if(k=="zhuge_avg_price"){
web = "诸葛找房"
}else if(k=="anjuke_avg_price"){
web = "安居客"
}else if(k=="fang_avg_price"){
web = "房天下"
}else if(k=="lianjia_avg_price"){
web = "链家"
}else if(k=="cityhouse_avg_price"){
web = "禧泰数据"
}else if(k=="maitian_avg_price"){
web = "麦田"
}else if(k=="ke_avg_price"){
web = "贝壳"
}else if(k=="taofang_avg_price"){
web = "淘房365"
}else if(k=="zhongyuan_avg_price"){
web = "中原"
}else if(k=="tbs_avg_price"){
web = "兔博士"
}else if(k=="qfang_avg_price"){
web = "Q房"
}else if(k=="fangtan_avg_price"){
web = "房探"
}else if(k=="fangduoduo_avg_price"){
web = "房多多"
}else if(k=="wiwj_avg_price"){
web = "我爱我家"
}
ret << sourceMap.get(web)
ret << web
ret << m.YEAR_MONTH
ret << v
ret << ""
ret << ""
ret << m.updatetime
results << ret
}
}
return results
}
def createCityDatas2(cityCodeDatas, distCodeMap, dists, thirdPriceMap) {
def rets = []
for (def d : dists) {
def third = thirdPriceMap.get(d.region + "##" + d.name)
if (third) {
def data = distCodeMap.get(d.region)
if (data) {
third.each{k,v->
def ret = []
ret << new ObjectId().toString()
ret << cityCodeDatas.get(d.city)
ret << data.district_code
ret << d._id.toString()
ret << d.name
def web = ""
if(k=="zhugezhaofang_esf_price"){
web = "诸葛找房"
}else if(k=="anjuke_esf_price"){
web = "安居客"
}else if(k=="soufang_esf_price"){
web = "房天下"
}else if(k=="lianjia_esf_price"){
web = "链家"
}else if(k=="cityhouse_esf_price"){
web = "禧泰数据"
}else if(k=="maitian_esf_price"){
web = "麦田"
}else if(k=="beike_esf_price"){
web = "贝壳"
}else if(k=="365house_esf_price"){
web = "淘房365"
}else if(k=="zhongyuan_esf_price"){
web = "中原"
}else if(k=="tuboshi_esf_price"){
web = "兔博士"
}else if(k=="qfang_esf_price"){
ret << "Q房"
}else if(k=="fangtan_esf_price"){
ret << "房探"
}else if(k=="fangduoduo_esf_price"){
ret << "房多多"
}else if(k=="5i5j_esf_price"){
ret << "我爱我家"
}else{
ret << ""
}
ret << sourceMap.get(web)
ret << web
ret << sDate
ret << v
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
}
return rets;
}
def createData3(cityid){
def sql = "select COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from THIRD_PARTY_PRICE_GJ where CITYID='${cityid}' and YEAR_MONTH=${sDate}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def dataMap = [:]
dataMap.COMMID = r.COMMID
dataMap.COMMNAME = r.COMMNAME
dataMap.CITYID = r.CITYID
dataMap.CITYNAME = r.CITYNAME
dataMap.DISTID = r.DISTID
dataMap.DISTNAME = r.DISTNAME
dataMap.YEAR_MONTH = r.YEAR_MONTH
def price_map = [:]
dataMap.price_map = price_map
if(r.ZHUGE_AVG_PRICE){
price_map.zhuge_avg_price = r.ZHUGE_AVG_PRICE
}
if(r.ANJUKE_AVG_PRICE){
price_map.anjuke_avg_price = r.ANJUKE_AVG_PRICE
}
if(r.FANG_AVG_PRICE){
price_map.fang_avg_price = r.FANG_AVG_PRICE
}
if(r.LIANJIA_AVG_PRICE){
price_map.lianjia_avg_price = r.LIANJIA_AVG_PRICE
}
if(r.CITYHOUSE_AVG_PRICE){
price_map.cityhouse_avg_price = r.CITYHOUSE_AVG_PRICE
}
if(r.MAITIAN_AVG_PRICE){
price_map.maitian_avg_price = r.MAITIAN_AVG_PRICE
}
if(r.KE_AVG_PRICE){
price_map.ke_avg_price = r.KE_AVG_PRICE
}
if(r.TAOFANG_AVG_PRICE){
price_map.taofang_avg_price = r.TAOFANG_AVG_PRICE
}
if(r.ZHONGYUAN_AVG_PRICE){
price_map.zhongyuan_avg_price = r.ZHONGYUAN_AVG_PRICE
}
if(r.TBS_AVG_PRICE){
price_map.tbs_avg_price = r.TBS_AVG_PRICE
}
if(r.QFANG_AVG_PRICE){
price_map.qfang_avg_price = r.QFANG_AVG_PRICE
}
if(r.FANGTAN_AVG_PRICE){
price_map.fangtan_avg_price = r.FANGTAN_AVG_PRICE
}
if(r.FANGDUODUO_AVG_PRICE){
price_map.fangduoduo_avg_price = r.FANGDUODUO_AVG_PRICE
}
if(r.WIWJ_AVG_PRICE){
price_map.wiwj_avg_price = r.WIWJ_AVG_PRICE
}
dataMap.updatetime = r.UPDATETIME
allRows << dataMap
});
def results = []
for(def m : allRows){
def priceMap = m.price_map
priceMap.each{k,v->
def ret = []
ret << new ObjectId().toString()
ret << m.CITYID
ret << m.DISTID
ret << m.COMMID
ret << m.COMMNAME
def web = ""
if(k=="zhuge_avg_price"){
web = "诸葛找房"
}else if(k=="anjuke_avg_price"){
web = "安居客"
}else if(k=="fang_avg_price"){
web = "房天下"
}else if(k=="lianjia_avg_price"){
web = "链家"
}else if(k=="cityhouse_avg_price"){
web = "禧泰数据"
}else if(k=="maitian_avg_price"){
web = "麦田"
}else if(k=="ke_avg_price"){
web = "贝壳"
}else if(k=="taofang_avg_price"){
web = "淘房365"
}else if(k=="zhongyuan_avg_price"){
web = "中原"
}else if(k=="tbs_avg_price"){
web = "兔博士"
}else if(k=="qfang_avg_price"){
web = "Q房"
}else if(k=="fangtan_avg_price"){
web = "房探"
}else if(k=="fangduoduo_avg_price"){
web = "房多多"
}else if(k=="wiwj_avg_price"){
web = "我爱我家"
}
ret << sourceMap.get(web)
ret << web
ret << m.YEAR_MONTH
ret << v
ret << ""
ret << ""
ret << m.updatetime
results << ret
}
}
return results
}
def findThirdPartyPrice(def city, def statMap){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
dataMap.put("id", db._id.toString())
map.put(key, dataMap)
}
if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price>3000){
dataMap.put("soufang_esf_price", db.soufang_esf_price)
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
dataMap.put("maitian_esf_price", db.maitian_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
dataMap.put("365house_esf_price", db.get("365house_esf_price"))
}
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
}
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
}
if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price && db.qfang_esf_price>3000){
dataMap.put("qfang_esf_price", db.qfang_esf_price)
}
if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price && db.fangtan_esf_price>3000){
dataMap.put("fangtan_esf_price", db.fangtan_esf_price)
}
if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price)
}
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
}
def resultMap = [:]
map.each {key, value->
if(value && value.size()>1){
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){
def price = statMap.get(key)
if(price){
def priceMap = [:]
def startPrice = (int)(price - price*0.3)
def endPrice = (int)(price*0.3 + price)
value.each{pk,pv->
if(pk.contains("price")){
if(pv>=startPrice && pv<=endPrice){
priceMap.put(pk,pv)
}
}else{
priceMap.put(pk,pv)
}
}
if(priceMap.size()>1){
resultMap.put(key, priceMap)
}
}
}else{
resultMap.put(key, value)
}
}
}
return resultMap
}
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"##"+d.district_name, d.avg_price)
}
return map
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.bigdata_third_party_price values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(50000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18])
}
}
println "end: "+new Date()
}
}
@@ -33,11 +33,11 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\指导价\\补充154价格.xlsx"
def filePath = "D:\\数据\\建行\\对标补充数据.xlsx"
// addGXDPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
addFjwPrice(filePath)
addThirdPrice(filePath)
// addFjwPrice(filePath)
// exportPkPrice()
// addRegion()
@@ -190,11 +190,11 @@ class AddPriceTest {
def addThirdPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['M']) {
def dist = CommonDatautils.findDistrictById(d['M'])
if (d['G']) {
def dist = CommonDatautils.findDistrictById(d['G'])
if(dist){
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
d['AN'] = thirdPrice
d['Z'] = thirdPrice
}
}
})
@@ -206,7 +206,7 @@ class AddPriceTest {
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', 202212).append('$lte',202302))
q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def price = null
@@ -216,10 +216,10 @@ class AddPriceTest {
break;
}
if(!price && db.beike_esf_price && db.beike_esf_price>2000){
price = db.beike_esf_price
break
}
// if(!price && db.beike_esf_price && db.beike_esf_price>2000){
// price = db.beike_esf_price
// break
// }
}
return price
}
@@ -256,14 +256,15 @@ class AddPriceTest {
def addFjwPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(!d['F']){
if(d['E'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['E'])
if(!d['W']){
if(d['G'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['G'])
if(dist){
// d['N'] = dist.region
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){
d['F'] = dbo.avg_price
d['W'] = dbo.avg_price
d['X'] = dbo.cal_type
// d['J'] = dbo.avg_price_2b
}
}
@@ -29,11 +29,11 @@ import static com.mongodb.client.model.Sorts.*
public class DbUtils{
public static final String ONLINE_MONGODB_DOMAIN = "mongo.db.fangjia.com";
public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log');
//public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log');
public static final String COMMON_DB = "fangjia";
public static final int COMMON_PORT = 27017;
public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater")
//public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater")
public static DSCollection seaweed = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public static DSCollection SEAWEED_SLAVE = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public static DSCollection seabird = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")