update
This commit is contained in:
+40
-3
@@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
@@ -22,6 +23,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCommbuildtypeBatch(), args);
|
||||
}
|
||||
@@ -29,14 +33,14 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
citys = citys.unique()
|
||||
def rets = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
// def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
|
||||
def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city));
|
||||
// def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city));
|
||||
def distIds = PgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city))
|
||||
def rows = createDatas(cityCodeDatas.get(city), distIds)
|
||||
rets.addAll(rows)
|
||||
println city +" "+rows.size()
|
||||
@@ -50,7 +54,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
|
||||
|
||||
def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
|
||||
FileUtils.batchCreateFile(rets, fileName,headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
println "txt 生成->导Oracle"
|
||||
|
||||
saveToPgSql(rets)
|
||||
|
||||
}
|
||||
|
||||
@@ -103,4 +109,35 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def saveToPgSql(rows){
|
||||
def sqls = []
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << '2023-02-28'
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
def sql = "insert into base.t0831_bigdata_commbuildtype_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
println "start: "+new Date()
|
||||
def index =0
|
||||
sqlInstance.withBatch(20000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
index++
|
||||
println index
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14])
|
||||
}
|
||||
}
|
||||
println "end: "+new Date()
|
||||
}
|
||||
}
|
||||
|
||||
+3
-23
@@ -41,7 +41,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
// citys = ["文昌"]
|
||||
citys = ["上海"]
|
||||
def rows = []
|
||||
|
||||
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
@@ -247,8 +247,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
// def sql = "insert into base.t0831_bigdata_community_gj_h values "
|
||||
def index = 0
|
||||
|
||||
sqlInstance.withBatch(20000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
@@ -256,26 +255,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
ps.toString()
|
||||
}
|
||||
}
|
||||
for(def r : rows){
|
||||
index ++
|
||||
if(index == 1){
|
||||
sql = sql + "("+r[0].toString()+","+r[1].toString()+","+r[2].toString()+")"
|
||||
}else{
|
||||
sql = sql + ",("+")"
|
||||
}
|
||||
}
|
||||
// sqlInstance.withBatch(50000, sql){ ps->
|
||||
// def index = 0
|
||||
// for(def r : rows){
|
||||
// index++
|
||||
// for(def i =0 ;i<r.size();i++){
|
||||
// r[i] = r[i]?r[i]:null
|
||||
// }
|
||||
// ps.addBatch(r[0],r[1],r[2],r[3],r[4],r[5],r[6],r[7],r[8],r[9],r[10],r[11],r[12],r[13],r[14],r[15],r[16],r[17],r[18],r[19],r[20],
|
||||
// r[21],r[22],r[23],r[24],r[25],r[26],r[27],r[28],r[29],r[30],r[31],null,'2023-02-28',null,'2999-12-31',null,null,null)
|
||||
// println index
|
||||
// }
|
||||
// }
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
}
|
||||
|
||||
+37
-37
@@ -69,43 +69,43 @@ class BigdataDealBatch extends BaseBatch{
|
||||
addDealPriceMatch(city);
|
||||
// }
|
||||
}
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData(statMap)
|
||||
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
if(trades){
|
||||
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
rows.addAll(rets)
|
||||
println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成 " + rows.size()
|
||||
// def rows = []
|
||||
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// def index = 0
|
||||
// for(def city : citys){
|
||||
// index ++
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
// def ppeDists = [:]
|
||||
//
|
||||
// def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
// if(city=="上海"){
|
||||
// def youDataTrade = getYouDaData(statMap)
|
||||
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
// rows.addAll(youdaDatas)
|
||||
// println city +" "+youdaDatas.size()
|
||||
// }
|
||||
//
|
||||
// def trades = getCityData(city, statMap, ppeDists)
|
||||
// if(trades){
|
||||
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
// rows.addAll(rets)
|
||||
// println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
//
|
||||
// def cityDatas = []
|
||||
// cityDatas << headers
|
||||
// cityDatas.addAll(rows)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
// }
|
||||
//
|
||||
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
// println "txt 生成 " + rows.size()
|
||||
}
|
||||
|
||||
def createData(def rets, cityCodeDatas, distCodeMap, source){
|
||||
|
||||
+25
-3
@@ -66,6 +66,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
//目前查询库中是否已存在,用的是Oracle
|
||||
def index = 0
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
@@ -74,7 +75,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221218, 20230119);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
@@ -607,7 +608,13 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
rets << rowMap
|
||||
def count = isExistFromHouse(seawater.id)
|
||||
if(count){
|
||||
rets << rowMap
|
||||
}else{
|
||||
println seawater.id +" 已存在"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -803,7 +810,12 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
rets << rowMap
|
||||
def count = isExistFromHouse(seawater.id)
|
||||
if(count){
|
||||
rets << rowMap
|
||||
}else{
|
||||
println seawater.id +" 已存在"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -997,4 +1009,14 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
return distIds;
|
||||
}
|
||||
|
||||
def isExistFromHouse(def houseId){
|
||||
def sql = "SELECT count(FORSALEID) as count FROM bigdata_forsale_gj WHERE FORSALEID = '"+houseId+"'";
|
||||
|
||||
def count = null
|
||||
sqlInstance.eachRow(sql,{it->
|
||||
count = it.count
|
||||
});
|
||||
return count;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ import org.apache.commons.codec.digest.DigestUtils
|
||||
|
||||
class CcbFtpTestUtils {
|
||||
|
||||
static def ftpServer = "192.168.0.86"//45
|
||||
static def ftpServer = "192.168.0.45"//45
|
||||
static def ftpPort = 17495
|
||||
static def ftpUser = "ccb"
|
||||
static def ftpPassword = "ccb!@#"
|
||||
|
||||
+4
-4
@@ -25,9 +25,9 @@ class DealPriceDataProcess {
|
||||
def task = new DealPriceDataProcess()
|
||||
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
|
||||
// task.addLianJiaPrice();
|
||||
// task.deleteThirdPartyData()
|
||||
task.deleteThirdPartyData()
|
||||
// task.delDealByPe()
|
||||
task.delDealData()
|
||||
// task.delDealData()
|
||||
}
|
||||
|
||||
def delDealByPe(){
|
||||
@@ -372,12 +372,12 @@ class DealPriceDataProcess {
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def month = 202211
|
||||
def month = 202212
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def set = new HashSet()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty(city, 202212)
|
||||
def rets = findPeGteThirdParty(city, 202301)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
if(rets){
|
||||
|
||||
+3
-2
@@ -13,6 +13,7 @@ import org.apache.commons.codec.digest.DigestUtils
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
//估值 新版 导出所有数据
|
||||
class ExportOracleAllDataTest {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
@@ -20,7 +21,7 @@ class ExportOracleAllDataTest {
|
||||
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
def citys = null
|
||||
def curDate = "20221228"//DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def curDate = "20220128"//DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportOracleAllDataTest()
|
||||
@@ -38,7 +39,7 @@ class ExportOracleAllDataTest {
|
||||
def run(args) {
|
||||
|
||||
citys = CcbCommonUtils.getRunningCitys(args)//["潍坊","济宁","威海","临沂","绵阳","陇南","塔城","阿勒泰","迪庆州","日喀则"] //CcbCommonUtils.getRunningCitys(args)
|
||||
def curMonth = "202212"//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
def new_dists = []
|
||||
|
||||
+2
-1
@@ -31,7 +31,7 @@ class ExportOracleEvaluationData {
|
||||
exportCityPrice(curMonth)
|
||||
exportRegionPrice(curMonth)
|
||||
|
||||
exportCommunityBasePriceGj(curMonth)//已合下面的合并
|
||||
// exportCommunityBasePriceGj(curMonth)//已合下面的合并
|
||||
exportCommunityBasePrice(curMonth)
|
||||
|
||||
exportCommunityPdescribeGj(curMonth)
|
||||
@@ -248,6 +248,7 @@ class ExportOracleEvaluationData {
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import groovy.sql.Sql
|
||||
|
||||
class PgsqlCommUtils {
|
||||
|
||||
public static def database = "warehouse"
|
||||
|
||||
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/"+database+"?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
public static def findAllDistrictIdsByOracle(def cityid){
|
||||
def sql = "select gujia_comm_id from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0";
|
||||
def set = new HashSet()
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
set << r.gujia_comm_id
|
||||
});
|
||||
return set
|
||||
}
|
||||
}
|
||||
+92
-9
@@ -45,8 +45,91 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
|
||||
def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
def citys = ["山南","黄南州","甘南","克孜勒苏","嘉峪关","海北州","阿拉善盟","怒江州","和田","日喀则","昌都","海南州","玉树州","吐鲁番","阿勒泰","三沙","塔城","甘孜州","果洛州","林芝","那曲","阿坝州","神农架"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\小区地址名单_估家科技V2 (3).xlsx",{d->
|
||||
if(d['J']){
|
||||
def cityId = cityCodeDataMap.get(d['H'])
|
||||
def map = findCcbPriceCredit(sqlInstance236, cityId, d['J'])
|
||||
println d['H'] +" "+d['J']+ map.toString()
|
||||
if(map){
|
||||
d['Q'] = map.credit
|
||||
d['R'] = map.month
|
||||
d['U'] = map.price
|
||||
}
|
||||
// def thirdMap = findThirdPriceBySource(sqlInstance236, cityId, d['J'])
|
||||
// if(thirdMap){
|
||||
// d['S'] = thirdMap.price
|
||||
// d['T'] = thirdMap.month
|
||||
// }
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
def findCcbPriceCredit(sqlInstance, cityid, commid){
|
||||
def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1"
|
||||
def map = [:]
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
map.month = r.year_month
|
||||
map.credit = r.CONFIDENCE
|
||||
map.price = r.avg_price
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
def findThirdPriceBySource(sqlInstance, cityid, commid){
|
||||
def sql = "select * from (select data_source, price_date, community_price,commid from bigdata_third_party_price_gj where cityid='${cityid}' and commid='${commid}' and data_source in ('贝壳','链家') \n" +
|
||||
"order by price_date desc) WHERE ROWNUM <= 1"
|
||||
def map = [:]
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
map.month = r.price_date
|
||||
map.price = r.community_price
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
for(def c : citys){
|
||||
def cityCode = cityCodeDataMap.get(c)
|
||||
println c +" "+cityCode
|
||||
def priceDists = findAllPriceDist(sqlInstance236,cityCode, 202301)
|
||||
//
|
||||
checkNoPriceDist(sqlInstance236, priceDists, cityCode)
|
||||
}
|
||||
|
||||
def checkNoPriceDist(sqlInstance236,priceDists, cityid){
|
||||
def commids = []
|
||||
def sql = "select commid from bigdata_community_gj where cityid='${cityid}' and status=0";
|
||||
sqlInstance236.eachRow(sql, { r ->
|
||||
commids << r.commid
|
||||
})
|
||||
|
||||
for(def commid : commids){
|
||||
if(!priceDists.contains(commid)){
|
||||
updateDistStatus(sqlInstance236, commid)
|
||||
}
|
||||
}
|
||||
return commids
|
||||
}
|
||||
|
||||
|
||||
def updateDistStatus(sqlInstance236, commid){
|
||||
def sql = "update bigdata_community_gj set status=1 where commid='${commid}'";
|
||||
sqlInstance236.executeUpdate(sql)
|
||||
}
|
||||
def findAllPriceDist(sqlInstance236, cityid, yearMonth){
|
||||
def commids = []
|
||||
def sql = "select commid from community_base_price_gj where cityid='${cityid}' and year_month=${yearMonth} ";
|
||||
sqlInstance236.eachRow(sql, { r ->
|
||||
commids << r.commid
|
||||
})
|
||||
return commids
|
||||
}
|
||||
//def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水']
|
||||
//
|
||||
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
@@ -1087,15 +1170,15 @@ def findThirdCommCount(cityid, sqlInstance){
|
||||
//}
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达价格-杭州.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
d['C'] = dist.city
|
||||
d['D'] = dist.region
|
||||
d['E'] = dist.name
|
||||
}
|
||||
})
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达价格-杭州.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// d['C'] = dist.city
|
||||
// d['D'] = dist.region
|
||||
// d['E'] = dist.name
|
||||
// }
|
||||
//})
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\0922入库数据.xlsx"
|
||||
//def datas = ExcelUtils.read(fileName, 0, 1)
|
||||
|
||||
+6
-6
@@ -40,7 +40,7 @@ class TradeStatLog {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.exportTrade()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
@@ -92,13 +92,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys_all;
|
||||
def rets = []
|
||||
rets << ["城市","9月总量","10月总量","11月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","10月总量","11月总量","12月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2022-09-01"
|
||||
def date = "2022-10-01"
|
||||
def endDate = "2022-11-01"
|
||||
def newDate = "2022-12-01"
|
||||
def startDate = "2022-10-01"
|
||||
def date = "2022-11-01"
|
||||
def endDate = "2022-12-01"
|
||||
def newDate = "2023-01-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
|
||||
+25
-14
@@ -1,8 +1,10 @@
|
||||
package com.fangjia.script.ccbevaluation.pgsql
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbFtpUtils
|
||||
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
|
||||
import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
@@ -43,13 +45,11 @@ class ExportPgsqlEvaluationResult {
|
||||
// exportCommunityPdescribeGj(curMonth)
|
||||
// exportCommCaseDetailGj(curMonth)
|
||||
//
|
||||
def curDate = curMonth.toString()+"28"
|
||||
// 生成MD5文件
|
||||
exportMd5()
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||
// ok文件
|
||||
exportOkFile()
|
||||
exportInOk()
|
||||
|
||||
SSHHelperTest jschCommand = new SSHHelperTest();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
|
||||
|
||||
}
|
||||
|
||||
def exportAvmEvaluationGJNB(yearMonth){
|
||||
@@ -363,20 +363,31 @@ class ExportPgsqlEvaluationResult {
|
||||
}
|
||||
|
||||
|
||||
def exportOkFile(){
|
||||
def dataCodeFileName = linux_prefix + "${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
dataCodeFileName = win_prefix + "${curDate}.ok"
|
||||
def exportInOk(){
|
||||
Vector files = CcbFtpTestUtils.listFileNames(null, curDate)
|
||||
Iterator fileVec = files.iterator()
|
||||
def fileRows = []
|
||||
while(fileVec.hasNext()){
|
||||
ChannelSftp.LsEntry file = fileVec.next()
|
||||
|
||||
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
|
||||
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate)
|
||||
}
|
||||
}
|
||||
createFileToFtp("ok txt", [], dataCodeFileName, curDate, false)
|
||||
def fileName = linux_prefix + "/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "${curDate}.ok"
|
||||
}
|
||||
|
||||
createFileToFtp("ok txt",fileRows, fileName, curDate, false)
|
||||
}
|
||||
|
||||
|
||||
def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend);
|
||||
println tableName+" 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
// println tableName+" 上传Ftp完成......"
|
||||
CcbFtpTestUtils.ftpUpload(null, fileName, curDate)
|
||||
println tableName+" 上传Ftp完成......"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+604
@@ -1,14 +1,618 @@
|
||||
package com.fangjia.script.ccbevaluation.pgsql
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbFtpUtils
|
||||
import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.jcraft.jsch.ChannelSftp
|
||||
import groovy.sql.Sql
|
||||
|
||||
class ExportPgsqlRonghe {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
def citys = []
|
||||
|
||||
def database = "base."
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
||||
def linux_prefix = "/ccb_ronghe/${curDate}/"
|
||||
def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\"
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportPgsqlRonghe()
|
||||
task.run(args)
|
||||
}
|
||||
|
||||
def run(args) {
|
||||
def new_dists = []
|
||||
citys = CcbCommonUtils.getRunningCitys(args)
|
||||
//按天导(每天一个文件夹)
|
||||
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date() + 1, -1), "yyyy-MM-dd"), new_dists)
|
||||
if (curDate.toString().endsWith("10")) {
|
||||
|
||||
exportEvaluateRate()
|
||||
exportBigdataBuildprojFjw()
|
||||
exportBigdataCommlayoutpicFjw()
|
||||
|
||||
|
||||
exportBigdataCommunityGJRH()//
|
||||
exportBigdataBuildingFJW()//
|
||||
exportBigdataCommfeeGj()//
|
||||
|
||||
exportBigdataCommbuildstruGJ()//
|
||||
exportBigdataCommhousetypeGJ()//
|
||||
exportBigdataCommbuildyearGj()//
|
||||
exportBigdataCommhouseuseGJ()//
|
||||
exportBigdataPositionGJ()//
|
||||
exportBigdataDistrictFJW()//
|
||||
exportBigdataAliasFJW()//
|
||||
|
||||
|
||||
//本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据
|
||||
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
|
||||
// exportBigdataCityPrice(queryDate)
|
||||
// exportBigdataDistPrice(queryDate)
|
||||
// exportBigdataCommPrice(queryDate)
|
||||
|
||||
exportInOk()
|
||||
|
||||
// }
|
||||
|
||||
// SSHHelperTest jschCommand = new SSHHelperTest();
|
||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataDealFJW2(curDate, dealdate, new_dists){
|
||||
|
||||
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
|
||||
def fileName = "FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
|
||||
def allRows = []
|
||||
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
if(curDate.toString().endsWith("10")){
|
||||
if(new_dists){
|
||||
def newstr = "('"+new_dists.join("','")+"')"
|
||||
def sql1 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " +
|
||||
"from "+database+"t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}"
|
||||
def rets1 = dealData(sql1)
|
||||
|
||||
def sql2 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " +
|
||||
"from "+database+"t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}"
|
||||
def rets2 = dealData(sql2)
|
||||
allRows.addAll(rets1)
|
||||
allRows.addAll(rets2)
|
||||
}else{
|
||||
def sql = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " +
|
||||
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd')";
|
||||
allRows.addAll(dealData(sql))
|
||||
}
|
||||
}
|
||||
}
|
||||
createFileToFtp("BIGDATA_DEAL_FJW", allRows, fileName, curDate, false)
|
||||
}
|
||||
|
||||
|
||||
|
||||
def dealData(def sql){
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
data << r.urbn_id
|
||||
data << r.mdl_id
|
||||
data << r.urbn_nm
|
||||
data << r.gujia_admndiv_id
|
||||
data << r.gujia_admndiv_nm
|
||||
data << r.gujia_str
|
||||
data << r.gujia_hsno
|
||||
data << r.gujia_comm_id
|
||||
data << r.comm_nm
|
||||
data << r.gujia_bld_id
|
||||
data << r.bld_id
|
||||
data << r.gujia_bld_nm
|
||||
data << r.gujia_unit_id
|
||||
data << r.unit_id
|
||||
data << r.gujia_unit_nm
|
||||
data << r.gujia_acc_id
|
||||
data << r.gujia_dtl_adr
|
||||
data << r.gujia_bld_yr_rc
|
||||
data << r.gujia_own_val
|
||||
data << r.gujia_own_yrlmt_val
|
||||
data << r.gujia_prd_val
|
||||
data << r.gujia_wbt_floor
|
||||
data << r.gujia_lyr_tp_val
|
||||
data << r.gujia_tot_floor_num
|
||||
if(r.gujia_bld_tp_val){
|
||||
def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.gujia_bld_stc
|
||||
data << r.gujiaconstructionarea
|
||||
data << r.gujia_set_innr_area
|
||||
data << r.gujia_acc_tp_val
|
||||
data << r.gujia_room
|
||||
data << r.gujia_dept
|
||||
data << r.gujia_ktch_num
|
||||
data << r.gujia_toliet
|
||||
data << r.gujia_hs_use
|
||||
data << r.gujia_indr_stc_val
|
||||
data << r.gujia_twrd_val
|
||||
data << r.gujia_dcrt_sttn
|
||||
data << r.gujia_ases_bt_ln_num
|
||||
data << r.gujia_trdtckt_prc
|
||||
data << r.gujia_mdl_tot_prc
|
||||
data << r.gujia_dcl_tot_prc
|
||||
data << r.gujia_ases_tot_prc
|
||||
data << r.gujiasigntonlinectrid
|
||||
data << r.txn_dt
|
||||
data << r.gujia_txn_tp_val
|
||||
data << r.gujia_sell_pymtmod
|
||||
data << r.gujia_mdl_cyc
|
||||
data << r.gujiawthrext_lift_ind
|
||||
data << r.gujiagvlndfargrdnarea
|
||||
data << r.gujia_gv_lu_tbl_area
|
||||
data << r.gujia_gv_sun_tbl_area
|
||||
data << r.gujiagvbaywindow_area
|
||||
data << r.gujiagvarea_tower_flr
|
||||
data << r.gujia_gv_grg_area
|
||||
data << r.gujiagvundgdroom_area
|
||||
data << r.gujia_gv_othr_area
|
||||
data << r.gujia_hs_src_dtsrc
|
||||
data << r.intr_co_id
|
||||
data << r.intr_co_nm
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
data << r.gujia_mkdam
|
||||
data << r.gujia_data_tm
|
||||
allRows << data
|
||||
});
|
||||
return allRows
|
||||
}
|
||||
|
||||
def exportEvaluateRate(){
|
||||
def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_ceft,gujia_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from t0831_evaluate_rate_fjw_w";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.gujia_ceft_id
|
||||
data << r.gujia_ceft_tp
|
||||
data << r.gujia_ceft
|
||||
data << r.gujia_seq
|
||||
data << r.gujia_urbn
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
allRows << data
|
||||
});
|
||||
def fileName = linux_prefix + "FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
|
||||
createFileToFtp("EVALUATE_RATE_FJW", allRows, fileName, curDate, false)
|
||||
}
|
||||
|
||||
def exportBigdataBuildprojFjw(){
|
||||
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
createFileToFtp("BIGDATA_BUILDPROJ_FJW", [], fileName, curDate, false)
|
||||
}
|
||||
|
||||
def exportBigdataCommlayoutpicFjw(){
|
||||
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
createFileToFtp("BIGDATA_COMMLAYOUTPIC_FJW", [], fileName, curDate, false)
|
||||
}
|
||||
|
||||
def exportBigdataCommunityGJRH(){
|
||||
|
||||
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
def allRows = []
|
||||
for(def city : citys){
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select urbn_id,gujia_comm_id,comm_nm,gujia_comm_fmnm,gujia_admndiv_id,gujia_str,gujia_hsno,gujia_dtl_adr,gujiaocpoflandtotarea,gujia_bld_tot_area,gujia_comm_tp_val,gujiawthr_new_pan_ind,gujia_grnrate,gujia_vol_rate,gujia_park_num,gujia_comm_intd,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujiaprhsld_hspty_prc,gujiacomm_tot_rsd_num,gujiacommrntfee_avprc,gujia_comm_avprc from "+database+"t0831_bigdata_community_fjw_w where urbn_id='${cityCode}'";
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
data << r.gujia_comm_id
|
||||
data << r.comm_nm
|
||||
data << r.gujia_comm_fmnm
|
||||
data << r.gujia_admndiv_id
|
||||
data << r.gujia_str
|
||||
data << r.gujia_hsno
|
||||
data << r.gujia_dtl_adr
|
||||
data << r.gujiaocpoflandtotarea
|
||||
data << r.gujia_bld_tot_area
|
||||
data << r.gujia_comm_tp_val
|
||||
data << r.gujiawthr_new_pan_ind
|
||||
data << r.gujia_grnrate
|
||||
data << r.gujia_vol_rate
|
||||
data << r.gujia_park_num
|
||||
data << r.gujia_comm_intd
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
createFileToFtp("BIGDATA_COMMUNITY_FJW", allRows, fileName, curDate, false)
|
||||
}
|
||||
|
||||
def exportBigdataCommbuildstruGJ(){
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select urbn_id, gujia_comm_id, gujia_bld_stc, rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+"t0831_bigdata_commbuildstru_fjw_w where urbn_id='${cityCode}'";
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
data << r.gujia_comm_id
|
||||
data << r.gujia_bld_stc
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
createFileToFtp("BIGDATA_COMMBUILDSTRU_FJW", allRows, fileName, curDate, false)
|
||||
|
||||
}
|
||||
|
||||
def exportBigdataCommfeeGj(){
|
||||
|
||||
def allRows = []
|
||||
for(def city : citys){
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select urbn_id,gujia_comm_id,co_id,gujia_eps_val,gujia_hs_use,gujia_bld_tp_val,gujia_comm_eps_tp,gujia_eps_cmnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commfee_fjw_w where urbn_id='${cityCode}'";
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
data << r.gujia_comm_id
|
||||
data << r.co_id
|
||||
data << r.gujia_eps_val
|
||||
data << r.gujia_hs_use
|
||||
if(r.gujia_bld_tp_val){
|
||||
def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.gujia_comm_eps_tp
|
||||
data << r.gujia_eps_cmnt
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
createFileToFtp("BIGDATA_COMMFEE_FJW", allRows, fileName, curDate, false)
|
||||
}
|
||||
|
||||
def exportBigdataCommhousetypeGJ(){
|
||||
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select urbn_id,gujia_comm_id,gujia_comm_hs_tp,rmrk_dsc,to_char(data_crt_tm'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commhousetype_fjw_w where urbn_id='${cityCode}'";
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
data << r.gujia_comm_id
|
||||
data << r.gujia_comm_hs_tp
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
createFileToFtp("BIGDATA_COMMHOUSETYPE_FJW", allRows, fileName, curDate, false)
|
||||
}
|
||||
|
||||
def exportBigdataCommbuildyearGj(){
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select urbn_id,gujia_comm_id,gujia_bld_yr_rc,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commbuildyear_fjw_w where urbn_id='${cityCode}'";//
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
data << r.gujia_comm_id
|
||||
data << r.gujia_bld_yr_rc
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
createFileToFtp("BIGDATA_COMMBUILDYEAR_FJW", allRows, fileName, curDate, false)
|
||||
}
|
||||
|
||||
def exportBigdataCommhouseuseGJ(){
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select urbn_id,gujia_comm_id,gujia_hs_use,gujia_ladder_accnum,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commhouseuse_fjw_w where urbn_id='${cityCode}'";//
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
data << r.gujia_comm_id
|
||||
data << r.gujia_hs_use
|
||||
data << r.gujia_ladder_accnum
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
createFileToFtp("BIGDATA_COMMHOUSEUSE_FJW", allRows, fileName, curDate, false)
|
||||
}
|
||||
|
||||
def exportBigdataPositionGJ(){
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select urbn_id,gujia_obj_tp,gujia_obj_id,gujia_lo_tp,gujia_sn,lgt,ltt,gujia_hignas_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_position_fjw_w where urbn_id='${cityCode}'";//
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
data << r.gujia_obj_tp
|
||||
data << r.gujia_obj_id
|
||||
data << r.gujia_lo_tp
|
||||
data << r.gujia_sn
|
||||
data << r.lgt
|
||||
data << r.ltt
|
||||
data << r.gujia_hignas_val
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
createFileToFtp("BIGDATA_POSITION_FJW", allRows, fileName, curDate, false)
|
||||
}
|
||||
|
||||
def exportBigdataDistrictFJW(){
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_district_fjw_w where urbn_id='${cityCode}'";//
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.gujia_admndiv_id
|
||||
data << r.gujia_admndiv_nm
|
||||
data << r.gujia_admndiv_nm_cpa
|
||||
data << r.urbn_id
|
||||
data << r.gujia_natstd_cd
|
||||
data << r.gujia_dspl_seq
|
||||
data << r.gujia_admndiv_cgy
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
createFileToFtp("BIGDATA_DISTRICT_FJW", allRows, fileName, curDate, false)
|
||||
}
|
||||
|
||||
def exportBigdataAliasFJW(){
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select urbn_id,rel_id,gujia_comm_id,comm_nm,gujia_pcsst,gujia_vld_dt,gujia_vrsn_dsc,gujia_wthr_prim_nm,gujia_dtsrc,gujia_dsc,gujia_als_lvl,gujia_adr_tp_ecd,gujia_wthr_adt_ind,gujia_wthr_dspl_ind from "+database+"t0831_address_alias_fjw_w where urbn_id='${cityCode}'";//
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
data << r.rel_id
|
||||
data << r.gujia_comm_id
|
||||
data << r.comm_nm
|
||||
data << r.gujia_pcsst
|
||||
data << r.gujia_vld_dt
|
||||
data << r.gujia_vrsn_dsc
|
||||
data << r.gujia_wthr_prim_nm
|
||||
data << r.gujia_dtsrc
|
||||
data << r.gujia_dsc
|
||||
data << r.gujia_als_lvl
|
||||
data << r.gujia_adr_tp_ecd
|
||||
data << r.gujia_wthr_adt_ind
|
||||
data << r.gujia_wthr_dspl_ind
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
def fileName = linux_prefix + "FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
createFileToFtp("ADDRESS_ALIAS_FJW", allRows, fileName, curDate, false)
|
||||
}
|
||||
|
||||
def exportBigdataBuildingFJW(){
|
||||
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
def index = 0
|
||||
for(def city : citys) {
|
||||
def allRows = []
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select urbn_id,gujia_bld_id,bld_id,gujia_bld_nm,gujia_bld_als_dsc,gujia_admndiv_id,gujia_comm_id,gujia_bld_yr_rc,gujia_bld_tp_val,gujia_bld_stc,floor_num,gujia_unit_num,gujia_accnum_val,gujiawthrext_lift_ind,gujia_lift_num,gujiajiladderjiactval,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_vld_ind from "+database+"t0831_bigdata_building_fjw_w where urbn_id='${cityCode}'";//
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
data << r.gujia_bld_id
|
||||
data << r.bld_id
|
||||
data << r.gujia_bld_nm
|
||||
data << r.gujia_bld_als_dsc
|
||||
data << r.gujia_admndiv_id
|
||||
data << r.gujia_comm_id
|
||||
data << r.gujia_bld_yr_rc
|
||||
if(r.gujia_bld_tp_val){
|
||||
def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.gujia_bld_stc
|
||||
data << r.floor_num
|
||||
data << r.gujia_unit_num
|
||||
data << r.gujia_accnum_val
|
||||
data << r.gujiawthrext_lift_ind
|
||||
data << r.gujia_lift_num
|
||||
data << r.gujiajiladderjiactval
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
data << r.gujia_vld_ind
|
||||
allRows << data
|
||||
});
|
||||
|
||||
if(allRows && allRows.size()>0){
|
||||
index = index + 1
|
||||
if(index == 1){
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
}else{
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCityPrice(queryDate){
|
||||
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
// def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";//
|
||||
def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from t0831_city_price_gj_h where CITYID='${cityCode}'";//
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.YMID
|
||||
data << r.SALEPRICE?r.SALEPRICE:''
|
||||
data << r.RENTPRICE?r.RENTPRICE:''
|
||||
if(r.SALECOUNT>0) {
|
||||
data << r.SALECOUNT
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
if(r.RENTCOUNT>0){
|
||||
data << r.RENTCOUNT
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.UPDATETIME
|
||||
data << r.HOUSEUSE
|
||||
data << r.SALEBLDGAREA?r.SALEBLDGAREA:''
|
||||
data << r.RENTBLDGAREA?r.RENTBLDGAREA:''
|
||||
data << r.SALETOTALPRICE?r.SALETOTALPRICE:''
|
||||
data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:''
|
||||
data << r.DATATYPE
|
||||
data << r.SDATERANGE
|
||||
data << r.RDATERANGE
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
createFileToFtp("BIGDATA_CITYPRICE", allRows, fileName, curDate, false)
|
||||
}
|
||||
|
||||
def exportInOk(){
|
||||
Vector files = CcbFtpTestUtils.listFileNames("rh", curDate)
|
||||
Iterator fileVec = files.iterator()
|
||||
def fileRows = []
|
||||
while(fileVec.hasNext()){
|
||||
ChannelSftp.LsEntry file = fileVec.next()
|
||||
|
||||
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
|
||||
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength("rh", file.getFilename(), curDate)
|
||||
}
|
||||
}
|
||||
def fileName = linux_prefix+"${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = win_prefix + "${curDate}.ok"
|
||||
}
|
||||
createFileToFtp("融合统计文件长度ok文件", fileRows, fileName, curDate, false)
|
||||
}
|
||||
|
||||
def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend);
|
||||
println tableName+" 数据文件输出完成......"
|
||||
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
|
||||
println tableName+" 上传Ftp完成......"
|
||||
}
|
||||
}
|
||||
|
||||
+703
@@ -0,0 +1,703 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class HailanTest {
|
||||
|
||||
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
static{
|
||||
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
|
||||
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
|
||||
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
}
|
||||
|
||||
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new HailanTest();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def inputfile = "D:\\数据\\海南\\统计.xlsx"
|
||||
// match(inputfile)
|
||||
// addPrice(inputfile)
|
||||
// addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx")
|
||||
// addHouse(inputfile)
|
||||
// stat(inputfile)
|
||||
hailanStat(inputfile)
|
||||
}
|
||||
|
||||
def addHouse(inputfile){
|
||||
|
||||
def datas = []
|
||||
datas << ["类型(住宅/别墅)","行政区划","详细地址","房号","小区名称","建筑面积","所在楼层","详细地址","小区名","交易日居住用房价格指数","评估日居住用房价格指数","是否有电梯","所在楼层","总楼层","内部装修","平面布置","朝向",
|
||||
"景观","建筑结构","建筑面积(平方米)","房屋可使用年限","房屋已使用年限","评估价(单价)","挂牌价(单价)","成交价(单价)","小区/楼盘均价(单价)"]
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify(inputfile, { row ->
|
||||
if(row['L']){
|
||||
index ++
|
||||
def dist = CommonDatautils.findDistrict(row['H'], row['I'], row['J'], "district")
|
||||
def rets = findHouses(row['H'], row['I'], "sell", 20200101,20230113, row['J'], null, row['k'])
|
||||
println index +" "+ row['C']+" "+ rets.size()
|
||||
if(rets){
|
||||
def lastRets = []
|
||||
for(def r : rets){
|
||||
def fjwPrice = row['M']
|
||||
if(fjwPrice){
|
||||
def precent = java.lang.Math.abs((fjwPrice-r.avg_price)*100/r.avg_price)
|
||||
if(precent>=10){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(row['A'] && row['A']=="别墅"){
|
||||
def inputAare = row['F'].toString().toDouble()
|
||||
def minArea = inputAare + 50
|
||||
def maxArea = inputAare + 100
|
||||
if(r.area<minArea && r.area>maxArea){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
lastRets << r
|
||||
if(lastRets.size()>=3){
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if(lastRets){
|
||||
for(def lr : lastRets){
|
||||
def data = []
|
||||
data << row['A']
|
||||
data << row['B']
|
||||
data << row['C']
|
||||
data << row['D']
|
||||
data << row['E']
|
||||
data << row['F']
|
||||
data << row['G']
|
||||
if(dist){
|
||||
data << dist.address
|
||||
}else{
|
||||
data << ""
|
||||
}
|
||||
if(dist){
|
||||
data << dist.name
|
||||
}else{
|
||||
data << ""
|
||||
}
|
||||
|
||||
data << ""
|
||||
data << ""
|
||||
if(lr.height){
|
||||
if(lr.height>6){
|
||||
data << "是"
|
||||
}else{
|
||||
data << "否"
|
||||
}
|
||||
}else{
|
||||
data << ""
|
||||
}
|
||||
data << lr.floor
|
||||
data << lr.height
|
||||
data << lr.fitment
|
||||
data << ""
|
||||
data << lr.direction
|
||||
data << ""
|
||||
data << lr.build_type
|
||||
data << lr.area
|
||||
data << dist.property_life
|
||||
// def year = dist.create_year
|
||||
// if(year){
|
||||
// data << getHouseYear(year.toInteger())//已使用年限
|
||||
// }else{
|
||||
// data << ""//已使用年限
|
||||
// }
|
||||
data << getHouseYear(dist.create_year , lr.build_date, dist.build_date)
|
||||
|
||||
data << getEvalutaionPrice(lr, dist)//评估价(单价)
|
||||
data << lr.avg_price
|
||||
data << getDealPrice(dist)//成交单价
|
||||
data << row['M']
|
||||
datas << data
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
ExcelUtils.write("D:\\数据\\海南\\海南样本数据.xlsx", datas, "样本数据")
|
||||
}
|
||||
|
||||
def getDealPrice(dist){
|
||||
def json = OpenApiClient.searchDealPrice(dist._id.toString(),"20220101", 100)
|
||||
def prices = []
|
||||
if(json && json.code==200){
|
||||
def result = json.result
|
||||
if(result){
|
||||
for(def r : result){
|
||||
prices << r.avgPrice
|
||||
}
|
||||
}
|
||||
}
|
||||
def dealAvgPrice = null
|
||||
if(prices){
|
||||
dealAvgPrice = (prices.sum()/prices.size()).toInteger()
|
||||
}
|
||||
return dealAvgPrice;
|
||||
}
|
||||
|
||||
def getHouseYear(create_year , house_build_date, dist_build_date){
|
||||
def year = null
|
||||
if(house_build_date){
|
||||
year = getYear(house_build_date)
|
||||
|
||||
}else if(dist_build_date){
|
||||
year = getYear(dist_build_date)
|
||||
}else if(create_year){
|
||||
year = create_year
|
||||
}
|
||||
if(year){
|
||||
year = year.toInteger()
|
||||
def curYear = new Date().getYear()+1900
|
||||
def modYear = curYear - year
|
||||
return modYear.toInteger().toString()
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
def getYear(finishBuildingDate){
|
||||
if(finishBuildingDate){
|
||||
Pattern pat3 = Pattern.compile("\\d{4}");
|
||||
def matcher3 = pat3.matcher(finishBuildingDate);
|
||||
if(matcher3.find()){
|
||||
return matcher3.group()
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def getEvalutaionPrice(house, dist){
|
||||
def params = [:]
|
||||
params.put("city", dist.city);
|
||||
params.put("district", dist.region);
|
||||
params.put("name", dist.name);
|
||||
params.put("size", String.valueOf(house.area));
|
||||
if(house.height) {
|
||||
params.put("maxFloor", dist.house_height.toString());
|
||||
params.put("direction", getDirectionCode(house.direction).toString());//朝向
|
||||
if (house.house_floor) {
|
||||
params.put("floor", house.house_floor.toString());
|
||||
}
|
||||
}
|
||||
def fitmentCode = null
|
||||
if(house.fitment){
|
||||
fitmentCode = getFitmentCode(house.fitment)
|
||||
params.put("decoration", fitmentCode.toString());//装修
|
||||
}
|
||||
params.put("scope", "1")
|
||||
def result = OpenApiClient.evaluateDistrictPrice(params);
|
||||
|
||||
if(result){
|
||||
return result.avgPrice
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def getDirectionCode(def s){
|
||||
def directMap = ["西南":1, "东南":2, "南":3, "西北":4, "东北":5, "东":6, "西":7, "北":8, "东西":9, "南北":10]
|
||||
return directMap.get(s);
|
||||
}
|
||||
def getFitmentCode(def s){
|
||||
if(s){
|
||||
if(["毛坯"].contains(s)){
|
||||
return 1;
|
||||
}else if(["简单装修","简装"].contains(s)){
|
||||
return 2;
|
||||
}else if(["精装修","精装"].contains(s)){
|
||||
return 3;
|
||||
}else if(["豪华装修","豪装"].contains(s)){
|
||||
return 4;
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
|
||||
def rets = []
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(5);
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
|
||||
if(estate_type2 && estate_type2=="普通住宅"){
|
||||
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
// def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||
// if(vallaFlag){
|
||||
// continue;
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
if(!seawater.total_price){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.avg_price){
|
||||
continue;
|
||||
}
|
||||
|
||||
// if(!seawater.height){
|
||||
// seawater.height = null
|
||||
// }
|
||||
|
||||
if(seawater.height && seawater.height>60){
|
||||
continue;
|
||||
}
|
||||
|
||||
// if(!seawater.room){
|
||||
// continue;
|
||||
// }
|
||||
// if(!seawater.room){
|
||||
// seawater.room = null
|
||||
// }
|
||||
|
||||
if(seawater.room){
|
||||
if(seawater.room>=11 || seawater.room<1){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.hall){
|
||||
seawater.hall = null
|
||||
}
|
||||
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.toilet){
|
||||
seawater.toilet = null
|
||||
}
|
||||
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
def floor = parseFloor(seawater.floor_info)
|
||||
if(seawater.height && floor && floor.isNumber()){
|
||||
if(floor.toInteger()>seawater.height){
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!floor){
|
||||
seawater.floor = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
|
||||
rowMap.houseId = seawater.id
|
||||
// rowMap.city = CcbCommonUtils.getCcbCity(city)
|
||||
//rowMap.region =seawater.region
|
||||
//rowMap.address = seawater.address
|
||||
rowMap.area = seawater.area
|
||||
rowMap.address = seawater.address
|
||||
rowMap.total_price = seawater.total_price
|
||||
rowMap.avg_price = seawater.avg_price
|
||||
rowMap.build_type = seawater.build_type
|
||||
rowMap.build_date = seawater.build_date
|
||||
rowMap.district_name = seawater.district_name
|
||||
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||
if(rowMap.height && rowMap.height<1){
|
||||
rowMap.height = null
|
||||
}
|
||||
rowMap.floor = floor=="null"?null:floor
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
|
||||
|
||||
def direction = parseDirection(seawater.direction)
|
||||
rowMap.direction = direction=="null"?null:direction
|
||||
|
||||
def fitment = parseFitment(seawater.fitment)
|
||||
rowMap.fitment = fitment=="null"?null:fitment
|
||||
|
||||
|
||||
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
}
|
||||
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
|
||||
|
||||
rets << rowMap
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
static def parseFloor(def floorDesc){
|
||||
if(floorDesc){
|
||||
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
|
||||
if(floorDesc){
|
||||
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
|
||||
Matcher matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
|
||||
matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
|
||||
// return "中层"
|
||||
// }
|
||||
//
|
||||
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
|
||||
// return "高层"
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
|
||||
// return "低层"
|
||||
// }
|
||||
if(floorDesc.isNumber()){
|
||||
return floorDesc;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static String parseDirection(String directionDesc) {
|
||||
if (directionDesc == null) {
|
||||
return null
|
||||
}
|
||||
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
|
||||
StringBuffer sb = new StringBuffer()
|
||||
for (int i = 0; i < directionDesc.length(); i++) {
|
||||
String c = directionDesc.substring(i, i + 1)
|
||||
if (DIRECTIONS.contains(c)) {
|
||||
if (directionDesc.substring(i).startsWith(c + "#_#")) {
|
||||
sb.append(c + " ")
|
||||
} else {
|
||||
sb.append(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().startsWith(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().contains(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static String parseFitment(String fitmentDesc) {
|
||||
if (fitmentDesc == null) {
|
||||
return null
|
||||
}
|
||||
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
|
||||
if (fitmentDesc.equals("非毛坯")) {
|
||||
return null
|
||||
}
|
||||
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
|
||||
.entrySet().iterator()
|
||||
Map.Entry<String, List<String>> entry
|
||||
while (it.hasNext()) {
|
||||
entry = it.next()
|
||||
if (fitmentDesc.contains(entry.getKey())) {
|
||||
return entry.getKey()
|
||||
}
|
||||
for (String fitmentKey : entry.getValue()) {
|
||||
if (fitmentDesc.contains(fitmentKey)) {
|
||||
return entry.getKey()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def addCommId(inputfile){
|
||||
ExcelUtils.modify(inputfile, { row ->
|
||||
if(row['H']){
|
||||
def dist = CommonDatautils.findDistrict(row['H'], row['I'], row['J'], "district")
|
||||
if(dist){
|
||||
row['L'] = dist._id.toString()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def commid = row['L']
|
||||
if(commid){
|
||||
def dist = CommonDatautils.findDistrictById(commid.trim())
|
||||
if(dist){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(priceObj){
|
||||
row['M'] = priceObj.avg_price
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def match(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['C']){
|
||||
String city = row['H']//RegionData.formatCity(row['B'])
|
||||
if(city){
|
||||
def address = row['C']
|
||||
// def region = row['C']
|
||||
println "[match] city:"+city+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, row['E'])
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['I'] = jsonObject.get("district")
|
||||
row['J'] = jsonObject.get("name")
|
||||
row['K'] = jsonObject.get("propertyType")
|
||||
row['L'] = jsonObject.get("id")
|
||||
}
|
||||
}else{
|
||||
println row['C']
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def stat(path){
|
||||
ExcelUtils.modify(path, { row ->
|
||||
if(row['L'] && !row['H']){
|
||||
def city = row['L']
|
||||
def region = row['M']
|
||||
def dists = null
|
||||
if(city && region){
|
||||
dists = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias","estate_type2"])
|
||||
}else{
|
||||
dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias","estate_type2"])
|
||||
}
|
||||
def distLists = []
|
||||
def villaLists = []
|
||||
for(def d : dists){
|
||||
if(d.estate_type2=="别墅"){
|
||||
villaLists << d
|
||||
}else{
|
||||
distLists << d
|
||||
}
|
||||
}
|
||||
row['J'] = dists.size()
|
||||
|
||||
def haveHouseList = findHaveHouseDist(distLists)
|
||||
def haveVillaList = findHaveHouseDist(villaLists)
|
||||
row['H'] = haveHouseList
|
||||
row['I'] = haveVillaList
|
||||
// row['N'] = distLists.size()
|
||||
// row['O'] = villaLists.size()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def hailanStat(path){
|
||||
def total = 0
|
||||
def distTotal = 0
|
||||
def villaTotal = 0
|
||||
def haveHouseDistTotal = 0
|
||||
def haveHouseVillaTotal = 0
|
||||
def buildStructureTotal = 0
|
||||
// ExcelUtils.modify(path, { row ->
|
||||
// if (row['L'] && !row['K']) {
|
||||
//"海口","三亚","三沙","儋州",
|
||||
def citys = ["五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
|
||||
for(def city : citys) {
|
||||
|
||||
def dists = null
|
||||
// if (city && region) {
|
||||
// dists = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city", "region", "name", "alias", "estate_type2", "build_structure"])
|
||||
// } else {
|
||||
dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias", "estate_type2", "build_structure"])
|
||||
// }
|
||||
def distLists = []
|
||||
def villaLists = []
|
||||
for (def d : dists) {
|
||||
if (d.estate_type2 == "别墅") {
|
||||
villaLists << d
|
||||
} else {
|
||||
distLists << d
|
||||
}
|
||||
}
|
||||
total = total + dists.size()
|
||||
|
||||
// def structureCount = 0
|
||||
// for(def d : dists){
|
||||
// if(d.build_structure){
|
||||
// structureCount ++
|
||||
// }
|
||||
// }
|
||||
// println structureCount
|
||||
// row['K'] = structureCount
|
||||
// buildStructureTotal = buildStructureTotal + structureCount
|
||||
//
|
||||
def haveHouseList = findHaveHouseDist(distLists)
|
||||
def haveVillaList = findHaveHouseDist(villaLists)
|
||||
haveHouseDistTotal = haveHouseDistTotal + haveHouseList
|
||||
haveHouseVillaTotal = haveHouseVillaTotal + haveVillaList
|
||||
distTotal = distTotal + distLists.size()
|
||||
villaTotal = villaTotal + villaLists.size()
|
||||
// println c +" "+total+" "+buildStructureTotal
|
||||
// }
|
||||
}
|
||||
// })
|
||||
|
||||
println total +" "+haveHouseDistTotal+" "+haveHouseVillaTotal+" "+distTotal+" "+villaTotal+" "+buildStructureTotal
|
||||
|
||||
}
|
||||
|
||||
def findHaveHouseDist(districts){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println "小区量:"+districts.size()
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def dist = districts[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def ret = findHouses(dist.city, dist.region, "sell", 20210101, null, dist.name, dist.alias, dist.estate_type2)
|
||||
if(!ret){
|
||||
ret = findThirdPrice(dist)
|
||||
}
|
||||
if(ret && ret.size()>0){
|
||||
rets.add(dist)
|
||||
}
|
||||
println totalCount.get() +" "+ ret.size()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return rets.size()
|
||||
}
|
||||
|
||||
|
||||
def findThirdPrice(dist){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
q.put("name", dist.name)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202201))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q)
|
||||
def prices = []
|
||||
if(dbcursor){
|
||||
for(def d : dbcursor){
|
||||
def set = d.keySet();
|
||||
Iterator it = set.iterator()
|
||||
while(it.hasNext()){
|
||||
def key = it.next()
|
||||
if(key && key.contains("_price") && key!="fangjia_price" && key!="last_price"){
|
||||
def price = d[key]
|
||||
if(price){
|
||||
prices << price
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return prices;
|
||||
}
|
||||
}
|
||||
+328
@@ -0,0 +1,328 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
|
||||
class MingShenPocStat {
|
||||
|
||||
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
private static final DBCollection district_sell_stat_coll = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo);
|
||||
|
||||
static main(args) {
|
||||
def task = new MingShenPocStat()
|
||||
// task.run()
|
||||
task.officeRun()
|
||||
}
|
||||
|
||||
def officeRun(){
|
||||
def path = "D:\\数据\\民生\\抽样-写字楼类.xlsx"
|
||||
def datas = []
|
||||
ExcelUtils.modify(path, { d ->
|
||||
if(d['B']){
|
||||
|
||||
def office = CommonDatautils.findDistrictById(d['C'])
|
||||
if(office){
|
||||
def alias = []
|
||||
alias << office.name
|
||||
if(office.alias){
|
||||
def nameAlias = office.alias
|
||||
for(def o : nameAlias){
|
||||
if(o){
|
||||
alias << o
|
||||
}
|
||||
}
|
||||
}
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
def houses = findOfficeHouseCount(office.city, office.region, aliasStr)
|
||||
if(houses){
|
||||
if(houses.size()>=2){
|
||||
houses = houses[0..1]
|
||||
}
|
||||
for(def h : houses){
|
||||
def obj = findOfficeAvgPrice(office)
|
||||
def rets = []
|
||||
rets << d['A']
|
||||
rets << d['B']
|
||||
rets << ''
|
||||
rets << h.region
|
||||
rets << office.address
|
||||
rets << h.area
|
||||
rets << h.floor
|
||||
rets << h.height
|
||||
def level = office.bed_kind
|
||||
rets << level
|
||||
rets << h.fitment
|
||||
def date = office.finish_building_date
|
||||
if(date){
|
||||
rets << date.substring(0, 4)
|
||||
}else{
|
||||
rets << ""
|
||||
}
|
||||
|
||||
rets << ""
|
||||
rets << ""
|
||||
def price = ""
|
||||
if(obj && obj.avg_price){
|
||||
price = obj.avg_price
|
||||
rets << obj.avg_price
|
||||
}else{
|
||||
rets << ""
|
||||
}
|
||||
|
||||
if(price){
|
||||
rets << (price*h.area).toInteger()
|
||||
}
|
||||
datas << rets
|
||||
}
|
||||
}
|
||||
}else{
|
||||
def rets = []
|
||||
rets << d['A']
|
||||
rets << d['B']
|
||||
datas << rets
|
||||
}
|
||||
|
||||
// def districts = CommonDatautils.findAllDistrictByCity(d['C'],"office",["city", "region", "name", "alias","address","bed_kind","finish_building_date"])
|
||||
|
||||
}
|
||||
})
|
||||
|
||||
ExcelUtils.write("D:\\数据\\民生\\抽样-写字楼类-样本.xlsx", datas, "样本")
|
||||
}
|
||||
|
||||
def run(){
|
||||
def path = "D:\\数据\\民生\\样本字段覆盖率.xlsx"
|
||||
// def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Downloads\\二期估值数据每周统计\\二期估值数据每周统计\\估值数据统计结果20230112.xlsx", 0 , 1)
|
||||
//
|
||||
// ExcelUtils.modify(path, { row ->
|
||||
// if(row['C']) {
|
||||
// def total = 0
|
||||
// def pre = 0
|
||||
// for(def d : datas){
|
||||
// if(row['C']==d['A']){
|
||||
// total = d['B']
|
||||
// pre = d['H']
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// if(total){
|
||||
// def c = (total*pre/100).toInteger()
|
||||
// row['E'] = c
|
||||
// row['F'] = pre+"%"
|
||||
// }
|
||||
// }
|
||||
// })
|
||||
|
||||
def totalcount = findHouseTotal("大连")
|
||||
def count = findHouseCount("大连", "height")
|
||||
|
||||
println DataHandleUtils.getDoubleRound(2, count*100/totalcount)
|
||||
|
||||
// ExcelUtils.multiThreadingModify(path, { row ->
|
||||
// if(row['C']) {
|
||||
//def districts = CommonDatautils.findAllDistrictByCity(row['C'],"district",["city", "region", "name", "alias"])
|
||||
// def districts = CommonDatautils.findDistrictByCityAndRegions("大连","district",["旅顺口"], ["city", "region", "name", "alias"])
|
||||
// def total = districts.size()
|
||||
// if(total){
|
||||
// def house_dist_count = findHouseStat(districts)
|
||||
// def pre = house_dist_count*100/total
|
||||
// pre = DataHandleUtils.getDoubleRound(2, pre)
|
||||
// println total+" "+house_dist_count+" "+pre
|
||||
// }
|
||||
// def count = findHouseCount(row['C'])
|
||||
// row['D'] = count
|
||||
// println row['C']+" "+count
|
||||
// row['H'] = findDistrictSellStat(row['C'])
|
||||
// row['G'] = getNewDist(row['C'])
|
||||
|
||||
// def totalcount = findHouseTotal(row['C'])
|
||||
|
||||
// def total = row['Z']
|
||||
// def count = findHouseCount(row['C'], "build_date")
|
||||
// row['Q'] = DataHandleUtils.getDoubleRound(2, count*100/total)
|
||||
// println count +" "+row['Q']
|
||||
// def obj = getFieldPrecent(row['C'])
|
||||
// if(obj){
|
||||
// def detail = obj.district_detail
|
||||
// row['E'] = detail.alias.toDouble()*100
|
||||
// row['F'] = detail.address.toDouble()*100
|
||||
// row['G'] = detail.lng.toDouble()*100
|
||||
// row['H'] = detail.buildType.toDouble()*100
|
||||
// row['I'] = detail.outPics.toDouble()*100
|
||||
// row['J'] = detail.developer.toDouble()*100
|
||||
// row['Q'] = detail.buildDate.toDouble()*100
|
||||
// row['R'] = detail.buildDate.toDouble()*100
|
||||
// }
|
||||
// }
|
||||
// });
|
||||
}
|
||||
|
||||
def getFieldPrecent(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
def rets = CITY_INFO_CONN.find(q).sort(new BasicDBObject('weekend', -1))
|
||||
if(rets){
|
||||
return rets[0]
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def getNewDist(city){
|
||||
String sql = "select count(*) as count from community where city='${city}' and source='贝壳找房-新房' "
|
||||
def count = null
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
count = data.count
|
||||
})
|
||||
return count;
|
||||
}
|
||||
|
||||
def findDistrictSellStat(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
def count = district_sell_stat_coll.count(q)
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
def findHouseStat(districts){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println "小区量:"+districts.size()
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def ret = findHouses(rMap.city, rMap.region, "sell", 20210101,rMap.name, rMap.alias)
|
||||
if(ret && ret.size()>0){
|
||||
rets.add(rMap)
|
||||
}
|
||||
println totalCount.get() +" "+ ret.size()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return rets.size()
|
||||
}
|
||||
|
||||
def findHouseTotal(city){
|
||||
def count = 0
|
||||
ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date >= 20220101 and city='${city}'", {
|
||||
count = it.count
|
||||
println("$city\t${it.count}")
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
def findHouseCount(city, field){
|
||||
def count = 0
|
||||
def sql = "select count(*) as count from seawater where city='${city}' and cat='sell' and "+field+" is not null"
|
||||
ElasticUtils.executeSqlQuery(sql, {
|
||||
count = it.count
|
||||
println("$city\t${it.count}")
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
|
||||
def rets = []
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
return houses;
|
||||
}
|
||||
|
||||
def findOfficeAvgPrice(office){
|
||||
def sql = "select * from office where city='${office.city}' and region='${office.region}' and name='${office.name}'"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql, {d->
|
||||
rets << d
|
||||
})
|
||||
if(rets){
|
||||
def obj = rets[0]
|
||||
return obj
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
def findOfficeHouseCount(city, region, names){
|
||||
def sql = "select * from seawater where city='${city}' and region='${region}' and cat='office_sell' and district_name in ('${names}') and root='58.com'"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql, {d->
|
||||
rets << d
|
||||
})
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 转义字符串中的特殊字符
|
||||
* 仅过滤Lucene regexps, 其他的勿用
|
||||
* @param keyword
|
||||
* @return
|
||||
*/
|
||||
public static String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
}
|
||||
+459
@@ -0,0 +1,459 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.BaiduMapUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
class MingShenPocTest {
|
||||
|
||||
def total = 5000
|
||||
static main(args) {
|
||||
def task = new MingShenPocTest()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
def path = "D:\\数据\\民生\\poc测试.xlsx"
|
||||
//9级标准化
|
||||
//structureV2(path)//针对海南poc地址数据此接口不合适使用
|
||||
//省市区拆分
|
||||
// ner(path)//城市,区域带市带区
|
||||
// match(path)
|
||||
// addPrice(path)
|
||||
// addLatLng(path)
|
||||
// addAroundPrice(path)
|
||||
// addAuthor(path)
|
||||
addHouse(path)
|
||||
}
|
||||
|
||||
def structureV2(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['A']){
|
||||
def jsonObj = FangjiaClientUtils.addressStructure(null, row['A'])
|
||||
if(jsonObj && jsonObj.result){
|
||||
|
||||
if(jsonObj.result.province){
|
||||
row['B'] = jsonObj.result.province
|
||||
}
|
||||
|
||||
if(jsonObj.result.city){
|
||||
def cityStr = null
|
||||
if(jsonObj.result.city=="市辖区"){
|
||||
cityStr = jsonObj.result.province
|
||||
}else{
|
||||
cityStr = jsonObj.result.city
|
||||
}
|
||||
row['C'] = RegionData.formatCity(cityStr)
|
||||
}
|
||||
|
||||
if(jsonObj.result.district){
|
||||
row['D'] = jsonObj.result.district
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def ner(path){
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(d['A'] && !d['B']) {
|
||||
def json = FangjiaClientUtils.addressStand(null, d['A'])
|
||||
if(json && json.entities){
|
||||
def entities = json.entities
|
||||
for(def m : entities){
|
||||
if(m.type=="prov"){
|
||||
d['B'] = m.word
|
||||
}
|
||||
if(m.type=="city"){
|
||||
def cityStr = m.word
|
||||
if(cityStr){
|
||||
d['C'] = cityStr//RegionData.formatCity(cityStr)
|
||||
}
|
||||
}
|
||||
if(m.type=="dist") {
|
||||
if(d['C']){
|
||||
d['D'] = m.word //RegionData.formatRegion(d['C'], m.word)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
//可以根据测试数据添加 推荐楼盘名
|
||||
def match(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['C'] && !row['E']){
|
||||
def city = RegionData.formatCity(row['C'])
|
||||
def region = null
|
||||
if(city){
|
||||
region = RegionData.formatRegion(city, row['D'])
|
||||
}
|
||||
def address = row['A']
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, null)//"property"
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['E'] = jsonObject.get("city")
|
||||
row['F'] = jsonObject.get("district")
|
||||
row['G'] = jsonObject.get("name")
|
||||
row['H'] = jsonObject.get("propertyType")
|
||||
row['I'] = jsonObject.get("id")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def commid = row['I']
|
||||
if(commid && !row['J']){
|
||||
def dist = CommonDatautils.findDistrictById(commid.trim())
|
||||
if(dist){
|
||||
def house = [:]
|
||||
house.area = 100
|
||||
def price = getEvalutaionPrice(house, dist)
|
||||
if(price){
|
||||
row['J'] = price
|
||||
}
|
||||
// def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||
// if(priceObj){
|
||||
// row['J'] = priceObj.avg_price
|
||||
// }
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def getEvalutaionPrice(house, dist){
|
||||
def params = [:]
|
||||
params.put("city", dist.city);
|
||||
params.put("district", dist.region);
|
||||
params.put("name", dist.name);
|
||||
params.put("size", String.valueOf(house.area));
|
||||
if(house.height) {
|
||||
params.put("maxFloor", dist.house_height.toString());
|
||||
params.put("direction", getDirectionCode(house.direction).toString());//朝向
|
||||
if (house.house_floor) {
|
||||
params.put("floor", house.house_floor.toString());
|
||||
}
|
||||
}
|
||||
def fitmentCode = null
|
||||
if(house.fitment){
|
||||
fitmentCode = getFitmentCode(house.fitment)
|
||||
params.put("decoration", fitmentCode.toString());//装修
|
||||
}
|
||||
params.put("scope", "1")
|
||||
def result = OpenApiClient.evaluateDistrictPrice(params);
|
||||
|
||||
if(result){
|
||||
return result.avgPrice
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def getDirectionCode(def s){
|
||||
def directMap = ["西南":1, "东南":2, "南":3, "西北":4, "东北":5, "东":6, "西":7, "北":8, "东西":9, "南北":10]
|
||||
return directMap.get(s);
|
||||
}
|
||||
def getFitmentCode(def s){
|
||||
if(s){
|
||||
if(["毛坯"].contains(s)){
|
||||
return 1;
|
||||
}else if(["简单装修","简装"].contains(s)){
|
||||
return 2;
|
||||
}else if(["精装修","精装"].contains(s)){
|
||||
return 3;
|
||||
}else if(["豪华装修","豪装"].contains(s)){
|
||||
return 4;
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def addLatLng(path){
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(!d['J'] && d['H'] && (d['H']=="普通住宅" || d['H']=="商住" || d['H']=="别墅")) {
|
||||
def json = BaiduMapUtils.searchByAddress(d['A'])
|
||||
if(json && json.location){
|
||||
d['K'] = json.location.lng
|
||||
d['L'] = json.location.lat
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def addAroundPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(!d['J'] && d['C'] && d['K']){
|
||||
def city = RegionData.formatCity(d['C'])
|
||||
def dist = new BasicDBObject()
|
||||
dist.city = city
|
||||
dist.lng = d['K'].toString().toDouble()*10000000000L
|
||||
dist.lat = d['L'].toString().toDouble()*10000000000L
|
||||
dist.estateType = "普通住宅"//d['H']
|
||||
if(dist.lat) {
|
||||
List<DBObject> list = DbUtils.findAroundDistrictByRadius(dist, 1000)
|
||||
if (list) {
|
||||
list = sortByDistance(dist, list);
|
||||
if (list && list.size() > 10) {
|
||||
list = list[0..9]
|
||||
}
|
||||
if (list) {
|
||||
def prices = []
|
||||
for (def s : list) {
|
||||
def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name)
|
||||
if (priceObj) {
|
||||
prices << priceObj.avg_price
|
||||
}
|
||||
}
|
||||
if (prices) {
|
||||
def arround_price = (int) (prices.sum() / prices.size())
|
||||
d['J'] = arround_price
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def sortByDistance(d, list){
|
||||
for(def nb : list){
|
||||
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
|
||||
nb.distance = distance
|
||||
}
|
||||
//排序
|
||||
Collections.sort(list, new Comparator() {
|
||||
@Override
|
||||
int compare(Object o1, Object o2) {
|
||||
if(o1.distance > o2.distance)
|
||||
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
|
||||
if(o1.distance == o2.distance)
|
||||
return 0;
|
||||
return -1;
|
||||
}
|
||||
})
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
def addAuthor(def path){
|
||||
def authors = ["董彦均","曹玉彦","付贵生","沈燕芳","Catherine", "王蕾","吉珉", "张宝剑"]
|
||||
def personTotal = authors.size()
|
||||
def avg = Math.ceil(total/personTotal)
|
||||
|
||||
def index = 0
|
||||
def count = 0;
|
||||
ExcelUtils.modify(path, { row ->
|
||||
if(row['A']){
|
||||
if(index < avg){
|
||||
row['M'] = authors[count]
|
||||
}
|
||||
index++
|
||||
if(index >= avg){
|
||||
count++
|
||||
index = 0
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def addHouse(path){
|
||||
def datas = []
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def commId= row['E']
|
||||
if(commId){
|
||||
def dist = CommonDatautils.findDistrictById(commId)
|
||||
if(dist){
|
||||
def rets = findHouses(row['H'], row['I'], "sell", 20200101,20230113, row['J'], null, row['k'])
|
||||
if(rets){
|
||||
def data = []
|
||||
for(def r : rets){
|
||||
data << r.city
|
||||
data << r.region
|
||||
data << r.district_name
|
||||
data << r.floor
|
||||
data << r.height
|
||||
data << r.fitment
|
||||
data << r.direction
|
||||
data << r.build_type
|
||||
data << r.area
|
||||
data << r.avg_price
|
||||
datas << data
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
ExcelUtils.write("D:\\数据\\民生\\民生样本数据.xlsx", datas, "样本数据")
|
||||
}
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
|
||||
def rets = []
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
|
||||
if(estate_type2 && estate_type2=="普通住宅"){
|
||||
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
// def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||
// if(vallaFlag){
|
||||
// continue;
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
if(!seawater.total_price){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.avg_price){
|
||||
continue;
|
||||
}
|
||||
|
||||
// if(!seawater.height){
|
||||
// seawater.height = null
|
||||
// }
|
||||
|
||||
if(seawater.height && seawater.height>60){
|
||||
continue;
|
||||
}
|
||||
|
||||
// if(!seawater.room){
|
||||
// continue;
|
||||
// }
|
||||
// if(!seawater.room){
|
||||
// seawater.room = null
|
||||
// }
|
||||
|
||||
if(seawater.room){
|
||||
if(seawater.room>=11 || seawater.room<1){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.hall){
|
||||
seawater.hall = null
|
||||
}
|
||||
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.toilet){
|
||||
seawater.toilet = null
|
||||
}
|
||||
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
def floor = parseFloor(seawater.floor_info)
|
||||
if(seawater.height && floor && floor.isNumber()){
|
||||
if(floor.toInteger()>seawater.height){
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!floor){
|
||||
seawater.floor = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
|
||||
rowMap.houseId = seawater.id
|
||||
// rowMap.city = CcbCommonUtils.getCcbCity(city)
|
||||
//rowMap.region =seawater.region
|
||||
//rowMap.address = seawater.address
|
||||
rowMap.area = seawater.area
|
||||
rowMap.address = seawater.address
|
||||
rowMap.total_price = seawater.total_price
|
||||
rowMap.avg_price = seawater.avg_price
|
||||
rowMap.build_type = seawater.build_type
|
||||
rowMap.build_date = seawater.build_date
|
||||
rowMap.district_name = seawater.district_name
|
||||
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||
if(rowMap.height && rowMap.height<1){
|
||||
rowMap.height = null
|
||||
}
|
||||
rowMap.floor = floor=="null"?null:floor
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
|
||||
|
||||
def direction = parseDirection(seawater.direction)
|
||||
rowMap.direction = direction=="null"?null:direction
|
||||
|
||||
def fitment = parseFitment(seawater.fitment)
|
||||
rowMap.fitment = fitment=="null"?null:fitment
|
||||
|
||||
|
||||
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
}
|
||||
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
|
||||
|
||||
rets << rowMap
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
}
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.commons.io.FileUtils
|
||||
|
||||
import javax.crypto.Cipher;
|
||||
import javax.crypto.SecretKey;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.util.Base64;
|
||||
|
||||
/**
|
||||
* 3DES加密工具类
|
||||
* @author qzz
|
||||
*/
|
||||
public class ThreeDESUtils {
|
||||
|
||||
/**
|
||||
* 加解密统一编码方式
|
||||
*/
|
||||
private final static String ENCODING = "utf-8";
|
||||
|
||||
/**
|
||||
* 加解密方式
|
||||
*/
|
||||
private final static String ALGORITHM = "DESede";
|
||||
|
||||
/**
|
||||
*加密模式及填充方式
|
||||
*/
|
||||
private final static String PATTERN = "DESede/ECB/pkcs5padding";
|
||||
|
||||
/**
|
||||
* 3DES加密
|
||||
*
|
||||
* @param plainText 普通文本
|
||||
* @param sK 秘钥(24位密码)
|
||||
* @return
|
||||
* @throws Exception
|
||||
*/
|
||||
public static String encode(String plainText,String sK) throws Exception {
|
||||
SecretKey secretKey = new SecretKeySpec(sK.getBytes(ENCODING), ALGORITHM);
|
||||
// 3DES加密采用pkcs5padding填充
|
||||
Cipher cipher = Cipher.getInstance(PATTERN);
|
||||
// 用密匙初始化Cipher对象
|
||||
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
|
||||
// 执行加密操作
|
||||
byte[] encryptData = cipher.doFinal(plainText.getBytes(ENCODING));
|
||||
return Base64.getEncoder().encodeToString(encryptData);
|
||||
}
|
||||
|
||||
/**
|
||||
* 3DES解密
|
||||
*
|
||||
* @param encryptText 加密文本
|
||||
* @return
|
||||
* @throws Exception
|
||||
*/
|
||||
public static String decode(String encryptText, String sK) throws Exception {
|
||||
SecretKey secretKey = new SecretKeySpec(sK.getBytes(ENCODING), ALGORITHM);
|
||||
// 3DES加密采用pkcs5padding填充
|
||||
Cipher cipher = Cipher.getInstance(PATTERN);
|
||||
|
||||
// 用密匙初始化Cipher对象
|
||||
cipher.init(Cipher.DECRYPT_MODE, secretKey);
|
||||
// 正式执行解密操作
|
||||
byte[] decryptData = cipher.doFinal(Base64.getDecoder().decode(encryptText));
|
||||
return new String(decryptData, ENCODING);
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
|
||||
def rets = []
|
||||
def datas = readTxt("D:\\数据\\民生\\抽样-住宅类(普通住宅、公寓).txt")
|
||||
def index = 0
|
||||
for(def d : datas){
|
||||
index++
|
||||
def row = []
|
||||
def str = decode(d,"111036369260679051122113")
|
||||
println str
|
||||
def id = str.substring(str.length()-4)
|
||||
row << id
|
||||
row << str.substring(0, str.indexOf(id))
|
||||
rets << row
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\民生\\民生银行测试数据.xlsx", rets, "抽样-住宅类(普通住宅、公寓)")
|
||||
|
||||
// ExcelUtils.multiThreadingModify("D:\\数据\\民生\\加密数据.xlsx", { row ->
|
||||
// //加密
|
||||
// String str = row['A']
|
||||
// //解密
|
||||
// row['B'] = (decode(str,"111036369260679051122113"));
|
||||
// });
|
||||
|
||||
}
|
||||
|
||||
public static def readTxt(def file){
|
||||
def lines = FileUtils.readLines(new File(file))
|
||||
def ret = [];
|
||||
for(line in lines) {
|
||||
ret << line;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
+2
@@ -18,6 +18,7 @@ import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
//小区月度均价表(带涨跌幅控制)
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
@@ -78,6 +79,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
}else{
|
||||
districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000)
|
||||
}
|
||||
print city +" "+districts.size()
|
||||
|
||||
def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价
|
||||
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202211.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202301.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
@@ -51,6 +51,7 @@ class AddAvgpriceBatch {
|
||||
for(def disKey : res2){
|
||||
def array = disKey.split("##");
|
||||
def estateType2 = districtKeysMap.get(disKey)
|
||||
//发重算均价命令
|
||||
CommonUtils.recalculationAverage(array[0], array[1], array[2], estateType2, 0, new Date())
|
||||
}
|
||||
}
|
||||
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
|
||||
class ShLongShangHangLocation {
|
||||
static main(args) {
|
||||
def task = new ShLongShangHangLocation()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def rets = ["上海市黄兴路1599号","上海市杨浦区长阳路1687号A栋","上海市杨浦区佳木思路432号","上海市杨浦区控江路1063号","上海市杨浦区鞍山路100号","上海市杨浦区延吉东路287号","上海市杨浦区殷行路408号","上海市杨浦区闸殷路1555号102室","上海市杨浦区平凉路1500号"]
|
||||
def map = [:]
|
||||
for(def r : rets){
|
||||
def location = getLocation(r)
|
||||
if(location){
|
||||
map.put(r, location)
|
||||
}
|
||||
}
|
||||
def titleMap = [0:'C',1:'D',2:'E',3:'F',4:'G',5:'H',6:'I',7:'J',8:'K']
|
||||
ExcelUtils.modify("D:\\数据\\上海农商行\\网点与居委分布20230118.xlsx",{row->
|
||||
if(row['B'] && !row['B'].contains("地址") && row['B'].contains('齐齐哈尔路')){
|
||||
for(def i=0;i<rets.size();i++){
|
||||
def s = rets.get(i)
|
||||
def mapLoc = map.get(s)
|
||||
def location = getLocation("杨浦区"+row['B'])
|
||||
if(location && mapLoc){
|
||||
def distance = GeoUtils.calculateDistance(mapLoc.lng.toDouble(), mapLoc.lat.toDouble(), location.lng.toDouble(), location.lat.toDouble())
|
||||
def title = titleMap.get(i)
|
||||
row[title] = distance.toInteger()
|
||||
}
|
||||
}
|
||||
println row['B']
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def getLocation(def address){
|
||||
def json = GaoDeApiUtils.geo("上海", address)
|
||||
if (json && json.status == "1") {
|
||||
def ret = json.geocodes
|
||||
if (ret) {
|
||||
def str = ret.get(0).location
|
||||
def arrs = str.split(",")
|
||||
return ["lng": arrs[0], "lat": arrs[1]]
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
package com.fangjia.script.gj.es
|
||||
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
|
||||
findLogs()
|
||||
|
||||
def findLogs(){
|
||||
def total = 0
|
||||
try{
|
||||
|
||||
// String sql = "select * from fangjia-log* where request like '/jnBank%'"
|
||||
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
|
||||
def q = QueryBuilders.fuzzyQuery("domain", ".fangjia.com")
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q)
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
|
||||
println houses.size()
|
||||
def set = new HashSet()
|
||||
def index = 0
|
||||
if(houses) {
|
||||
for (def d : houses) {
|
||||
def request = d.request
|
||||
if(request && !d.domain.contains("k8s2.fangjia.com")){
|
||||
index ++
|
||||
println "[${index}] "+d.domain +" "+ request
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println set
|
||||
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
+2
-2
@@ -401,13 +401,13 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
// if(e.text().contains("福州")){
|
||||
if(e.text().contains("福州")){
|
||||
def cityMap = [:]
|
||||
cityMap.city = e.text()
|
||||
cityMap.url = "https:"+href
|
||||
list << cityMap
|
||||
// cityMap.put(e.text(), "https:"+href)
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return list
|
||||
|
||||
+1
-1
@@ -478,7 +478,7 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia")){
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
}
|
||||
|
||||
+3
-3
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20221101", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20221201", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20230101", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230201", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
@@ -38,6 +38,6 @@ while (startDate.before(endDate)) {
|
||||
|
||||
startDate = nextDate
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行.xlsx", data, "202211")
|
||||
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user