This commit is contained in:
2023-02-17 14:06:44 +08:00
parent aa5ada6ba6
commit 1ccce2f38a
25 changed files with 2561 additions and 110 deletions
@@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -22,6 +23,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new BigdataCommbuildtypeBatch(), args);
}
@@ -29,14 +33,14 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
citys = citys.unique()
def rets = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city));
// def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city));
def distIds = PgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city))
def rows = createDatas(cityCodeDatas.get(city), distIds)
rets.addAll(rows)
println city +" "+rows.size()
@@ -50,7 +54,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
FileUtils.batchCreateFile(rets, fileName,headers, "utf-8", false);
println "txt 生成"
println "txt 生成->导Oracle"
saveToPgSql(rets)
}
@@ -103,4 +109,35 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
}
return rets;
}
def saveToPgSql(rows){
def sqls = []
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
def sql = "insert into base.t0831_bigdata_commbuildtype_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
println "start: "+new Date()
def index =0
sqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
index++
println index
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14])
}
}
println "end: "+new Date()
}
}
@@ -41,7 +41,7 @@ class BigdataCommunityBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
// citys = ["文昌"]
citys = ["上海"]
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
@@ -247,8 +247,7 @@ class BigdataCommunityBatch extends BaseBatch{
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
// def sql = "insert into base.t0831_bigdata_community_gj_h values "
def index = 0
sqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
@@ -256,26 +255,7 @@ class BigdataCommunityBatch extends BaseBatch{
ps.toString()
}
}
for(def r : rows){
index ++
if(index == 1){
sql = sql + "("+r[0].toString()+","+r[1].toString()+","+r[2].toString()+")"
}else{
sql = sql + ",("+")"
}
}
// sqlInstance.withBatch(50000, sql){ ps->
// def index = 0
// for(def r : rows){
// index++
// for(def i =0 ;i<r.size();i++){
// r[i] = r[i]?r[i]:null
// }
// ps.addBatch(r[0],r[1],r[2],r[3],r[4],r[5],r[6],r[7],r[8],r[9],r[10],r[11],r[12],r[13],r[14],r[15],r[16],r[17],r[18],r[19],r[20],
// r[21],r[22],r[23],r[24],r[25],r[26],r[27],r[28],r[29],r[30],r[31],null,'2023-02-28',null,'2999-12-31',null,null,null)
// println index
// }
// }
println "end: "+new Date()
}
}
@@ -69,43 +69,43 @@ class BigdataDealBatch extends BaseBatch{
addDealPriceMatch(city);
// }
}
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
if(trades){
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
}
}
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size()
// def rows = []
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// def index = 0
// for(def city : citys){
// index ++
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def ppeDists = [:]
//
// def statMap = [:]//getDistrictStatByCity(city, weekend)
// if(city=="上海"){
// def youDataTrade = getYouDaData(statMap)
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
// rows.addAll(youdaDatas)
// println city +" "+youdaDatas.size()
// }
//
// def trades = getCityData(city, statMap, ppeDists)
// if(trades){
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
// rows.addAll(rets)
// println "【${index}】"+city +" 成交共:"+rets.size()
//
// def cityDatas = []
// cityDatas << headers
// cityDatas.addAll(rows)
// }
//
// }
//
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
// if(CommonDatautils.isWindow()){
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
// }
//
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
// println "txt 生成 " + rows.size()
}
def createData(def rets, cityCodeDatas, distCodeMap, source){
@@ -66,6 +66,7 @@ class BigdataForsaleBatch extends BaseBatch{
@Override
public void process(Object args) {
//目前查询库中是否已存在,用的是Oracle
def index = 0
def citys = CcbCommonUtils.getRunningCitys(args)
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -74,7 +75,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221218, 20230119);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
@@ -607,7 +608,13 @@ class BigdataForsaleBatch extends BaseBatch{
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
}
rets << rowMap
def count = isExistFromHouse(seawater.id)
if(count){
rets << rowMap
}else{
println seawater.id +" 已存在"
}
}
}
@@ -803,7 +810,12 @@ class BigdataForsaleBatch extends BaseBatch{
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
}
rets << rowMap
def count = isExistFromHouse(seawater.id)
if(count){
rets << rowMap
}else{
println seawater.id +" 已存在"
}
}
}
@@ -997,4 +1009,14 @@ class BigdataForsaleBatch extends BaseBatch{
return distIds;
}
def isExistFromHouse(def houseId){
def sql = "SELECT count(FORSALEID) as count FROM bigdata_forsale_gj WHERE FORSALEID = '"+houseId+"'";
def count = null
sqlInstance.eachRow(sql,{it->
count = it.count
});
return count;
}
}
@@ -7,7 +7,7 @@ import org.apache.commons.codec.digest.DigestUtils
class CcbFtpTestUtils {
static def ftpServer = "192.168.0.86"//45
static def ftpServer = "192.168.0.45"//45
static def ftpPort = 17495
static def ftpUser = "ccb"
static def ftpPassword = "ccb!@#"
@@ -25,9 +25,9 @@ class DealPriceDataProcess {
def task = new DealPriceDataProcess()
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice();
// task.deleteThirdPartyData()
task.deleteThirdPartyData()
// task.delDealByPe()
task.delDealData()
// task.delDealData()
}
def delDealByPe(){
@@ -372,12 +372,12 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all
def month = 202211
def month = 202212
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202212)
def rets = findPeGteThirdParty(city, 202301)
def total = 0
println city +" "+rets.size()
if(rets){
@@ -13,6 +13,7 @@ import org.apache.commons.codec.digest.DigestUtils
import java.sql.ResultSet
//估值 新版 导出所有数据
class ExportOracleAllDataTest {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
@@ -20,7 +21,7 @@ class ExportOracleAllDataTest {
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def citys = null
def curDate = "20221228"//DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def curDate = "20220128"//DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
static main(args) {
def task = new ExportOracleAllDataTest()
@@ -38,7 +39,7 @@ class ExportOracleAllDataTest {
def run(args) {
citys = CcbCommonUtils.getRunningCitys(args)//["潍坊","济宁","威海","临沂","绵阳","陇南","塔城","阿勒泰","迪庆州","日喀则"] //CcbCommonUtils.getRunningCitys(args)
def curMonth = "202212"//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
def new_dists = []
@@ -31,7 +31,7 @@ class ExportOracleEvaluationData {
exportCityPrice(curMonth)
exportRegionPrice(curMonth)
exportCommunityBasePriceGj(curMonth)//已合下面的合并
// exportCommunityBasePriceGj(curMonth)//已合下面的合并
exportCommunityBasePrice(curMonth)
exportCommunityPdescribeGj(curMonth)
@@ -248,6 +248,7 @@ class ExportOracleEvaluationData {
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
@@ -0,0 +1,20 @@
package com.fangjia.script.ccbevaluation
import groovy.sql.Sql
class PgsqlCommUtils {
public static def database = "warehouse"
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/"+database+"?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
public static def findAllDistrictIdsByOracle(def cityid){
def sql = "select gujia_comm_id from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0";
def set = new HashSet()
sqlInstance.eachRow(sql,{r->
set << r.gujia_comm_id
});
return set
}
}
@@ -45,8 +45,91 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
def citys = ["山南","黄南州","甘南","克孜勒苏","嘉峪关","海北州","阿拉善盟","怒江州","和田","日喀则","昌都","海南州","玉树州","吐鲁番","阿勒泰","三沙","塔城","甘孜州","果洛州","林芝","那曲","阿坝州","神农架"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\小区地址名单_估家科技V2 (3).xlsx",{d->
if(d['J']){
def cityId = cityCodeDataMap.get(d['H'])
def map = findCcbPriceCredit(sqlInstance236, cityId, d['J'])
println d['H'] +" "+d['J']+ map.toString()
if(map){
d['Q'] = map.credit
d['R'] = map.month
d['U'] = map.price
}
// def thirdMap = findThirdPriceBySource(sqlInstance236, cityId, d['J'])
// if(thirdMap){
// d['S'] = thirdMap.price
// d['T'] = thirdMap.month
// }
}
})
def findCcbPriceCredit(sqlInstance, cityid, commid){
def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1"
def map = [:]
sqlInstance.eachRow(sql,{r->
map.month = r.year_month
map.credit = r.CONFIDENCE
map.price = r.avg_price
});
return map
}
def findThirdPriceBySource(sqlInstance, cityid, commid){
def sql = "select * from (select data_source, price_date, community_price,commid from bigdata_third_party_price_gj where cityid='${cityid}' and commid='${commid}' and data_source in ('贝壳','链家') \n" +
"order by price_date desc) WHERE ROWNUM <= 1"
def map = [:]
sqlInstance.eachRow(sql,{r->
map.month = r.price_date
map.price = r.community_price
});
return map
}
for(def c : citys){
def cityCode = cityCodeDataMap.get(c)
println c +" "+cityCode
def priceDists = findAllPriceDist(sqlInstance236,cityCode, 202301)
//
checkNoPriceDist(sqlInstance236, priceDists, cityCode)
}
def checkNoPriceDist(sqlInstance236,priceDists, cityid){
def commids = []
def sql = "select commid from bigdata_community_gj where cityid='${cityid}' and status=0";
sqlInstance236.eachRow(sql, { r ->
commids << r.commid
})
for(def commid : commids){
if(!priceDists.contains(commid)){
updateDistStatus(sqlInstance236, commid)
}
}
return commids
}
def updateDistStatus(sqlInstance236, commid){
def sql = "update bigdata_community_gj set status=1 where commid='${commid}'";
sqlInstance236.executeUpdate(sql)
}
def findAllPriceDist(sqlInstance236, cityid, yearMonth){
def commids = []
def sql = "select commid from community_base_price_gj where cityid='${cityid}' and year_month=${yearMonth} ";
sqlInstance236.eachRow(sql, { r ->
commids << r.commid
})
return commids
}
//def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水']
//
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -1087,15 +1170,15 @@ def findThirdCommCount(cityid, sqlInstance){
//}
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达价格-杭州.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
d['C'] = dist.city
d['D'] = dist.region
d['E'] = dist.name
}
})
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达价格-杭州.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// d['C'] = dist.city
// d['D'] = dist.region
// d['E'] = dist.name
// }
//})
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\0922入库数据.xlsx"
//def datas = ExcelUtils.read(fileName, 0, 1)
@@ -40,7 +40,7 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
// task.run()//统计最近3个月抓取的成交数据量
task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
@@ -92,13 +92,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","9月总量","10月总量","11月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","10月总量","11月总量","12月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2022-09-01"
def date = "2022-10-01"
def endDate = "2022-11-01"
def newDate = "2022-12-01"
def startDate = "2022-10-01"
def date = "2022-11-01"
def endDate = "2022-12-01"
def newDate = "2023-01-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -1,8 +1,10 @@
package com.fangjia.script.ccbevaluation.pgsql
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
@@ -43,13 +45,11 @@ class ExportPgsqlEvaluationResult {
// exportCommunityPdescribeGj(curMonth)
// exportCommCaseDetailGj(curMonth)
//
def curDate = curMonth.toString()+"28"
// 生成MD5文件
exportMd5()
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
// ok文件
exportOkFile()
exportInOk()
SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
}
def exportAvmEvaluationGJNB(yearMonth){
@@ -363,20 +363,31 @@ class ExportPgsqlEvaluationResult {
}
def exportOkFile(){
def dataCodeFileName = linux_prefix + "${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = win_prefix + "${curDate}.ok"
def exportInOk(){
Vector files = CcbFtpTestUtils.listFileNames(null, curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate)
}
}
createFileToFtp("ok txt", [], dataCodeFileName, curDate, false)
def fileName = linux_prefix + "/${curDate}.ok"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "${curDate}.ok"
}
createFileToFtp("ok txt",fileRows, fileName, curDate, false)
}
def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend);
println tableName+" 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
// println tableName+" 上传Ftp完成......"
CcbFtpTestUtils.ftpUpload(null, fileName, curDate)
println tableName+" 上传Ftp完成......"
}
}
@@ -1,14 +1,618 @@
package com.fangjia.script.ccbevaluation.pgsql
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql
class ExportPgsqlRonghe {
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres",
"postgres", "postgres", "org.postgresql.Driver")
def citys = []
def database = "base."
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def linux_prefix = "/ccb_ronghe/${curDate}/"
def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\"
static main(args) {
def task = new ExportPgsqlRonghe()
task.run(args)
}
def run(args) {
def new_dists = []
citys = CcbCommonUtils.getRunningCitys(args)
//按天导(每天一个文件夹)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date() + 1, -1), "yyyy-MM-dd"), new_dists)
if (curDate.toString().endsWith("10")) {
exportEvaluateRate()
exportBigdataBuildprojFjw()
exportBigdataCommlayoutpicFjw()
exportBigdataCommunityGJRH()//
exportBigdataBuildingFJW()//
exportBigdataCommfeeGj()//
exportBigdataCommbuildstruGJ()//
exportBigdataCommhousetypeGJ()//
exportBigdataCommbuildyearGj()//
exportBigdataCommhouseuseGJ()//
exportBigdataPositionGJ()//
exportBigdataDistrictFJW()//
exportBigdataAliasFJW()//
//本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
// exportBigdataCityPrice(queryDate)
// exportBigdataDistPrice(queryDate)
// exportBigdataCommPrice(queryDate)
exportInOk()
// }
// SSHHelperTest jschCommand = new SSHHelperTest();
// System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}"));
}
}
def exportBigdataDealFJW2(curDate, dealdate, new_dists){
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def fileName = "FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
if(curDate.toString().endsWith("10")){
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " +
"from "+database+"t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}"
def rets1 = dealData(sql1)
def sql2 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " +
"from "+database+"t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}"
def rets2 = dealData(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " +
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd')";
allRows.addAll(dealData(sql))
}
}
}
createFileToFtp("BIGDATA_DEAL_FJW", allRows, fileName, curDate, false)
}
def dealData(def sql){
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.urbn_id
data << r.mdl_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_str
data << r.gujia_hsno
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_bld_id
data << r.bld_id
data << r.gujia_bld_nm
data << r.gujia_unit_id
data << r.unit_id
data << r.gujia_unit_nm
data << r.gujia_acc_id
data << r.gujia_dtl_adr
data << r.gujia_bld_yr_rc
data << r.gujia_own_val
data << r.gujia_own_yrlmt_val
data << r.gujia_prd_val
data << r.gujia_wbt_floor
data << r.gujia_lyr_tp_val
data << r.gujia_tot_floor_num
if(r.gujia_bld_tp_val){
def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.gujia_bld_stc
data << r.gujiaconstructionarea
data << r.gujia_set_innr_area
data << r.gujia_acc_tp_val
data << r.gujia_room
data << r.gujia_dept
data << r.gujia_ktch_num
data << r.gujia_toliet
data << r.gujia_hs_use
data << r.gujia_indr_stc_val
data << r.gujia_twrd_val
data << r.gujia_dcrt_sttn
data << r.gujia_ases_bt_ln_num
data << r.gujia_trdtckt_prc
data << r.gujia_mdl_tot_prc
data << r.gujia_dcl_tot_prc
data << r.gujia_ases_tot_prc
data << r.gujiasigntonlinectrid
data << r.txn_dt
data << r.gujia_txn_tp_val
data << r.gujia_sell_pymtmod
data << r.gujia_mdl_cyc
data << r.gujiawthrext_lift_ind
data << r.gujiagvlndfargrdnarea
data << r.gujia_gv_lu_tbl_area
data << r.gujia_gv_sun_tbl_area
data << r.gujiagvbaywindow_area
data << r.gujiagvarea_tower_flr
data << r.gujia_gv_grg_area
data << r.gujiagvundgdroom_area
data << r.gujia_gv_othr_area
data << r.gujia_hs_src_dtsrc
data << r.intr_co_id
data << r.intr_co_nm
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << r.gujia_mkdam
data << r.gujia_data_tm
allRows << data
});
return allRows
}
def exportEvaluateRate(){
def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_ceft,gujia_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from t0831_evaluate_rate_fjw_w";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.gujia_ceft_id
data << r.gujia_ceft_tp
data << r.gujia_ceft
data << r.gujia_seq
data << r.gujia_urbn
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
def fileName = linux_prefix + "FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("EVALUATE_RATE_FJW", allRows, fileName, curDate, false)
}
def exportBigdataBuildprojFjw(){
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_BUILDPROJ_FJW", [], fileName, curDate, false)
}
def exportBigdataCommlayoutpicFjw(){
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMLAYOUTPIC_FJW", [], fileName, curDate, false)
}
def exportBigdataCommunityGJRH(){
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys){
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,comm_nm,gujia_comm_fmnm,gujia_admndiv_id,gujia_str,gujia_hsno,gujia_dtl_adr,gujiaocpoflandtotarea,gujia_bld_tot_area,gujia_comm_tp_val,gujiawthr_new_pan_ind,gujia_grnrate,gujia_vol_rate,gujia_park_num,gujia_comm_intd,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujiaprhsld_hspty_prc,gujiacomm_tot_rsd_num,gujiacommrntfee_avprc,gujia_comm_avprc from "+database+"t0831_bigdata_community_fjw_w where urbn_id='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_comm_fmnm
data << r.gujia_admndiv_id
data << r.gujia_str
data << r.gujia_hsno
data << r.gujia_dtl_adr
data << r.gujiaocpoflandtotarea
data << r.gujia_bld_tot_area
data << r.gujia_comm_tp_val
data << r.gujiawthr_new_pan_ind
data << r.gujia_grnrate
data << r.gujia_vol_rate
data << r.gujia_park_num
data << r.gujia_comm_intd
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
createFileToFtp("BIGDATA_COMMUNITY_FJW", allRows, fileName, curDate, false)
}
def exportBigdataCommbuildstruGJ(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id, gujia_comm_id, gujia_bld_stc, rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+"t0831_bigdata_commbuildstru_fjw_w where urbn_id='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_bld_stc
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMBUILDSTRU_FJW", allRows, fileName, curDate, false)
}
def exportBigdataCommfeeGj(){
def allRows = []
for(def city : citys){
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,co_id,gujia_eps_val,gujia_hs_use,gujia_bld_tp_val,gujia_comm_eps_tp,gujia_eps_cmnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commfee_fjw_w where urbn_id='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.co_id
data << r.gujia_eps_val
data << r.gujia_hs_use
if(r.gujia_bld_tp_val){
def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.gujia_comm_eps_tp
data << r.gujia_eps_cmnt
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMFEE_FJW", allRows, fileName, curDate, false)
}
def exportBigdataCommhousetypeGJ(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,gujia_comm_hs_tp,rmrk_dsc,to_char(data_crt_tm'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commhousetype_fjw_w where urbn_id='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_comm_hs_tp
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMHOUSETYPE_FJW", allRows, fileName, curDate, false)
}
def exportBigdataCommbuildyearGj(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,gujia_bld_yr_rc,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commbuildyear_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_bld_yr_rc
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMBUILDYEAR_FJW", allRows, fileName, curDate, false)
}
def exportBigdataCommhouseuseGJ(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,gujia_hs_use,gujia_ladder_accnum,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commhouseuse_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_hs_use
data << r.gujia_ladder_accnum
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMHOUSEUSE_FJW", allRows, fileName, curDate, false)
}
def exportBigdataPositionGJ(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_obj_tp,gujia_obj_id,gujia_lo_tp,gujia_sn,lgt,ltt,gujia_hignas_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_position_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_obj_tp
data << r.gujia_obj_id
data << r.gujia_lo_tp
data << r.gujia_sn
data << r.lgt
data << r.ltt
data << r.gujia_hignas_val
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_POSITION_FJW", allRows, fileName, curDate, false)
}
def exportBigdataDistrictFJW(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_district_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_admndiv_nm_cpa
data << r.urbn_id
data << r.gujia_natstd_cd
data << r.gujia_dspl_seq
data << r.gujia_admndiv_cgy
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_DISTRICT_FJW", allRows, fileName, curDate, false)
}
def exportBigdataAliasFJW(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,rel_id,gujia_comm_id,comm_nm,gujia_pcsst,gujia_vld_dt,gujia_vrsn_dsc,gujia_wthr_prim_nm,gujia_dtsrc,gujia_dsc,gujia_als_lvl,gujia_adr_tp_ecd,gujia_wthr_adt_ind,gujia_wthr_dspl_ind from "+database+"t0831_address_alias_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.rel_id
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_pcsst
data << r.gujia_vld_dt
data << r.gujia_vrsn_dsc
data << r.gujia_wthr_prim_nm
data << r.gujia_dtsrc
data << r.gujia_dsc
data << r.gujia_als_lvl
data << r.gujia_adr_tp_ecd
data << r.gujia_wthr_adt_ind
data << r.gujia_wthr_dspl_ind
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("ADDRESS_ALIAS_FJW", allRows, fileName, curDate, false)
}
def exportBigdataBuildingFJW(){
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
}
def index = 0
for(def city : citys) {
def allRows = []
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_bld_id,bld_id,gujia_bld_nm,gujia_bld_als_dsc,gujia_admndiv_id,gujia_comm_id,gujia_bld_yr_rc,gujia_bld_tp_val,gujia_bld_stc,floor_num,gujia_unit_num,gujia_accnum_val,gujiawthrext_lift_ind,gujia_lift_num,gujiajiladderjiactval,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_vld_ind from "+database+"t0831_bigdata_building_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_bld_id
data << r.bld_id
data << r.gujia_bld_nm
data << r.gujia_bld_als_dsc
data << r.gujia_admndiv_id
data << r.gujia_comm_id
data << r.gujia_bld_yr_rc
if(r.gujia_bld_tp_val){
def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.gujia_bld_stc
data << r.floor_num
data << r.gujia_unit_num
data << r.gujia_accnum_val
data << r.gujiawthrext_lift_ind
data << r.gujia_lift_num
data << r.gujiajiladderjiactval
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << r.gujia_vld_ind
allRows << data
});
if(allRows && allRows.size()>0){
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
}
}
}
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
}
def exportBigdataCityPrice(queryDate){
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
// def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";//
def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from t0831_city_price_gj_h where CITYID='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.YMID
data << r.SALEPRICE?r.SALEPRICE:''
data << r.RENTPRICE?r.RENTPRICE:''
if(r.SALECOUNT>0) {
data << r.SALECOUNT
}else{
data << ''
}
if(r.RENTCOUNT>0){
data << r.RENTCOUNT
}else{
data << ''
}
data << r.UPDATETIME
data << r.HOUSEUSE
data << r.SALEBLDGAREA?r.SALEBLDGAREA:''
data << r.RENTBLDGAREA?r.RENTBLDGAREA:''
data << r.SALETOTALPRICE?r.SALETOTALPRICE:''
data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:''
data << r.DATATYPE
data << r.SDATERANGE
data << r.RDATERANGE
data << r.SOURCECODE
allRows << data
});
}
createFileToFtp("BIGDATA_CITYPRICE", allRows, fileName, curDate, false)
}
def exportInOk(){
Vector files = CcbFtpTestUtils.listFileNames("rh", curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength("rh", file.getFilename(), curDate)
}
}
def fileName = linux_prefix+"${curDate}.ok"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "${curDate}.ok"
}
createFileToFtp("融合统计文件长度ok文件", fileRows, fileName, curDate, false)
}
def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend);
println tableName+" 数据文件输出完成......"
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
println tableName+" 上传Ftp完成......"
}
}
@@ -0,0 +1,703 @@
package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import net.sf.json.JSONObject
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
class HailanTest {
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static{
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new HailanTest();
task.run();
}
def run(){
def inputfile = "D:\\数据\\海南\\统计.xlsx"
// match(inputfile)
// addPrice(inputfile)
// addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx")
// addHouse(inputfile)
// stat(inputfile)
hailanStat(inputfile)
}
def addHouse(inputfile){
def datas = []
datas << ["类型(住宅/别墅)","行政区划","详细地址","房号","小区名称","建筑面积","所在楼层","详细地址","小区名","交易日居住用房价格指数","评估日居住用房价格指数","是否有电梯","所在楼层","总楼层","内部装修","平面布置","朝向",
"景观","建筑结构","建筑面积(平方米)","房屋可使用年限","房屋已使用年限","评估价(单价)","挂牌价(单价)","成交价(单价)","小区/楼盘均价(单价)"]
def index = 0
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(row['L']){
index ++
def dist = CommonDatautils.findDistrict(row['H'], row['I'], row['J'], "district")
def rets = findHouses(row['H'], row['I'], "sell", 20200101,20230113, row['J'], null, row['k'])
println index +" "+ row['C']+" "+ rets.size()
if(rets){
def lastRets = []
for(def r : rets){
def fjwPrice = row['M']
if(fjwPrice){
def precent = java.lang.Math.abs((fjwPrice-r.avg_price)*100/r.avg_price)
if(precent>=10){
continue;
}
}
if(row['A'] && row['A']=="别墅"){
def inputAare = row['F'].toString().toDouble()
def minArea = inputAare + 50
def maxArea = inputAare + 100
if(r.area<minArea && r.area>maxArea){
continue;
}
}
lastRets << r
if(lastRets.size()>=3){
break;
}
}
if(lastRets){
for(def lr : lastRets){
def data = []
data << row['A']
data << row['B']
data << row['C']
data << row['D']
data << row['E']
data << row['F']
data << row['G']
if(dist){
data << dist.address
}else{
data << ""
}
if(dist){
data << dist.name
}else{
data << ""
}
data << ""
data << ""
if(lr.height){
if(lr.height>6){
data << "是"
}else{
data << "否"
}
}else{
data << ""
}
data << lr.floor
data << lr.height
data << lr.fitment
data << ""
data << lr.direction
data << ""
data << lr.build_type
data << lr.area
data << dist.property_life
// def year = dist.create_year
// if(year){
// data << getHouseYear(year.toInteger())//已使用年限
// }else{
// data << ""//已使用年限
// }
data << getHouseYear(dist.create_year , lr.build_date, dist.build_date)
data << getEvalutaionPrice(lr, dist)//评估价(单价)
data << lr.avg_price
data << getDealPrice(dist)//成交单价
data << row['M']
datas << data
}
}
}
}
})
ExcelUtils.write("D:\\数据\\海南\\海南样本数据.xlsx", datas, "样本数据")
}
def getDealPrice(dist){
def json = OpenApiClient.searchDealPrice(dist._id.toString(),"20220101", 100)
def prices = []
if(json && json.code==200){
def result = json.result
if(result){
for(def r : result){
prices << r.avgPrice
}
}
}
def dealAvgPrice = null
if(prices){
dealAvgPrice = (prices.sum()/prices.size()).toInteger()
}
return dealAvgPrice;
}
def getHouseYear(create_year , house_build_date, dist_build_date){
def year = null
if(house_build_date){
year = getYear(house_build_date)
}else if(dist_build_date){
year = getYear(dist_build_date)
}else if(create_year){
year = create_year
}
if(year){
year = year.toInteger()
def curYear = new Date().getYear()+1900
def modYear = curYear - year
return modYear.toInteger().toString()
}
return null
}
def getYear(finishBuildingDate){
if(finishBuildingDate){
Pattern pat3 = Pattern.compile("\\d{4}");
def matcher3 = pat3.matcher(finishBuildingDate);
if(matcher3.find()){
return matcher3.group()
}
}
return null
}
def getEvalutaionPrice(house, dist){
def params = [:]
params.put("city", dist.city);
params.put("district", dist.region);
params.put("name", dist.name);
params.put("size", String.valueOf(house.area));
if(house.height) {
params.put("maxFloor", dist.house_height.toString());
params.put("direction", getDirectionCode(house.direction).toString());//朝向
if (house.house_floor) {
params.put("floor", house.house_floor.toString());
}
}
def fitmentCode = null
if(house.fitment){
fitmentCode = getFitmentCode(house.fitment)
params.put("decoration", fitmentCode.toString());//装修
}
params.put("scope", "1")
def result = OpenApiClient.evaluateDistrictPrice(params);
if(result){
return result.avgPrice
}
return null
}
def getDirectionCode(def s){
def directMap = ["西南":1, "东南":2, "南":3, "西北":4, "东北":5, "东":6, "西":7, "北":8, "东西":9, "南北":10]
return directMap.get(s);
}
def getFitmentCode(def s){
if(s){
if(["毛坯"].contains(s)){
return 1;
}else if(["简单装修","简装"].contains(s)){
return 2;
}else if(["精装修","精装"].contains(s)){
return 3;
}else if(["豪华装修","豪装"].contains(s)){
return 4;
}
}
return null
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def rets = []
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(5);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
// def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
// if(vallaFlag){
// continue;
// }
}
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
// if(!seawater.height){
// seawater.height = null
// }
if(seawater.height && seawater.height>60){
continue;
}
// if(!seawater.room){
// continue;
// }
// if(!seawater.room){
// seawater.room = null
// }
if(seawater.room){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
rowMap.houseId = seawater.id
// rowMap.city = CcbCommonUtils.getCcbCity(city)
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
rowMap.address = seawater.address
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.build_type = seawater.build_type
rowMap.build_date = seawater.build_date
rowMap.district_name = seawater.district_name
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
rets << rowMap
}
}
return rets;
}
static def parseFloor(def floorDesc){
if(floorDesc){
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
if(floorDesc){
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
Matcher matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
// return "中层"
// }
//
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
// return "高层"
// }
//
//
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
// return "低层"
// }
if(floorDesc.isNumber()){
return floorDesc;
}
}
}
return null;
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
def addCommId(inputfile){
ExcelUtils.modify(inputfile, { row ->
if(row['H']){
def dist = CommonDatautils.findDistrict(row['H'], row['I'], row['J'], "district")
if(dist){
row['L'] = dist._id.toString()
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['L']
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['M'] = priceObj.avg_price
}
}
}
})
}
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['C']){
String city = row['H']//RegionData.formatCity(row['B'])
if(city){
def address = row['C']
// def region = row['C']
println "[match] city:"+city+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, row['E'])
if(jsonObject != null && jsonObject.precise!=-1) {
row['I'] = jsonObject.get("district")
row['J'] = jsonObject.get("name")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("id")
}
}else{
println row['C']
}
}
})
}
def stat(path){
ExcelUtils.modify(path, { row ->
if(row['L'] && !row['H']){
def city = row['L']
def region = row['M']
def dists = null
if(city && region){
dists = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias","estate_type2"])
}else{
dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias","estate_type2"])
}
def distLists = []
def villaLists = []
for(def d : dists){
if(d.estate_type2=="别墅"){
villaLists << d
}else{
distLists << d
}
}
row['J'] = dists.size()
def haveHouseList = findHaveHouseDist(distLists)
def haveVillaList = findHaveHouseDist(villaLists)
row['H'] = haveHouseList
row['I'] = haveVillaList
// row['N'] = distLists.size()
// row['O'] = villaLists.size()
}
})
}
def hailanStat(path){
def total = 0
def distTotal = 0
def villaTotal = 0
def haveHouseDistTotal = 0
def haveHouseVillaTotal = 0
def buildStructureTotal = 0
// ExcelUtils.modify(path, { row ->
// if (row['L'] && !row['K']) {
//"海口","三亚","三沙","儋州",
def citys = ["五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
for(def city : citys) {
def dists = null
// if (city && region) {
// dists = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city", "region", "name", "alias", "estate_type2", "build_structure"])
// } else {
dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias", "estate_type2", "build_structure"])
// }
def distLists = []
def villaLists = []
for (def d : dists) {
if (d.estate_type2 == "别墅") {
villaLists << d
} else {
distLists << d
}
}
total = total + dists.size()
// def structureCount = 0
// for(def d : dists){
// if(d.build_structure){
// structureCount ++
// }
// }
// println structureCount
// row['K'] = structureCount
// buildStructureTotal = buildStructureTotal + structureCount
//
def haveHouseList = findHaveHouseDist(distLists)
def haveVillaList = findHaveHouseDist(villaLists)
haveHouseDistTotal = haveHouseDistTotal + haveHouseList
haveHouseVillaTotal = haveHouseVillaTotal + haveVillaList
distTotal = distTotal + distLists.size()
villaTotal = villaTotal + villaLists.size()
// println c +" "+total+" "+buildStructureTotal
// }
}
// })
println total +" "+haveHouseDistTotal+" "+haveHouseVillaTotal+" "+distTotal+" "+villaTotal+" "+buildStructureTotal
}
def findHaveHouseDist(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println "小区量:"+districts.size()
def rets = new CopyOnWriteArrayList()
for(def i=0;i<districts.size();i++){
final def dist = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def ret = findHouses(dist.city, dist.region, "sell", 20210101, null, dist.name, dist.alias, dist.estate_type2)
if(!ret){
ret = findThirdPrice(dist)
}
if(ret && ret.size()>0){
rets.add(dist)
}
println totalCount.get() +" "+ ret.size()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return rets.size()
}
def findThirdPrice(dist){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
q.put("name", dist.name)
q.put("s_date", new BasicDBObject('$gte', 202201))
def dbcursor = AVERAGE_STATISTICS.find(q)
def prices = []
if(dbcursor){
for(def d : dbcursor){
def set = d.keySet();
Iterator it = set.iterator()
while(it.hasNext()){
def key = it.next()
if(key && key.contains("_price") && key!="fangjia_price" && key!="last_price"){
def price = d[key]
if(price){
prices << price
}
}
}
}
}
return prices;
}
}
@@ -0,0 +1,328 @@
package com.fangjia.script.chongqing
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import org.apache.lucene.queryparser.classic.QueryParser
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class MingShenPocStat {
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
private static final DBCollection district_sell_stat_coll = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo);
static main(args) {
def task = new MingShenPocStat()
// task.run()
task.officeRun()
}
def officeRun(){
def path = "D:\\数据\\民生\\抽样-写字楼类.xlsx"
def datas = []
ExcelUtils.modify(path, { d ->
if(d['B']){
def office = CommonDatautils.findDistrictById(d['C'])
if(office){
def alias = []
alias << office.name
if(office.alias){
def nameAlias = office.alias
for(def o : nameAlias){
if(o){
alias << o
}
}
}
def aliasStr = escapeExprSpecialWord(alias.join("','"))
def houses = findOfficeHouseCount(office.city, office.region, aliasStr)
if(houses){
if(houses.size()>=2){
houses = houses[0..1]
}
for(def h : houses){
def obj = findOfficeAvgPrice(office)
def rets = []
rets << d['A']
rets << d['B']
rets << ''
rets << h.region
rets << office.address
rets << h.area
rets << h.floor
rets << h.height
def level = office.bed_kind
rets << level
rets << h.fitment
def date = office.finish_building_date
if(date){
rets << date.substring(0, 4)
}else{
rets << ""
}
rets << ""
rets << ""
def price = ""
if(obj && obj.avg_price){
price = obj.avg_price
rets << obj.avg_price
}else{
rets << ""
}
if(price){
rets << (price*h.area).toInteger()
}
datas << rets
}
}
}else{
def rets = []
rets << d['A']
rets << d['B']
datas << rets
}
// def districts = CommonDatautils.findAllDistrictByCity(d['C'],"office",["city", "region", "name", "alias","address","bed_kind","finish_building_date"])
}
})
ExcelUtils.write("D:\\数据\\民生\\抽样-写字楼类-样本.xlsx", datas, "样本")
}
def run(){
def path = "D:\\数据\\民生\\样本字段覆盖率.xlsx"
// def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Downloads\\二期估值数据每周统计\\二期估值数据每周统计\\估值数据统计结果20230112.xlsx", 0 , 1)
//
// ExcelUtils.modify(path, { row ->
// if(row['C']) {
// def total = 0
// def pre = 0
// for(def d : datas){
// if(row['C']==d['A']){
// total = d['B']
// pre = d['H']
// break
// }
// }
// if(total){
// def c = (total*pre/100).toInteger()
// row['E'] = c
// row['F'] = pre+"%"
// }
// }
// })
def totalcount = findHouseTotal("大连")
def count = findHouseCount("大连", "height")
println DataHandleUtils.getDoubleRound(2, count*100/totalcount)
// ExcelUtils.multiThreadingModify(path, { row ->
// if(row['C']) {
//def districts = CommonDatautils.findAllDistrictByCity(row['C'],"district",["city", "region", "name", "alias"])
// def districts = CommonDatautils.findDistrictByCityAndRegions("大连","district",["旅顺口"], ["city", "region", "name", "alias"])
// def total = districts.size()
// if(total){
// def house_dist_count = findHouseStat(districts)
// def pre = house_dist_count*100/total
// pre = DataHandleUtils.getDoubleRound(2, pre)
// println total+" "+house_dist_count+" "+pre
// }
// def count = findHouseCount(row['C'])
// row['D'] = count
// println row['C']+" "+count
// row['H'] = findDistrictSellStat(row['C'])
// row['G'] = getNewDist(row['C'])
// def totalcount = findHouseTotal(row['C'])
// def total = row['Z']
// def count = findHouseCount(row['C'], "build_date")
// row['Q'] = DataHandleUtils.getDoubleRound(2, count*100/total)
// println count +" "+row['Q']
// def obj = getFieldPrecent(row['C'])
// if(obj){
// def detail = obj.district_detail
// row['E'] = detail.alias.toDouble()*100
// row['F'] = detail.address.toDouble()*100
// row['G'] = detail.lng.toDouble()*100
// row['H'] = detail.buildType.toDouble()*100
// row['I'] = detail.outPics.toDouble()*100
// row['J'] = detail.developer.toDouble()*100
// row['Q'] = detail.buildDate.toDouble()*100
// row['R'] = detail.buildDate.toDouble()*100
// }
// }
// });
}
def getFieldPrecent(city){
def q = new BasicDBObject()
q.put("city", city)
def rets = CITY_INFO_CONN.find(q).sort(new BasicDBObject('weekend', -1))
if(rets){
return rets[0]
}
return null
}
def getNewDist(city){
String sql = "select count(*) as count from community where city='${city}' and source='贝壳找房-新房' "
def count = null
ElasticUtils.executeSqlQuery(sql,{ data->
count = data.count
})
return count;
}
def findDistrictSellStat(def city){
def q = new BasicDBObject()
q.put("city", city)
def count = district_sell_stat_coll.count(q)
return count
}
def findHouseStat(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println "小区量:"+districts.size()
def rets = new CopyOnWriteArrayList()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def ret = findHouses(rMap.city, rMap.region, "sell", 20210101,rMap.name, rMap.alias)
if(ret && ret.size()>0){
rets.add(rMap)
}
println totalCount.get() +" "+ ret.size()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return rets.size()
}
def findHouseTotal(city){
def count = 0
ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date >= 20220101 and city='${city}'", {
count = it.count
println("$city\t${it.count}")
})
return count
}
def findHouseCount(city, field){
def count = 0
def sql = "select count(*) as count from seawater where city='${city}' and cat='sell' and "+field+" is not null"
ElasticUtils.executeSqlQuery(sql, {
count = it.count
println("$city\t${it.count}")
})
return count
}
def findHouses(def city, def region, def cat, def startDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
return houses;
}
def findOfficeAvgPrice(office){
def sql = "select * from office where city='${office.city}' and region='${office.region}' and name='${office.name}'"
def rets = []
ElasticUtils.executeSqlQuery(sql, {d->
rets << d
})
if(rets){
def obj = rets[0]
return obj
}
return null
}
def findOfficeHouseCount(city, region, names){
def sql = "select * from seawater where city='${city}' and region='${region}' and cat='office_sell' and district_name in ('${names}') and root='58.com'"
def rets = []
ElasticUtils.executeSqlQuery(sql, {d->
rets << d
})
return rets
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
* @param keyword
* @return
*/
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
}
@@ -0,0 +1,459 @@
package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
class MingShenPocTest {
def total = 5000
static main(args) {
def task = new MingShenPocTest()
task.run()
}
def run(){
def path = "D:\\数据\\民生\\poc测试.xlsx"
//9级标准化
//structureV2(path)//针对海南poc地址数据此接口不合适使用
//省市区拆分
// ner(path)//城市,区域带市带区
// match(path)
// addPrice(path)
// addLatLng(path)
// addAroundPrice(path)
// addAuthor(path)
addHouse(path)
}
def structureV2(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['A']){
def jsonObj = FangjiaClientUtils.addressStructure(null, row['A'])
if(jsonObj && jsonObj.result){
if(jsonObj.result.province){
row['B'] = jsonObj.result.province
}
if(jsonObj.result.city){
def cityStr = null
if(jsonObj.result.city=="市辖区"){
cityStr = jsonObj.result.province
}else{
cityStr = jsonObj.result.city
}
row['C'] = RegionData.formatCity(cityStr)
}
if(jsonObj.result.district){
row['D'] = jsonObj.result.district
}
}
}
})
}
def ner(path){
ExcelUtils.multiThreadingModify(path, { d ->
if(d['A'] && !d['B']) {
def json = FangjiaClientUtils.addressStand(null, d['A'])
if(json && json.entities){
def entities = json.entities
for(def m : entities){
if(m.type=="prov"){
d['B'] = m.word
}
if(m.type=="city"){
def cityStr = m.word
if(cityStr){
d['C'] = cityStr//RegionData.formatCity(cityStr)
}
}
if(m.type=="dist") {
if(d['C']){
d['D'] = m.word //RegionData.formatRegion(d['C'], m.word)
}
}
}
}
}
});
}
//可以根据测试数据添加 推荐楼盘名
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['C'] && !row['E']){
def city = RegionData.formatCity(row['C'])
def region = null
if(city){
region = RegionData.formatRegion(city, row['D'])
}
def address = row['A']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, null)//"property"
if(jsonObject != null && jsonObject.precise!=-1) {
row['E'] = jsonObject.get("city")
row['F'] = jsonObject.get("district")
row['G'] = jsonObject.get("name")
row['H'] = jsonObject.get("propertyType")
row['I'] = jsonObject.get("id")
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['I']
if(commid && !row['J']){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
def house = [:]
house.area = 100
def price = getEvalutaionPrice(house, dist)
if(price){
row['J'] = price
}
// def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
// if(priceObj){
// row['J'] = priceObj.avg_price
// }
}
}
})
}
def getEvalutaionPrice(house, dist){
def params = [:]
params.put("city", dist.city);
params.put("district", dist.region);
params.put("name", dist.name);
params.put("size", String.valueOf(house.area));
if(house.height) {
params.put("maxFloor", dist.house_height.toString());
params.put("direction", getDirectionCode(house.direction).toString());//朝向
if (house.house_floor) {
params.put("floor", house.house_floor.toString());
}
}
def fitmentCode = null
if(house.fitment){
fitmentCode = getFitmentCode(house.fitment)
params.put("decoration", fitmentCode.toString());//装修
}
params.put("scope", "1")
def result = OpenApiClient.evaluateDistrictPrice(params);
if(result){
return result.avgPrice
}
return null
}
def getDirectionCode(def s){
def directMap = ["西南":1, "东南":2, "南":3, "西北":4, "东北":5, "东":6, "西":7, "北":8, "东西":9, "南北":10]
return directMap.get(s);
}
def getFitmentCode(def s){
if(s){
if(["毛坯"].contains(s)){
return 1;
}else if(["简单装修","简装"].contains(s)){
return 2;
}else if(["精装修","精装"].contains(s)){
return 3;
}else if(["豪华装修","豪装"].contains(s)){
return 4;
}
}
return null
}
def addLatLng(path){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['J'] && d['H'] && (d['H']=="普通住宅" || d['H']=="商住" || d['H']=="别墅")) {
def json = BaiduMapUtils.searchByAddress(d['A'])
if(json && json.location){
d['K'] = json.location.lng
d['L'] = json.location.lat
}
}
})
}
def addAroundPrice(path){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['J'] && d['C'] && d['K']){
def city = RegionData.formatCity(d['C'])
def dist = new BasicDBObject()
dist.city = city
dist.lng = d['K'].toString().toDouble()*10000000000L
dist.lat = d['L'].toString().toDouble()*10000000000L
dist.estateType = "普通住宅"//d['H']
if(dist.lat) {
List<DBObject> list = DbUtils.findAroundDistrictByRadius(dist, 1000)
if (list) {
list = sortByDistance(dist, list);
if (list && list.size() > 10) {
list = list[0..9]
}
if (list) {
def prices = []
for (def s : list) {
def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name)
if (priceObj) {
prices << priceObj.avg_price
}
}
if (prices) {
def arround_price = (int) (prices.sum() / prices.size())
d['J'] = arround_price
}
}
}
}
}
});
}
def sortByDistance(d, list){
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
}
//排序
Collections.sort(list, new Comparator() {
@Override
int compare(Object o1, Object o2) {
if(o1.distance > o2.distance)
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
if(o1.distance == o2.distance)
return 0;
return -1;
}
})
return list;
}
def addAuthor(def path){
def authors = ["董彦均","曹玉彦","付贵生","沈燕芳","Catherine", "王蕾","吉珉", "张宝剑"]
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(row['A']){
if(index < avg){
row['M'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
}
});
}
def addHouse(path){
def datas = []
ExcelUtils.multiThreadingModify(path, { row ->
def commId= row['E']
if(commId){
def dist = CommonDatautils.findDistrictById(commId)
if(dist){
def rets = findHouses(row['H'], row['I'], "sell", 20200101,20230113, row['J'], null, row['k'])
if(rets){
def data = []
for(def r : rets){
data << r.city
data << r.region
data << r.district_name
data << r.floor
data << r.height
data << r.fitment
data << r.direction
data << r.build_type
data << r.area
data << r.avg_price
datas << data
}
}
}
}
});
ExcelUtils.write("D:\\数据\\民生\\民生样本数据.xlsx", datas, "样本数据")
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
// def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
// if(vallaFlag){
// continue;
// }
}
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
// if(!seawater.height){
// seawater.height = null
// }
if(seawater.height && seawater.height>60){
continue;
}
// if(!seawater.room){
// continue;
// }
// if(!seawater.room){
// seawater.room = null
// }
if(seawater.room){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
rowMap.houseId = seawater.id
// rowMap.city = CcbCommonUtils.getCcbCity(city)
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
rowMap.address = seawater.address
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.build_type = seawater.build_type
rowMap.build_date = seawater.build_date
rowMap.district_name = seawater.district_name
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
rets << rowMap
}
}
return rets;
}
}
@@ -0,0 +1,105 @@
package com.fangjia.script.chongqing
import com.fangjia.script.util.ExcelUtils
import org.apache.commons.io.FileUtils
import javax.crypto.Cipher;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import java.util.Base64;
/**
* 3DES加密工具类
* @author qzz
*/
public class ThreeDESUtils {
/**
* 加解密统一编码方式
*/
private final static String ENCODING = "utf-8";
/**
* 加解密方式
*/
private final static String ALGORITHM = "DESede";
/**
*加密模式及填充方式
*/
private final static String PATTERN = "DESede/ECB/pkcs5padding";
/**
* 3DES加密
*
* @param plainText 普通文本
* @param sK 秘钥(24位密码)
* @return
* @throws Exception
*/
public static String encode(String plainText,String sK) throws Exception {
SecretKey secretKey = new SecretKeySpec(sK.getBytes(ENCODING), ALGORITHM);
// 3DES加密采用pkcs5padding填充
Cipher cipher = Cipher.getInstance(PATTERN);
// 用密匙初始化Cipher对象
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
// 执行加密操作
byte[] encryptData = cipher.doFinal(plainText.getBytes(ENCODING));
return Base64.getEncoder().encodeToString(encryptData);
}
/**
* 3DES解密
*
* @param encryptText 加密文本
* @return
* @throws Exception
*/
public static String decode(String encryptText, String sK) throws Exception {
SecretKey secretKey = new SecretKeySpec(sK.getBytes(ENCODING), ALGORITHM);
// 3DES加密采用pkcs5padding填充
Cipher cipher = Cipher.getInstance(PATTERN);
// 用密匙初始化Cipher对象
cipher.init(Cipher.DECRYPT_MODE, secretKey);
// 正式执行解密操作
byte[] decryptData = cipher.doFinal(Base64.getDecoder().decode(encryptText));
return new String(decryptData, ENCODING);
}
public static void main(String[] args) throws Exception {
def rets = []
def datas = readTxt("D:\\数据\\民生\\抽样-住宅类(普通住宅、公寓).txt")
def index = 0
for(def d : datas){
index++
def row = []
def str = decode(d,"111036369260679051122113")
println str
def id = str.substring(str.length()-4)
row << id
row << str.substring(0, str.indexOf(id))
rets << row
}
ExcelUtils.write("D:\\数据\\民生\\民生银行测试数据.xlsx", rets, "抽样-住宅类(普通住宅、公寓)")
// ExcelUtils.multiThreadingModify("D:\\数据\\民生\\加密数据.xlsx", { row ->
// //加密
// String str = row['A']
// //解密
// row['B'] = (decode(str,"111036369260679051122113"));
// });
}
public static def readTxt(def file){
def lines = FileUtils.readLines(new File(file))
def ret = [];
for(line in lines) {
ret << line;
}
return ret;
}
}
@@ -18,6 +18,7 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
//小区月度均价表(带涨跌幅控制)
@BatchOwner(Watcher.GaoJuan)
@@ -78,6 +79,7 @@ class DistrictSellStatBatch extends BaseBatch {
}else{
districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000)
}
print city +" "+districts.size()
def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202211.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202301.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -51,6 +51,7 @@ class AddAvgpriceBatch {
for(def disKey : res2){
def array = disKey.split("##");
def estateType2 = districtKeysMap.get(disKey)
//发重算均价命令
CommonUtils.recalculationAverage(array[0], array[1], array[2], estateType2, 0, new Date())
}
}
@@ -0,0 +1,52 @@
package com.fangjia.script.gj
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
class ShLongShangHangLocation {
static main(args) {
def task = new ShLongShangHangLocation()
task.run()
}
def run(){
def rets = ["上海市黄兴路1599号","上海市杨浦区长阳路1687号A栋","上海市杨浦区佳木思路432号","上海市杨浦区控江路1063号","上海市杨浦区鞍山路100号","上海市杨浦区延吉东路287号","上海市杨浦区殷行路408号","上海市杨浦区闸殷路1555号102室","上海市杨浦区平凉路1500号"]
def map = [:]
for(def r : rets){
def location = getLocation(r)
if(location){
map.put(r, location)
}
}
def titleMap = [0:'C',1:'D',2:'E',3:'F',4:'G',5:'H',6:'I',7:'J',8:'K']
ExcelUtils.modify("D:\\数据\\上海农商行\\网点与居委分布20230118.xlsx",{row->
if(row['B'] && !row['B'].contains("地址") && row['B'].contains('齐齐哈尔路')){
for(def i=0;i<rets.size();i++){
def s = rets.get(i)
def mapLoc = map.get(s)
def location = getLocation("杨浦区"+row['B'])
if(location && mapLoc){
def distance = GeoUtils.calculateDistance(mapLoc.lng.toDouble(), mapLoc.lat.toDouble(), location.lng.toDouble(), location.lat.toDouble())
def title = titleMap.get(i)
row[title] = distance.toInteger()
}
}
println row['B']
}
},0)
}
def getLocation(def address){
def json = GaoDeApiUtils.geo("上海", address)
if (json && json.status == "1") {
def ret = json.geocodes
if (ret) {
def str = ret.get(0).location
def arrs = str.split(",")
return ["lng": arrs[0], "lat": arrs[1]]
}
}
return null
}
}
@@ -0,0 +1,42 @@
package com.fangjia.script.gj.es
import com.fangjia.script.util.ElasticUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
findLogs()
def findLogs(){
def total = 0
try{
// String sql = "select * from fangjia-log* where request like '/jnBank%'"
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
def q = QueryBuilders.fuzzyQuery("domain", ".fangjia.com")
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
println houses.size()
def set = new HashSet()
def index = 0
if(houses) {
for (def d : houses) {
def request = d.request
if(request && !d.domain.contains("k8s2.fangjia.com")){
index ++
println "[${index}] "+d.domain +" "+ request
}
}
}
println set
}catch(Exception e){
e.printStackTrace()
}
return total
}
@@ -401,13 +401,13 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
// if(e.text().contains("福州")){
if(e.text().contains("福州")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
// cityMap.put(e.text(), "https:"+href)
// }
}
}
}
return list
@@ -478,7 +478,7 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia")){
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityMap.put(e.text(), href)
}
}
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20221101", "yyyyMMdd")
def endDate = DateUtil.parseDate("20221201", "yyyyMMdd")
def startDate = DateUtil.parseDate("20230101", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230201", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -38,6 +38,6 @@ while (startDate.before(endDate)) {
startDate = nextDate
}
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行.xlsx", data, "202211")
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301")