Merge branch 'script_gj' into 'master'

Script gj

See merge request !411
This commit is contained in:
2022-10-25 14:23:52 +08:00
7 changed files with 100 additions and 58 deletions
@@ -58,7 +58,7 @@ class BigdataDealBatch extends BaseBatch{
public void process(Object args) { public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys; // def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args) def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
if(args && args.size()>=2){ if(args && args.size()>=2){
curDate = args[1].toString().toInteger() curDate = args[1].toString().toInteger()
} }
@@ -25,32 +25,32 @@ class ExportOracleData {
def run(args) { def run(args) {
citys = CcbCommonUtils.getRunningCitys(args) citys = CcbCommonUtils.getRunningCitys(args)
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
def new_dists = [] def new_dists = []
//不会变 //不会变 10.25
exportEvaluationRateData() // exportEvaluationRateData()
exportBigdataPoiData()// // exportBigdataPoiData()//
exportBigdataDistrictData()// // exportBigdataDistrictData()//
//
exportGuidedPrice()//随人工改变// // exportGuidedPrice()//随人工改变//
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// // exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
//
// //处理小区情况 //// //处理小区情况
exportBigdataCommunity()// // exportBigdataCommunity()//
exportCommunityStaging()//没cityid // exportCommunityStaging()//没cityid
exportCommunityBuileType()// exportCommunityBuileType()//
//
exportThirdPartyPrice2(lastMonth, new_dists)// // exportThirdPartyPrice2(lastMonth, new_dists)//
exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 // exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19
exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 // exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
//
//第三方网站权重表 // //第三方网站权重表
exportWebsiteRate(); // exportWebsiteRate();
//
//
//计算结果相关 // //计算结果相关
// exportAvmEvaluationGJNB(curMonth) // exportAvmEvaluationGJNB(curMonth)
// exportAVMEvaluationItemGJNB(curMonth) // exportAVMEvaluationItemGJNB(curMonth)
// //
@@ -62,11 +62,11 @@ class ExportOracleData {
// //
// exportCommunityPdescribeGj(curMonth) // exportCommunityPdescribeGj(curMonth)
// exportCommCaseDetailGj(curMonth) // exportCommCaseDetailGj(curMonth)
//
// def curDate = curMonth.toString()+"28" // def curDate = curMonth.toString()+"28"
// SSHHelper jschCommand = new SSHHelper(); // SSHHelper jschCommand = new SSHHelper();
// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); // System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
//ok文件 //// ok文件
// exportLastFile() // exportLastFile()
} }
@@ -229,8 +229,8 @@ class ExportOracleData {
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......" println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate) // CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......" // println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......"
} }
def exportBigdataDistrictData() { def exportBigdataDistrictData() {
@@ -377,7 +377,7 @@ class ExportOracleData {
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
"from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')";
allRows = dealData(sql) allRows.addAll(dealData(sql))
} }
} }
@@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
def citys = CcbCommonUtils.citys_252; def citys = CcbCommonUtils.getRunningCitys(args)
def rows = [] def rows = []
for (def city : citys) { for (def city : citys) {
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价 //添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2021-09-01", "2022-10-12") def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-21", "2022-10-22")
if(prices){ if(prices){
def distMap = getDistrictMap(city) def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -1001,10 +1001,10 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入人工查价_0922_300.xlsx" def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx"
//ExcelUtils.multiThreadingModify(fileName, {d-> ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['A']){ if(d['A']){
// def region = RegionData.formatRegion(d['A'], d['B']) // def region = RegionData.formatRegion(d['A'], d['B'])
// if(region){ // if(region){
// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") // def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
@@ -1013,18 +1013,18 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
// d['F'] = dist._id.toString() // d['F'] = dist._id.toString()
// } // }
// } // }
//// if(!d['Q']){ if(!d['F']){
//// def dist = CommonDatautils.findDistrictById(d['A']) def dist = CommonDatautils.findDistrictById(d['A'])
//////
//// if(dist){
//// d['H'] = dist.region
////// d['G'] = dist.name
//// d['I'] = dist.status
//// }
//// }
// //
// } if(dist){
//},0) // d['H'] = dist.region
// d['G'] = dist.name
d['F'] = dist.status
}
}
}
},0)
@@ -29,7 +29,30 @@ class DataMatch {
// task.run3() // task.run3()
// println task.findAddress("昌州街道棠香路") // println task.findAddress("昌州街道棠香路")
task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") // task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
task.fzWeiyuanshuMatch("D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表_匹配.xlsx")
}
def fzWeiyuanshuMatch(fileName){
ExcelUtils.modify(fileName,{d->
if(!d['K'] && d['C']){
def arr = OpenApiClient.search("福州", d['G'], d['C'])
if(arr){
d['H'] = arr.city
d['I'] = arr.district
d['J'] = arr.name
}
}
// def results = null
// if(!d['K'] && d['C']){
// results = OpenApiClient.addressMatch("福州",d['C'], "property", false, "长乐", null)
// }
// if(results && results.precise>=0){
// d['L'] = results.district
// d['M'] = results.name
// }
},0)
} }
def run1(){ def run1(){
@@ -91,11 +91,9 @@ class DataStandardBatch extends BaseBatch{
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl) // ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
cityMap.each{city,alias-> cityMap.each{city,alias->
// if(city == "北京"){ println "开始加载【${city}】"
println "开始加载【${city}】" def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) districtMap.putAll(detailMap)
districtMap.putAll(detailMap)
// }
} }
while(flag){ while(flag){
@@ -122,9 +120,9 @@ class DataStandardBatch extends BaseBatch{
def final AtomicInteger count_ = new AtomicInteger(0) def final AtomicInteger count_ = new AtomicInteger(0)
def final AtomicInteger successIndex = new AtomicInteger(0) def final AtomicInteger successIndex = new AtomicInteger(0)
datas.each { dbo -> datas.each { dbo ->
fixedThreadPool.execute( // fixedThreadPool.execute(
new Runnable() { // new Runnable() {
void run() { // void run() {
try{ try{
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】" println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
if(dbo.name){ if(dbo.name){
@@ -164,12 +162,12 @@ class DataStandardBatch extends BaseBatch{
}finally { }finally {
latch.countDown() latch.countDown()
} }
} // }
} // }
); // );
} }
latch.await() // latch.await()
long end = System.currentTimeMillis() long end = System.currentTimeMillis()
println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss") println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss")
@@ -502,6 +500,8 @@ class DataStandardBatch extends BaseBatch{
type = 12;//区域不同 type = 12;//区域不同
district = null; district = null;
}else{ }else{
def score = CosineSimilarAlgorithm.getSimilarity(district.name, copyObj.name)
district.score = score //此处用来做测试
type=15 type=15
} }
} }
@@ -922,7 +922,7 @@ class DataStandardBatch extends BaseBatch{
qdata.put("s_date", SHORT_DATE) qdata.put("s_date", SHORT_DATE)
// qdata.put("city","北京") // qdata.put("city","北京")
// qdata.put("source","安居客") // qdata.put("source","安居客")
// qdata.put("name",Pattern.compile("金隅澜湾.*")) // qdata.put("name",Pattern.compile("龚家宅.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
def ret = [] def ret = []
try{ try{
@@ -1065,6 +1065,9 @@ class DataStandardBatch extends BaseBatch{
udata2.put("match_region", district.region); udata2.put("match_region", district.region);
udata2.put("match_name", district.name); udata2.put("match_name", district.name);
udata2.put("match_address", district.address); udata2.put("match_address", district.address);
if(district.score){
udata2.put("score", district.score);
}
udata2.put("avg_price", data.avg_price); udata2.put("avg_price", data.avg_price);
udata2.put("type", type); udata2.put("type", type);
udata2.put("tid", data._id.toString()); udata2.put("tid", data._id.toString());
@@ -52,6 +52,22 @@ public class OpenApiClient {
return false return false
} }
public static def search(def city, def district, def name){
Map<String, String> params = new HashMap<String, String>()
params.put("city", city);
params.put("keyword", name);
params.put("district", district)
params.put("token", TOKEN);
String content = new HttpRequest().get("http://open.fangjia.com/property/search", params, "utf-8")
JSONObject jsonObj = JSONObject.fromObject(content)
JSONArray array = jsonObj.get("result")
def matchObj = null
if(array){
matchObj = array[0]
}
return matchObj
}
public static List searchDistrict(double lng, double lat, int r, String city) { public static List searchDistrict(double lng, double lat, int r, String city) {
DecimalFormat format = new DecimalFormat("#.0000000000"); DecimalFormat format = new DecimalFormat("#.0000000000");
try { try {