Merge branch 'script_gj' into 'master'

Script gj

See merge request !411
This commit is contained in:
2022-10-25 14:23:52 +08:00
7 changed files with 100 additions and 58 deletions
@@ -58,7 +58,7 @@ class BigdataDealBatch extends BaseBatch{
public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
if(args && args.size()>=2){
curDate = args[1].toString().toInteger()
}
@@ -25,32 +25,32 @@ class ExportOracleData {
def run(args) {
citys = CcbCommonUtils.getRunningCitys(args)
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
def new_dists = []
//不会变
exportEvaluationRateData()
exportBigdataPoiData()//
exportBigdataDistrictData()//
exportGuidedPrice()//随人工改变//
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
// //处理小区情况
exportBigdataCommunity()//
exportCommunityStaging()//没cityid
//不会变 10.25
// exportEvaluationRateData()
// exportBigdataPoiData()//
// exportBigdataDistrictData()//
//
// exportGuidedPrice()//随人工改变//
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
//
//// //处理小区情况
// exportBigdataCommunity()//
// exportCommunityStaging()//没cityid
exportCommunityBuileType()//
exportThirdPartyPrice2(lastMonth, new_dists)//
exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19
exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
//第三方网站权重表
exportWebsiteRate();
//计算结果相关
//
// exportThirdPartyPrice2(lastMonth, new_dists)//
// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19
// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
//
// //第三方网站权重表
// exportWebsiteRate();
//
//
// //计算结果相关
// exportAvmEvaluationGJNB(curMonth)
// exportAVMEvaluationItemGJNB(curMonth)
//
@@ -62,11 +62,11 @@ class ExportOracleData {
//
// exportCommunityPdescribeGj(curMonth)
// exportCommCaseDetailGj(curMonth)
//
// def curDate = curMonth.toString()+"28"
// SSHHelper jschCommand = new SSHHelper();
// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
//ok文件
//// ok文件
// exportLastFile()
}
@@ -229,8 +229,8 @@ class ExportOracleData {
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......"
// CcbFtpUtils.ftpUpload(null, fileName, curDate)
// println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......"
}
def exportBigdataDistrictData() {
@@ -377,7 +377,7 @@ class ExportOracleData {
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
"from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')";
allRows = dealData(sql)
allRows.addAll(dealData(sql))
}
}
@@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys_252;
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
for (def city : citys) {
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2021-09-01", "2022-10-12")
def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-21", "2022-10-22")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -1001,10 +1001,10 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入人工查价_0922_300.xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx"
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['A']){
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['A']){
// def region = RegionData.formatRegion(d['A'], d['B'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
@@ -1013,18 +1013,18 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
// d['F'] = dist._id.toString()
// }
// }
//// if(!d['Q']){
//// def dist = CommonDatautils.findDistrictById(d['A'])
//////
//// if(dist){
//// d['H'] = dist.region
////// d['G'] = dist.name
//// d['I'] = dist.status
//// }
//// }
if(!d['F']){
def dist = CommonDatautils.findDistrictById(d['A'])
//
// }
//},0)
if(dist){
// d['H'] = dist.region
// d['G'] = dist.name
d['F'] = dist.status
}
}
}
},0)
@@ -29,7 +29,30 @@ class DataMatch {
// task.run3()
// println task.findAddress("昌州街道棠香路")
task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
task.fzWeiyuanshuMatch("D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表_匹配.xlsx")
}
def fzWeiyuanshuMatch(fileName){
ExcelUtils.modify(fileName,{d->
if(!d['K'] && d['C']){
def arr = OpenApiClient.search("福州", d['G'], d['C'])
if(arr){
d['H'] = arr.city
d['I'] = arr.district
d['J'] = arr.name
}
}
// def results = null
// if(!d['K'] && d['C']){
// results = OpenApiClient.addressMatch("福州",d['C'], "property", false, "长乐", null)
// }
// if(results && results.precise>=0){
// d['L'] = results.district
// d['M'] = results.name
// }
},0)
}
def run1(){
@@ -91,11 +91,9 @@ class DataStandardBatch extends BaseBatch{
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
cityMap.each{city,alias->
// if(city == "北京"){
println "开始加载【${city}】"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
// }
println "开始加载【${city}】"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
}
while(flag){
@@ -122,9 +120,9 @@ class DataStandardBatch extends BaseBatch{
def final AtomicInteger count_ = new AtomicInteger(0)
def final AtomicInteger successIndex = new AtomicInteger(0)
datas.each { dbo ->
fixedThreadPool.execute(
new Runnable() {
void run() {
// fixedThreadPool.execute(
// new Runnable() {
// void run() {
try{
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
if(dbo.name){
@@ -164,12 +162,12 @@ class DataStandardBatch extends BaseBatch{
}finally {
latch.countDown()
}
}
}
);
// }
// }
// );
}
latch.await()
// latch.await()
long end = System.currentTimeMillis()
println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss")
@@ -502,6 +500,8 @@ class DataStandardBatch extends BaseBatch{
type = 12;//区域不同
district = null;
}else{
def score = CosineSimilarAlgorithm.getSimilarity(district.name, copyObj.name)
district.score = score //此处用来做测试
type=15
}
}
@@ -922,7 +922,7 @@ class DataStandardBatch extends BaseBatch{
qdata.put("s_date", SHORT_DATE)
// qdata.put("city","北京")
// qdata.put("source","安居客")
// qdata.put("name",Pattern.compile("金隅澜湾.*"))
// qdata.put("name",Pattern.compile("龚家宅.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
def ret = []
try{
@@ -1065,6 +1065,9 @@ class DataStandardBatch extends BaseBatch{
udata2.put("match_region", district.region);
udata2.put("match_name", district.name);
udata2.put("match_address", district.address);
if(district.score){
udata2.put("score", district.score);
}
udata2.put("avg_price", data.avg_price);
udata2.put("type", type);
udata2.put("tid", data._id.toString());
@@ -52,6 +52,22 @@ public class OpenApiClient {
return false
}
public static def search(def city, def district, def name){
Map<String, String> params = new HashMap<String, String>()
params.put("city", city);
params.put("keyword", name);
params.put("district", district)
params.put("token", TOKEN);
String content = new HttpRequest().get("http://open.fangjia.com/property/search", params, "utf-8")
JSONObject jsonObj = JSONObject.fromObject(content)
JSONArray array = jsonObj.get("result")
def matchObj = null
if(array){
matchObj = array[0]
}
return matchObj
}
public static List searchDistrict(double lng, double lat, int r, String city) {
DecimalFormat format = new DecimalFormat("#.0000000000");
try {