Merge branch 'script_gj' into 'master'
Script gj See merge request !411
This commit is contained in:
+1
-1
@@ -58,7 +58,7 @@ class BigdataDealBatch extends BaseBatch{
|
|||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||||
// def citys = CcbCommonUtils.citys;
|
// def citys = CcbCommonUtils.citys;
|
||||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
|
||||||
if(args && args.size()>=2){
|
if(args && args.size()>=2){
|
||||||
curDate = args[1].toString().toInteger()
|
curDate = args[1].toString().toInteger()
|
||||||
}
|
}
|
||||||
|
|||||||
+27
-27
@@ -25,32 +25,32 @@ class ExportOracleData {
|
|||||||
|
|
||||||
def run(args) {
|
def run(args) {
|
||||||
citys = CcbCommonUtils.getRunningCitys(args)
|
citys = CcbCommonUtils.getRunningCitys(args)
|
||||||
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||||
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||||
def new_dists = []
|
def new_dists = []
|
||||||
//不会变
|
//不会变 10.25
|
||||||
exportEvaluationRateData()
|
// exportEvaluationRateData()
|
||||||
exportBigdataPoiData()//
|
// exportBigdataPoiData()//
|
||||||
exportBigdataDistrictData()//
|
// exportBigdataDistrictData()//
|
||||||
|
//
|
||||||
exportGuidedPrice()//随人工改变//
|
// exportGuidedPrice()//随人工改变//
|
||||||
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
|
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
|
||||||
|
//
|
||||||
// //处理小区情况
|
//// //处理小区情况
|
||||||
exportBigdataCommunity()//
|
// exportBigdataCommunity()//
|
||||||
exportCommunityStaging()//没cityid
|
// exportCommunityStaging()//没cityid
|
||||||
exportCommunityBuileType()//
|
exportCommunityBuileType()//
|
||||||
|
//
|
||||||
exportThirdPartyPrice2(lastMonth, new_dists)//
|
// exportThirdPartyPrice2(lastMonth, new_dists)//
|
||||||
exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19
|
// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19
|
||||||
exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
|
// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
|
||||||
|
//
|
||||||
//第三方网站权重表
|
// //第三方网站权重表
|
||||||
exportWebsiteRate();
|
// exportWebsiteRate();
|
||||||
|
//
|
||||||
|
//
|
||||||
//计算结果相关
|
// //计算结果相关
|
||||||
// exportAvmEvaluationGJNB(curMonth)
|
// exportAvmEvaluationGJNB(curMonth)
|
||||||
// exportAVMEvaluationItemGJNB(curMonth)
|
// exportAVMEvaluationItemGJNB(curMonth)
|
||||||
//
|
//
|
||||||
@@ -62,11 +62,11 @@ class ExportOracleData {
|
|||||||
//
|
//
|
||||||
// exportCommunityPdescribeGj(curMonth)
|
// exportCommunityPdescribeGj(curMonth)
|
||||||
// exportCommCaseDetailGj(curMonth)
|
// exportCommCaseDetailGj(curMonth)
|
||||||
|
//
|
||||||
// def curDate = curMonth.toString()+"28"
|
// def curDate = curMonth.toString()+"28"
|
||||||
// SSHHelper jschCommand = new SSHHelper();
|
// SSHHelper jschCommand = new SSHHelper();
|
||||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||||
//ok文件
|
//// ok文件
|
||||||
// exportLastFile()
|
// exportLastFile()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -229,8 +229,8 @@ class ExportOracleData {
|
|||||||
|
|
||||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||||
println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......"
|
println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......"
|
||||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
// CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||||
println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......"
|
// println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......"
|
||||||
}
|
}
|
||||||
|
|
||||||
def exportBigdataDistrictData() {
|
def exportBigdataDistrictData() {
|
||||||
@@ -377,7 +377,7 @@ class ExportOracleData {
|
|||||||
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
|
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
|
||||||
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
|
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
|
||||||
"from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')";
|
"from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')";
|
||||||
allRows = dealData(sql)
|
allRows.addAll(dealData(sql))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
def citys = CcbCommonUtils.citys_252;
|
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||||
def rows = []
|
def rows = []
|
||||||
|
|
||||||
for (def city : citys) {
|
for (def city : citys) {
|
||||||
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
|||||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||||
|
|
||||||
//添加自己库中的人工指导价
|
//添加自己库中的人工指导价
|
||||||
def prices = CcbCommonUtils.findPersonPrice(city, "2021-09-01", "2022-10-12")
|
def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-21", "2022-10-22")
|
||||||
if(prices){
|
if(prices){
|
||||||
def distMap = getDistrictMap(city)
|
def distMap = getDistrictMap(city)
|
||||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||||
|
|||||||
+14
-14
@@ -1001,10 +1001,10 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入人工查价_0922_300.xlsx"
|
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx"
|
||||||
|
|
||||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||||
// if(d['A']){
|
if(d['A']){
|
||||||
// def region = RegionData.formatRegion(d['A'], d['B'])
|
// def region = RegionData.formatRegion(d['A'], d['B'])
|
||||||
// if(region){
|
// if(region){
|
||||||
// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
|
// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
|
||||||
@@ -1013,18 +1013,18 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
|
|||||||
// d['F'] = dist._id.toString()
|
// d['F'] = dist._id.toString()
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
//// if(!d['Q']){
|
if(!d['F']){
|
||||||
//// def dist = CommonDatautils.findDistrictById(d['A'])
|
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||||
//////
|
|
||||||
//// if(dist){
|
|
||||||
//// d['H'] = dist.region
|
|
||||||
////// d['G'] = dist.name
|
|
||||||
//// d['I'] = dist.status
|
|
||||||
//// }
|
|
||||||
//// }
|
|
||||||
//
|
//
|
||||||
// }
|
if(dist){
|
||||||
//},0)
|
// d['H'] = dist.region
|
||||||
|
// d['G'] = dist.name
|
||||||
|
d['F'] = dist.status
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
},0)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+24
-1
@@ -29,7 +29,30 @@ class DataMatch {
|
|||||||
// task.run3()
|
// task.run3()
|
||||||
// println task.findAddress("昌州街道棠香路")
|
// println task.findAddress("昌州街道棠香路")
|
||||||
|
|
||||||
task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
|
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
|
||||||
|
task.fzWeiyuanshuMatch("D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表_匹配.xlsx")
|
||||||
|
}
|
||||||
|
|
||||||
|
def fzWeiyuanshuMatch(fileName){
|
||||||
|
ExcelUtils.modify(fileName,{d->
|
||||||
|
if(!d['K'] && d['C']){
|
||||||
|
def arr = OpenApiClient.search("福州", d['G'], d['C'])
|
||||||
|
if(arr){
|
||||||
|
d['H'] = arr.city
|
||||||
|
d['I'] = arr.district
|
||||||
|
d['J'] = arr.name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// def results = null
|
||||||
|
// if(!d['K'] && d['C']){
|
||||||
|
// results = OpenApiClient.addressMatch("福州",d['C'], "property", false, "长乐", null)
|
||||||
|
// }
|
||||||
|
// if(results && results.precise>=0){
|
||||||
|
// d['L'] = results.district
|
||||||
|
// d['M'] = results.name
|
||||||
|
// }
|
||||||
|
},0)
|
||||||
}
|
}
|
||||||
|
|
||||||
def run1(){
|
def run1(){
|
||||||
|
|||||||
+16
-13
@@ -91,11 +91,9 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
|
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
|
||||||
|
|
||||||
cityMap.each{city,alias->
|
cityMap.each{city,alias->
|
||||||
// if(city == "北京"){
|
println "开始加载【${city}】"
|
||||||
println "开始加载【${city}】"
|
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
districtMap.putAll(detailMap)
|
||||||
districtMap.putAll(detailMap)
|
|
||||||
// }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
while(flag){
|
while(flag){
|
||||||
@@ -122,9 +120,9 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||||
def final AtomicInteger successIndex = new AtomicInteger(0)
|
def final AtomicInteger successIndex = new AtomicInteger(0)
|
||||||
datas.each { dbo ->
|
datas.each { dbo ->
|
||||||
fixedThreadPool.execute(
|
// fixedThreadPool.execute(
|
||||||
new Runnable() {
|
// new Runnable() {
|
||||||
void run() {
|
// void run() {
|
||||||
try{
|
try{
|
||||||
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
|
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
|
||||||
if(dbo.name){
|
if(dbo.name){
|
||||||
@@ -164,12 +162,12 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
}finally {
|
}finally {
|
||||||
latch.countDown()
|
latch.countDown()
|
||||||
}
|
}
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
);
|
// );
|
||||||
}
|
}
|
||||||
|
|
||||||
latch.await()
|
// latch.await()
|
||||||
long end = System.currentTimeMillis()
|
long end = System.currentTimeMillis()
|
||||||
|
|
||||||
println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss")
|
println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss")
|
||||||
@@ -502,6 +500,8 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
type = 12;//区域不同
|
type = 12;//区域不同
|
||||||
district = null;
|
district = null;
|
||||||
}else{
|
}else{
|
||||||
|
def score = CosineSimilarAlgorithm.getSimilarity(district.name, copyObj.name)
|
||||||
|
district.score = score //此处用来做测试
|
||||||
type=15
|
type=15
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -922,7 +922,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
qdata.put("s_date", SHORT_DATE)
|
qdata.put("s_date", SHORT_DATE)
|
||||||
// qdata.put("city","北京")
|
// qdata.put("city","北京")
|
||||||
// qdata.put("source","安居客")
|
// qdata.put("source","安居客")
|
||||||
// qdata.put("name",Pattern.compile("金隅澜湾.*"))
|
// qdata.put("name",Pattern.compile("龚家宅.*"))
|
||||||
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
||||||
def ret = []
|
def ret = []
|
||||||
try{
|
try{
|
||||||
@@ -1065,6 +1065,9 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
udata2.put("match_region", district.region);
|
udata2.put("match_region", district.region);
|
||||||
udata2.put("match_name", district.name);
|
udata2.put("match_name", district.name);
|
||||||
udata2.put("match_address", district.address);
|
udata2.put("match_address", district.address);
|
||||||
|
if(district.score){
|
||||||
|
udata2.put("score", district.score);
|
||||||
|
}
|
||||||
udata2.put("avg_price", data.avg_price);
|
udata2.put("avg_price", data.avg_price);
|
||||||
udata2.put("type", type);
|
udata2.put("type", type);
|
||||||
udata2.put("tid", data._id.toString());
|
udata2.put("tid", data._id.toString());
|
||||||
|
|||||||
@@ -52,6 +52,22 @@ public class OpenApiClient {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static def search(def city, def district, def name){
|
||||||
|
Map<String, String> params = new HashMap<String, String>()
|
||||||
|
params.put("city", city);
|
||||||
|
params.put("keyword", name);
|
||||||
|
params.put("district", district)
|
||||||
|
params.put("token", TOKEN);
|
||||||
|
String content = new HttpRequest().get("http://open.fangjia.com/property/search", params, "utf-8")
|
||||||
|
JSONObject jsonObj = JSONObject.fromObject(content)
|
||||||
|
JSONArray array = jsonObj.get("result")
|
||||||
|
def matchObj = null
|
||||||
|
if(array){
|
||||||
|
matchObj = array[0]
|
||||||
|
}
|
||||||
|
return matchObj
|
||||||
|
}
|
||||||
|
|
||||||
public static List searchDistrict(double lng, double lat, int r, String city) {
|
public static List searchDistrict(double lng, double lat, int r, String city) {
|
||||||
DecimalFormat format = new DecimalFormat("#.0000000000");
|
DecimalFormat format = new DecimalFormat("#.0000000000");
|
||||||
try {
|
try {
|
||||||
|
|||||||
Reference in New Issue
Block a user