Merge branch 'script_gj' into 'master'
Script gj See merge request !411
This commit is contained in:
+1
-1
@@ -58,7 +58,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
|
||||
if(args && args.size()>=2){
|
||||
curDate = args[1].toString().toInteger()
|
||||
}
|
||||
|
||||
+27
-27
@@ -25,32 +25,32 @@ class ExportOracleData {
|
||||
|
||||
def run(args) {
|
||||
citys = CcbCommonUtils.getRunningCitys(args)
|
||||
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
def new_dists = []
|
||||
//不会变
|
||||
exportEvaluationRateData()
|
||||
exportBigdataPoiData()//
|
||||
exportBigdataDistrictData()//
|
||||
|
||||
exportGuidedPrice()//随人工改变//
|
||||
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
|
||||
|
||||
// //处理小区情况
|
||||
exportBigdataCommunity()//
|
||||
exportCommunityStaging()//没cityid
|
||||
//不会变 10.25
|
||||
// exportEvaluationRateData()
|
||||
// exportBigdataPoiData()//
|
||||
// exportBigdataDistrictData()//
|
||||
//
|
||||
// exportGuidedPrice()//随人工改变//
|
||||
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
|
||||
//
|
||||
//// //处理小区情况
|
||||
// exportBigdataCommunity()//
|
||||
// exportCommunityStaging()//没cityid
|
||||
exportCommunityBuileType()//
|
||||
|
||||
exportThirdPartyPrice2(lastMonth, new_dists)//
|
||||
exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19
|
||||
exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
|
||||
|
||||
//第三方网站权重表
|
||||
exportWebsiteRate();
|
||||
|
||||
|
||||
//计算结果相关
|
||||
//
|
||||
// exportThirdPartyPrice2(lastMonth, new_dists)//
|
||||
// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19
|
||||
// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
|
||||
//
|
||||
// //第三方网站权重表
|
||||
// exportWebsiteRate();
|
||||
//
|
||||
//
|
||||
// //计算结果相关
|
||||
// exportAvmEvaluationGJNB(curMonth)
|
||||
// exportAVMEvaluationItemGJNB(curMonth)
|
||||
//
|
||||
@@ -62,11 +62,11 @@ class ExportOracleData {
|
||||
//
|
||||
// exportCommunityPdescribeGj(curMonth)
|
||||
// exportCommCaseDetailGj(curMonth)
|
||||
|
||||
//
|
||||
// def curDate = curMonth.toString()+"28"
|
||||
// SSHHelper jschCommand = new SSHHelper();
|
||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||
//ok文件
|
||||
//// ok文件
|
||||
// exportLastFile()
|
||||
}
|
||||
|
||||
@@ -229,8 +229,8 @@ class ExportOracleData {
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......"
|
||||
// CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
// println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataDistrictData() {
|
||||
@@ -377,7 +377,7 @@ class ExportOracleData {
|
||||
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
|
||||
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
|
||||
"from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')";
|
||||
allRows = dealData(sql)
|
||||
allRows.addAll(dealData(sql))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys_252;
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
for (def city : citys) {
|
||||
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2021-09-01", "2022-10-12")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-21", "2022-10-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+14
-14
@@ -1001,10 +1001,10 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
|
||||
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入人工查价_0922_300.xlsx"
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx"
|
||||
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['A']){
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['A']){
|
||||
// def region = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
|
||||
@@ -1013,18 +1013,18 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
|
||||
// d['F'] = dist._id.toString()
|
||||
// }
|
||||
// }
|
||||
//// if(!d['Q']){
|
||||
//// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
//////
|
||||
//// if(dist){
|
||||
//// d['H'] = dist.region
|
||||
////// d['G'] = dist.name
|
||||
//// d['I'] = dist.status
|
||||
//// }
|
||||
//// }
|
||||
if(!d['F']){
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
//
|
||||
// }
|
||||
//},0)
|
||||
if(dist){
|
||||
// d['H'] = dist.region
|
||||
// d['G'] = dist.name
|
||||
d['F'] = dist.status
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
},0)
|
||||
|
||||
|
||||
|
||||
|
||||
+24
-1
@@ -29,7 +29,30 @@ class DataMatch {
|
||||
// task.run3()
|
||||
// println task.findAddress("昌州街道棠香路")
|
||||
|
||||
task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
|
||||
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
|
||||
task.fzWeiyuanshuMatch("D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表_匹配.xlsx")
|
||||
}
|
||||
|
||||
def fzWeiyuanshuMatch(fileName){
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
if(!d['K'] && d['C']){
|
||||
def arr = OpenApiClient.search("福州", d['G'], d['C'])
|
||||
if(arr){
|
||||
d['H'] = arr.city
|
||||
d['I'] = arr.district
|
||||
d['J'] = arr.name
|
||||
}
|
||||
}
|
||||
|
||||
// def results = null
|
||||
// if(!d['K'] && d['C']){
|
||||
// results = OpenApiClient.addressMatch("福州",d['C'], "property", false, "长乐", null)
|
||||
// }
|
||||
// if(results && results.precise>=0){
|
||||
// d['L'] = results.district
|
||||
// d['M'] = results.name
|
||||
// }
|
||||
},0)
|
||||
}
|
||||
|
||||
def run1(){
|
||||
|
||||
+16
-13
@@ -91,11 +91,9 @@ class DataStandardBatch extends BaseBatch{
|
||||
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
|
||||
|
||||
cityMap.each{city,alias->
|
||||
// if(city == "北京"){
|
||||
println "开始加载【${city}】"
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||
districtMap.putAll(detailMap)
|
||||
// }
|
||||
println "开始加载【${city}】"
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||
districtMap.putAll(detailMap)
|
||||
}
|
||||
|
||||
while(flag){
|
||||
@@ -122,9 +120,9 @@ class DataStandardBatch extends BaseBatch{
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
def final AtomicInteger successIndex = new AtomicInteger(0)
|
||||
datas.each { dbo ->
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
// fixedThreadPool.execute(
|
||||
// new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
|
||||
if(dbo.name){
|
||||
@@ -164,12 +162,12 @@ class DataStandardBatch extends BaseBatch{
|
||||
}finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
// }
|
||||
// }
|
||||
// );
|
||||
}
|
||||
|
||||
latch.await()
|
||||
// latch.await()
|
||||
long end = System.currentTimeMillis()
|
||||
|
||||
println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss")
|
||||
@@ -502,6 +500,8 @@ class DataStandardBatch extends BaseBatch{
|
||||
type = 12;//区域不同
|
||||
district = null;
|
||||
}else{
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(district.name, copyObj.name)
|
||||
district.score = score //此处用来做测试
|
||||
type=15
|
||||
}
|
||||
}
|
||||
@@ -922,7 +922,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
qdata.put("s_date", SHORT_DATE)
|
||||
// qdata.put("city","北京")
|
||||
// qdata.put("source","安居客")
|
||||
// qdata.put("name",Pattern.compile("金隅澜湾.*"))
|
||||
// qdata.put("name",Pattern.compile("龚家宅.*"))
|
||||
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
||||
def ret = []
|
||||
try{
|
||||
@@ -1065,6 +1065,9 @@ class DataStandardBatch extends BaseBatch{
|
||||
udata2.put("match_region", district.region);
|
||||
udata2.put("match_name", district.name);
|
||||
udata2.put("match_address", district.address);
|
||||
if(district.score){
|
||||
udata2.put("score", district.score);
|
||||
}
|
||||
udata2.put("avg_price", data.avg_price);
|
||||
udata2.put("type", type);
|
||||
udata2.put("tid", data._id.toString());
|
||||
|
||||
@@ -52,6 +52,22 @@ public class OpenApiClient {
|
||||
return false
|
||||
}
|
||||
|
||||
public static def search(def city, def district, def name){
|
||||
Map<String, String> params = new HashMap<String, String>()
|
||||
params.put("city", city);
|
||||
params.put("keyword", name);
|
||||
params.put("district", district)
|
||||
params.put("token", TOKEN);
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/property/search", params, "utf-8")
|
||||
JSONObject jsonObj = JSONObject.fromObject(content)
|
||||
JSONArray array = jsonObj.get("result")
|
||||
def matchObj = null
|
||||
if(array){
|
||||
matchObj = array[0]
|
||||
}
|
||||
return matchObj
|
||||
}
|
||||
|
||||
public static List searchDistrict(double lng, double lat, int r, String city) {
|
||||
DecimalFormat format = new DecimalFormat("#.0000000000");
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user