From 6ff9f665b0bb204d31843058b0441b2a1e3e471c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Sat, 17 Jul 2021 21:42:28 +0800 Subject: [PATCH 01/80] geng xin --- .../fangjia/script/gj/GaoDeMapCrawl.groovy | 2 - .../script/wenyan/auto/ReCalcAvgPrice.groovy | 32 +++++ .../wenyan/auto/SaveMapPoiToSeaweed.groovy | 113 ++++++++++++++++++ .../fangjia/script/wenyan/temp/QinLin.groovy | 8 +- .../wenyan/temp/SiChuanJingCheng.groovy | 6 +- .../fangjia/script/wenyan/temp/Test7.groovy | 85 ++++++++++++- 6 files changed, 233 insertions(+), 13 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/ReCalcAvgPrice.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/SaveMapPoiToSeaweed.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index e93f98720..41bb9db8e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -38,9 +38,7 @@ class GaoDeMapCrawl extends BaseBatch{ DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018) def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc" - final def dbUrl = "mongodb://192.168.0.235:27018"; - ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) final AtomicInteger count = new AtomicInteger() def updateLists = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/ReCalcAvgPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/ReCalcAvgPrice.groovy new file mode 100644 index 000000000..5fceee582 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/ReCalcAvgPrice.groovy @@ -0,0 +1,32 @@ +package com.fangjia.script.wenyan.auto + +import com.fangjia.core.util.DateUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject + +//def keys = new BasicDBObject() +//keys.city = 1 +//keys.region = 1 +//keys.name = 1 +//keys.estate_type2 = 1 +//["重庆"].each {def city-> +// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject seaweed-> +// BasicDBObject stat = DbUtils.findDistrictStat(seaweed, DateUtil.getFirstDayOfWeek(new Date())) +// if(!stat){ +// CommonUtils.recalculationAverage(seaweed.city, seaweed.region, seaweed.name, seaweed.estate_type2) +// println(seaweed) +// } +// }, keys) +// +//} +ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\成交价——预测家.xlsx",{List item-> + BasicDBObject query = DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"]) + BasicDBObject seaweed = DbUtils.seaweed.findOne(query) + if(seaweed){ + BasicDBObject stat = DbUtils.findDistrictStat(seaweed, item["E"]) + item["H"] = stat?.credit + } + println(item) +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/SaveMapPoiToSeaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/SaveMapPoiToSeaweed.groovy new file mode 100644 index 000000000..b603f367e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/SaveMapPoiToSeaweed.groovy @@ -0,0 +1,113 @@ +package com.fangjia.script.wenyan.auto + +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +import org.apache.lucene.index.Term +import org.apache.lucene.search.BooleanClause +import org.apache.lucene.search.BooleanQuery +import org.apache.lucene.search.TermQuery + +/** + * 移除小区名中的区域信息 + * @param name + * @param region + * @return + */ +String removeRegion(String name, String region){ + if(name.startsWith(region) && name.length() - region.length() > 2){ + return name.replaceFirst(region, "") + } + return name; +} + +def String getStandardAddress(def city, def addr) { + if (StringUtils.isBlank(addr)) { + return null + } + JSONObject result = OpenApiClient.addressStandardize(city, addr) + if (result.containsKey("standardAddress") && result.getBoolean("standardAddress")) { + def standardAddr = result.getString("street") + if (result.containsKey("streetNumber")) { + standardAddr += result.getString("streetNumber") + } + return standardAddr + } + return null +} +def NOW = new Date() +def getType(String type){ + if(type == null){ + return "普通住宅" + } + if(type.contains("商住两用")){ + return "商住" + }else if(type.contains("别墅")){ + return "别墅" + }else{ + return "普通住宅" + } +} + +ExcelUtils.modify("D:\\nextcloud\\共享文档\\项目文档\\重庆农商行\\新增小区4.xlsx",{List item-> + if(item["C"] != "明佳园"){ + return + } + BasicDBObject seaweed = new BasicDBObject() + seaweed.city = item["A"] + seaweed.region = item["B"] + seaweed.name = item["C"] + def query = new BasicDBObject("city", seaweed.city).append("region", seaweed.region) + .append('name', seaweed.name) + .append("visible", 0) + .append("cat", "district"); + BasicDBObject one = DbUtils.seaweed.findOne(query) + if(!one){ + query.removeField("name") + query.append("alias", seaweed.name) + one = DbUtils.seaweed.findOne(query) + } + if (one && one.status == 0) { + item["i"] = "已存在" + return; + } + seaweed.address = getStandardAddress( seaweed.city, item["D"]) + String source = item["H"] + Double lng = item["E"] + Double lat = item["F"] + if(lng){ + if(source.contains("高德")){ + seaweed.lng = (long)(lng * 10000000000l) + seaweed.lat = (long)(lat * 10000000000l) + JSONObject geo = BaiDuApiUtils.googleToBaidu(lat, lng) + seaweed.lng2 = (long)(geo.lng * 10000000000l) + seaweed.lat2 = (long)(geo.lat * 10000000000l) + }else if(source.contains("百度")){ + Map point = GeoUtils.geoconvBaiduToGoogle(lng, lat); + long google_lat = (long)(point.get("lat") * 10000000000l); + long google_lng = (long)(point.get("lng") * 10000000000l); + long baiDu_lat = (long)(lat * 10000000000l) + long baiDu_lng = (long)(lng * 10000000000l) + seaweed.lng = google_lng + seaweed.lat = google_lat + seaweed.lat2 = baiDu_lat + seaweed.lng2 = baiDu_lng + } + } + seaweed.cat = "district" + seaweed.status = 0 + seaweed.c_date = NOW + seaweed.ip = "MACHINE_SAVE" + seaweed.estate_type2 = getType(item["G"]) + Map markMap = new HashMap() + seaweed.mark_map = markMap + markMap.put("data_source", "机器入库") + DbUtils.seaweed.save(seaweed) + println(seaweed) +}) + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index e2790a624..ca851bcb7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20210401", "yyyyMMdd") -def endDate = DateUtil.parseDate("20210501", "yyyyMMdd") +def startDate = DateUtil.parseDate("20210501", "yyyyMMdd") +def endDate = DateUtil.parseDate("20210601", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -22,10 +22,10 @@ while (startDate.before(endDate)) { calendar.add(Calendar.DATE, 1) def nextDate = calendar.getTime() count++ - OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> + OPEN_URI_LOG.find(and(eq("user_id", "1007757"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> def result = JSONObject.fromObject(doc.res_map.result) if (!hashSet.contains(result)) { - def item = ["5c6535b99c213017e0768ee1", "qlkj", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] + def item = ["60a3202c0b42a00001aad385", "qlkjgx", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] data << item hashSet << result } else { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy index 08bf4c5b7..82cf47113 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20210401", "yyyyMMdd") -def endDate = DateUtil.parseDate("20210501", "yyyyMMdd") +def startDate = DateUtil.parseDate("20210601", "yyyyMMdd") +def endDate = DateUtil.parseDate("20210701", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -42,7 +42,7 @@ while (startDate.before(endDate)) { keys.addAll(params.keys()) def key = "${params.address}_${params.size}".toString() if (!hashSet.contains(key)) { - def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date.format("yyy"), doc.data_status, doc.code, doc.msg] + def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg] param_keys.each { item << params[it] } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test7.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test7.groovy index 168eaaf37..91e428cd4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test7.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test7.groovy @@ -1,6 +1,11 @@ +import com.fangjia.core.util.CNNMFilter +import com.fangjia.core.util.HttpRequest import com.fangjia.datastore.driver.DSCollection import com.fangjia.datastore.driver.DataStore +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject +import net.sf.json.JSONObject import org.apache.commons.io.FileUtils // @@ -35,7 +40,79 @@ import org.apache.commons.io.FileUtils // println("copy:" + file.getAbsolutePath() + "\tto:" + ("E:\\成交数据\\DBF汇总\\" + file.getName())) // } //} -DSCollection seawater = new DataStore("192.168.9.118", 27777).getCollection("fangjia.seawater") -seawater.find(new BasicDBObject("cat", "sell").append("city", "上海").append("region", "长宁").append("district_name", "新泾三村")).each { - println(it) -} \ No newline at end of file +//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\副本地址标准化验证结果分析.xlsx",{List item-> +// HttpRequest request = new HttpRequest() +// JSONObject json = request.getJsonContent2("http://localhost:8082/structure", ["address": item["A"]], "UTF-8") +// if(json){ +// item["D"] = json.result.province +// item["E"] = json.result.city +// item["F"] = json.result.district +// item["G"] = json.result.dev +// item["H"] = json.result.township +// item["I"] = json.result.village +// item["J"] = json.result.group +// item["K"] = json.result.street +// item["L"] = json.result.streetNumber +// item["M"] = json.result.name +// item["N"] = json.result.buildingNumber +// item["O"] = json.result.unitNumber +// item["P"] = json.result.roomNumber +// item["q"] = json.result.floor +// item["r"] = json.result.formatAddress +// } +// println(item) +//}) +//for(int i=1; i < 100; i++){ +// println("${i}号楼#@#building_number") +// println("${i}栋#@#building_number") +// println("${i}幢#@#building_number") +// println("${i}单元#@#unit_number") +// if(i< 20){ +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}号楼#@#building_number") +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}栋#@#building_number") +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}幢#@#building_number") +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}单元#@#unit_number") +// } +//} +//nums= "abcdefghijklmn" +//for(def num : nums.toCharArray()){ +// println("${num}栋#@#building_number") +// println("${num}幢#@#building_number") +// println("${num}单元#@#unit_number") +//} +//nums= "abcdefghijklmn" +//for(def num : nums.toCharArray()){ +// println("${num}座#@#building_number") +// for(int i = 1; i <11; i++){ +// println("${num}${i}座#@#building_number") +//// println("${num}${i}幢#@#building_number") +//// println("${num}${i}单元#@#unit_number") +// } +//} +//for(int i = 1; i <20; i++){ +// println("${i}区#@#part_number") +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}区#@#part_number") +// println("${i}期#@#stage_number") +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}期#@#stage_number") +//} +// +//for(def num : "东南北中西甲乙丙丁".toCharArray()){ +// println("${num}区#@#part_number") +//} +//for(int i = -5; i < 100; i++){ +// println("${i}楼#@#floor") +// if(i < 10 && i > 0){ +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}楼#@#floor") +// } +//} +//for(def num : "abcdefg东南西北".toCharArray()){ +// println("${num}户#@#room") +//} +//for(i in 1.. 40){ +// for(j in 1..9){ +// println("${i}0${j}室#@#room") +// } +//} +ElasticUtils.executeSqlQuery("select * from seawater where cat='sell' and district_name='新泾三村' and city='上海' and p_date > 20210101",{Map doc-> + println(doc.avg_price + "\t" + doc.root + "\t" + doc.p_date) +}) \ No newline at end of file From b1f117c18f07cf75ea3029387512bd80dd3f799e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 8 Sep 2021 16:15:13 +0800 Subject: [PATCH 02/80] update --- .../BigdataCommbuildtypeBatch.groovy | 2 +- .../ccbevaluation/BigdataDealBatch.groovy | 2 + .../ccbevaluation/BigdataDistrictBatch.groovy | 2 +- .../ccbevaluation/CcbCommonUtils.groovy | 4 +- .../ccbevaluation/EvaluateRateBatch.groovy | 2 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 131 ++++++++++++++++-- .../ccbevaluation/StagingDistrictBatch.groovy | 4 +- .../script/commData/CommonDatautils.groovy | 1 + 8 files changed, 132 insertions(+), 16 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index c1126334f..e6b9defe3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -39,7 +39,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ println city +" "+rows.size() } def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] - FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.txt",headers, "utf-8", false); + FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.dat",headers, "utf-8", false); println "txt 生成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 82766d351..0f3230368 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -49,6 +49,7 @@ class BigdataDealBatch extends BaseBatch{ def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def statMap = getDistrictStatByCity(city, weekend) @@ -445,6 +446,7 @@ class BigdataDealBatch extends BaseBatch{ def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) def q = new BasicDBObject() q.put("contractTime", new BasicDBObject('$gte', startDate)); + q.put("area", new BasicDBObject('$gte', 20)) def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000) def rets = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy index 815336ae3..08c478e6d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy @@ -41,7 +41,7 @@ class BigdataDistrictBatch extends BaseBatch{ rets << headers rets.addAll(allDatas) ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_district.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_district.txt",headers, "utf-8",false); + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_district.dat",headers, "utf-8",false); } def createDatas(rows){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index e80d7fd22..9a41d91ad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -79,7 +79,7 @@ class CcbCommonUtils { def q = new BasicDBObject() q.put("city", dist.city) q.put("region", dist.region) - + q.put("area", new BasicDBObject('$gte', 20)) def alias = [] if(dist.alias){ if(!dist.alias.contains(dist.name)){ @@ -174,7 +174,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name", "美辰香醒") +// q.put("name", "石泉六村") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy index d64c08dd6..7b40980e1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy @@ -30,7 +30,7 @@ class EvaluateRateBatch extends BaseBatch{ rets << headers rets.addAll(allDatas) ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\evaluate_rate.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.txt",headers, "utf-8", false); + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.dat",headers, "utf-8", false); } def createDatas(def datas){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 49c60f702..1842c3e5b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -4,13 +4,20 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors - +import java.util.concurrent.atomic.AtomicLong +import org.bson.types.ObjectId /** * 建行模型估值项目 * 人工指导价同步 @@ -20,6 +27,8 @@ import java.util.concurrent.Executors class GuiDingPriceBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol(); static main(args) { runBatch(new GuiDingPriceBatch(), args); @@ -34,6 +43,7 @@ class GuiDingPriceBatch extends BaseBatch{ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + //添加自己库中的人工指导价 def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6)) if(prices){ def distMap = getDistrictMap(city) @@ -42,18 +52,88 @@ class GuiDingPriceBatch extends BaseBatch{ println city + " " + rets.size() } + //添加搜房的参考价 +// def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); +// def datas = findFangDatas(districts) +// def fangRets = createDatas(cityCodeDatas, distCodeMap, datas) +// rows.addAll(fangRets) + + //添加保亭文昌的指导价 +// def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx") +// def cityDataMap = [:] +// for(def d : cityDatas){ +// cityDataMap.put(d['A'], d['E']) +// } +// def personDatas = findPersonDatas(districts, cityDataMap); +// def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas) +// rows.addAll(personRets) + } - def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.txt", headers, "utf-8", false); + def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.dat", headers, "utf-8", true); - def rets = [] - rets << headers - rets.addAll(rows) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据") +// def rets = [] +// rets << headers +// rets.addAll(rows) +// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据") } + def findFangDatas(def districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println " 小区量:"+districts.size() + def result = Collections.synchronizedList(new ArrayList()); + for(def i=0;i Date: Wed, 8 Sep 2021 18:51:47 +0800 Subject: [PATCH 03/80] update --- .../fangjia/script/ccbevaluation/BigdataDealBatch.groovy | 8 ++++---- .../script/ccbevaluation/BigdataForsaleBatch.groovy | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 0f3230368..8566cf085 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -421,8 +421,8 @@ class BigdataDealBatch extends BaseBatch{ startPrice = (int)(price - price*0.1) endPrice = (int)(price*0.5 + price) }else{ - startPrice = 1//(int)(price - price*0.5) - endPrice = 500000//(int)(price*0.5 + price) + startPrice = (int)(price - price*0.5) + endPrice = (int)(price*0.5 + price) } } def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice) @@ -461,8 +461,8 @@ class BigdataDealBatch extends BaseBatch{ startPrice = (int)(price - price*0.1) endPrice = (int)(price*0.5 + price) }else{ - startPrice = 1//(int)(price - price*0.5) - endPrice = 500000//(int)(price*0.5 + price) + startPrice = (int)(price - price*0.5) + endPrice = (int)(price*0.5 + price) } if(obj.price>=startPrice && obj.price<=endPrice){ rets << obj; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index b8af43278..d10aab75c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -75,6 +75,7 @@ class BigdataForsaleBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210831); + // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); } } From 17ee2f7f17f24d8e8286e53099da564385daffb9 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 9 Sep 2021 15:16:08 +0800 Subject: [PATCH 04/80] =?UTF-8?q?=E6=9D=91=E5=BA=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/GaoDeMapCrawl.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index 5c5cf081d..39cce1aaa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -50,14 +50,14 @@ class GaoDeMapCrawl extends BaseBatch{ // def categorys = ["141202", "141203"]//中学,小学 - def prefix = "district_" - def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关 +// def prefix = "district_" +// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关 // def prefix = "road_" // def categorys = ["190301"]//道路相关 -// def prefix = "village_" -// def categorys = ["190108","190109"]//村庄 + def prefix = "village_" + def categorys = ["190108","190109"]//村庄 // def prefix = "company_" // def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"] @@ -119,8 +119,8 @@ class GaoDeMapCrawl extends BaseBatch{ def run(){//,"西安","重庆" //def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] - def citys =["南昌"] -// def citys = CommonUtils.findAllCity() +// def citys =["南昌"] + def citys = CommonUtils.findAllCity() def totalCount = 0; for(def city : citys){ def stop = false From 924f9754b9904bfb192c2cdf8a178799a910d77d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 9 Sep 2021 17:20:47 +0800 Subject: [PATCH 05/80] =?UTF-8?q?=E4=BC=98=E5=85=88=E5=9F=8E=E5=B8=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index 39cce1aaa..05f491b09 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -119,8 +119,8 @@ class GaoDeMapCrawl extends BaseBatch{ def run(){//,"西安","重庆" //def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] -// def citys =["南昌"] - def citys = CommonUtils.findAllCity() + def citys =["上海","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"] +// def citys = CommonUtils.findAllCity() def totalCount = 0; for(def city : citys){ def stop = false From 5a1844b9267520d9caeb87158bf0f8bb6019bff9 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 10 Sep 2021 11:00:39 +0800 Subject: [PATCH 06/80] update --- .../thirdPriceCrawl/DataStandardBatch.groovy | 137 ++++++++++++++++-- 1 file changed, 125 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index c8ab1b6d8..cbe499054 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -4,6 +4,8 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.NameProcess +import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.linxsh.util.CommonUtil import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CosineSimilarAlgorithm @@ -86,9 +88,9 @@ class DataStandardBatch extends BaseBatch{ // ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl) cityMap.each{city,alias-> - println "开始加载【${city}】" - def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"]) - districtMap.putAll(detailMap) + println "开始加载【${city}】" + def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"]) + districtMap.putAll(detailMap) } while(flag){ @@ -126,15 +128,24 @@ class DataStandardBatch extends BaseBatch{ def data = standardCity(dbo); if(data){ //匹配区域 - data = standardRegion(data); - - if(data){ - //标准化小区名 - def data3 = standardDistrict(data, dbo.avg_price); + if(data.city=="三亚" && data.region=="陵水"){ + data.city = "陵水" + data.region = "" + def data3 = standardDistrictSimple(data, dbo.avg_price); if(data3){ successCount++; } + }else if(data){ + data = standardRegion(data); + if(data){ + //标准化小区名 + def data3 = standardDistrict(data, dbo.avg_price); + if(data3){ + successCount++; + } + } } + } // }else{ // saveLog(dbo, "无价格"); @@ -253,6 +264,104 @@ class DataStandardBatch extends BaseBatch{ return data; } + def standardDistrict2(def data, def avg_price){ + def newName = NameProcess.nameStandard(data.city, data.region, data.name) + def district = null + if(newName){ + district = SeaweedDataMatchProcess.doMatch(data.city, data.region, data.name, data.address); + if(district){ + + } + } + + if(district && avg_price){ + removeErrorLog(data);//删除错误日志表中区域不匹配的,正式运行时删除 + saveSuccessLog(data, district, type); + updateStatus(data,statusMap.get("小区匹配成功")) + println district.city+" "+district.name+" success" + }else{ + if(district){ +// data.type = type + saveLog(data, "无价格"); + updateStatus(data,statusMap.get("无价格")) + println data.city+" "+data.name+" fail(无价格)" + }else{ + println "[匹配失败]" + saveLog(data, "小区匹配失败"); + updateStatus(data,statusMap.get("小区匹配失败")) + println data.city+" "+data.name+" fail" + } + } + return district; + + } + + def standardDistrictSimple(def data, def avg_price){ + def newName1 = null + def newName2 = null + if(data.name.contains("别名")){ + newName1 = data.name.toString().substring(0, data.name.toString().indexOf("别名")-1) + newName2 = data.name.toString().substring(data.name.toString().indexOf("别名")+3) + }else{ + newName1 = NameProcess.nameStandard(data.city, data.region, data.name) + } + def district = null + def type = null + if(newName1){ + district = SeaweedDataMatchProcess.doMatch(data.city, data.region, newName1, data.address); + if(district && (district.name == newName1 || (district.alias && district.alias.contains(newName1)))){ + district._id = data._id + type = 1 + }else{ + if(newName2){ + district = SeaweedDataMatchProcess.doMatch(data.city, data.region, newName2, data.address); + if(district && (district.name == newName2 || (district.alias && district.alias.contains(newName2)))){ + district._id = data._id + type = 1 + }else{ + def score = CosineSimilarAlgorithm.getSimilarity(district.name, newName2) + if(score>0.8){ + district._id = data._id + type = 11 + }else{ + district = null + } + } + }else{ + if(district){ + def score = CosineSimilarAlgorithm.getSimilarity(district.name, newName1) + if(score>0.8){ + district._id = data._id + type = 11 + }else{ + district = null + } + } + } + } + } + + if(district && avg_price){ + removeErrorLog(data);//删除错误日志表中区域不匹配的,正式运行时删除 + saveSuccessLog(data, district, type); + updateStatus(data,statusMap.get("小区匹配成功")) + println district.city+" "+district.name+" success" + }else{ + if(district){ + data.type = type + saveLog(data, "无价格"); + updateStatus(data,statusMap.get("无价格")) + println data.city+" "+data.name+" fail(无价格)" + }else{ + println "[匹配失败]" + saveLog(data, "小区匹配失败"); + updateStatus(data,statusMap.get("小区匹配失败")) + println data.city+" "+data.name+" fail" + } + } + return district; + } + def standardDistrict(def data, def avg_price){ //名称匹配,名称别名匹配,地址别名匹配 //def district = getMatchName(data.city, data.region, data.name, data.address,"district") @@ -368,7 +477,7 @@ class DataStandardBatch extends BaseBatch{ } if(district){ if(data.region != district.region){ - //type = 12;//区域不同 + type = 12;//区域不同 district = null; }else{ def score1 = 0; @@ -495,10 +604,14 @@ class DataStandardBatch extends BaseBatch{ }*/ - def getMatchNameFromMap(def city, def region, def disName, def address){ + def getMatchNameFromMap(def city, def region, def disName, String address){ //查询房价网是否存在该小区 def dbo = findSameNameFromMap(city, region, disName); - dbo = addressCheck(dbo, region, address); + def addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') + if(addrFlag){ + dbo = addressCheck(dbo, region, address); + } + if(!dbo){ //别名 def distrs = findAliasNameFromMap(city, region, disName); @@ -506,7 +619,7 @@ class DataStandardBatch extends BaseBatch{ dbo = distrs[0] }else if(distrs && distrs.size()>1 && address){ for(def d : distrs){ - if(d.address.contains(address)||d.address==address || address.contains(d.address)){ + if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){ dbo = d; break; } From 4b3dee53081d2257c7535ff0664964d9e203f2f0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 10 Sep 2021 11:01:36 +0800 Subject: [PATCH 07/80] update --- .../ccbevaluation/CcbCommonUtils.groovy | 13 ++ .../DistrictGuidePriceBatch.groovy | 85 ++++++++++++ .../ccbevaluation/GuiDingPriceBatch.groovy | 26 ++-- .../script/ccbevaluation/LoadGuide.groovy | 98 ++++++++++++++ .../fangjia/script/ccbevaluation/Test.groovy | 104 +++++++-------- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 59 +++++++-- .../fangjia/script/gj/YouDaDataParser.groovy | 16 +-- .../fangjia/script/gj/data/FieldStat.groovy | 37 +++++- .../guangfa/GuangfaBlankSelectDistrict.groovy | 122 +++++++++++++++++- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 20 +-- .../TestCrawlTotalCount.groovy | 4 +- 11 files changed, 485 insertions(+), 99 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 9a41d91ad..999d4b347 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -27,6 +27,8 @@ class CcbCommonUtils { public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017) + public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] public static DSCollection dataStoreConnection = null; @@ -75,6 +77,17 @@ class CcbCommonUtils { return rets } + public static def findGuidePrice(city){ + def q = new BasicDBObject() + q.put("city", city) + def dbcursor = DISTRICT_GUIDE_PRICE_COLL.find(q) + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets + } + public static def findTradesDetail(dist, startDate, endDate, startPrice, endPrice){ def q = new BasicDBObject() q.put("city", dist.city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy new file mode 100644 index 000000000..3f73a32d6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy @@ -0,0 +1,85 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils + + +@BatchOwner(Watcher.GaoJuan) +class DistrictGuidePriceBatch extends BaseBatch{ + + + static main(args) { + runBatch(new DistrictGuidePriceBatch(), args); + } + + @Override + public void process(Object args) { + def citys = ["深圳"]//CcbCommonUtils.citys; + def rows = [] + + for (def city : citys) { + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + //添加自己库中的人工指导价 + def prices = CcbCommonUtils.findGuidePrice(city) + if(prices){ + def distMap = getDistrictMap(city) + def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) + rows.addAll(rets) + println city + " " + rets.size() + } + } + +// def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"] +// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\district_guied_price.dat", headers, "utf-8", false); + + } + + + def getDistrictMap(def city){ + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); + def distMap = [:] + for(def d : districts){ + distMap.put(d.region+"##"+d.name, d) + } + return distMap; + } + + def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){ + def rets = [] + if(prices){ + for(def ps : prices){ + def dist = distMap.get(ps.match_region+"##"+ps.match_name) + if(dist){ + def ret = [] + def areaData = distCodeMap.get(dist.region) + if(areaData){ + ret << ps._id.toString() + ret << dist._id.toString() + ret << dist.name + ret << cityCodeDatas.get(dist.city) + ret << dist.city + ret << areaData.district_code + ret << areaData.district + ret << ps.avg_price + ret << ps.source_type + ret << "" + ret << DateUtils.formatDateByPattern(ps.c_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(ps.c_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } + }else{ + println ps.match_region+"##"+ps.match_name + } + } + } + return rets; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 1842c3e5b..68eb41ed3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -53,25 +53,25 @@ class GuiDingPriceBatch extends BaseBatch{ } //添加搜房的参考价 -// def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); // def datas = findFangDatas(districts) // def fangRets = createDatas(cityCodeDatas, distCodeMap, datas) // rows.addAll(fangRets) //添加保亭文昌的指导价 -// def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx") -// def cityDataMap = [:] -// for(def d : cityDatas){ -// cityDataMap.put(d['A'], d['E']) -// } -// def personDatas = findPersonDatas(districts, cityDataMap); -// def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas) -// rows.addAll(personRets) + def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx") + def cityDataMap = [:] + for(def d : cityDatas){ + cityDataMap.put(d['A'], d['E']) + } + def personDatas = findPersonDatas(districts, cityDataMap); + def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas) + rows.addAll(personRets) } - def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.dat", headers, "utf-8", true); + def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.dat", headers, "utf-8", false); // def rets = [] // rets << headers @@ -146,8 +146,8 @@ class GuiDingPriceBatch extends BaseBatch{ ret << ps._id.toString() ret << dist._id.toString() ret << dist.name - ret << cityCodeDatas.city_code - ret << cityCodeDatas.fangjia_city + ret << cityCodeDatas.get(dist.city) + ret << dist.city ret << areaData.district_code ret << areaData.district ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy new file mode 100644 index 000000000..c6fa7f82c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy @@ -0,0 +1,98 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class LoadGuide { + public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017) + + static main(args) { + def task = new LoadGuide() + task.run() + } + + def run(){ +// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞二手房政府指导价.xls") +// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx") +// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx") +// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx") +// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安成交参考价格.xlsx") +// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx") + savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx") + + } + + def savePrice(filePath){ + def datas = ExcelUtils.read(filePath,0, 1) + def rets = [] + for(def d : datas){ + if(d['E']){ + def o = new BasicDBObject() + o.city = d['E'] + o.region = d['A'] + o.name = d['C'] + o.avg_price = d['D'].toString().toDouble().toInteger() + o.match_city = d['E'] + o.match_region = d['F'] + o.match_name = d['G'] + o.source_type = 1 //1:成交参考价 2:参考价 + o.c_date = new Date() + rets << o + } + } + if(rets){ + println rets.size() + DISTRICT_GUIDE_PRICE_COLL.insert(rets) + } + } + + def saveDongGuangPrice(def filePath){ + def datas = ExcelUtils.read(filePath,0, 0) + def rets = [] + for(def d : datas){ + if(d['E']){ + def o = new BasicDBObject() + o.city = d['E'] + o.region = d['A'] + o.name = d['B'] + o.avg_price = d['D'].toString().toDouble().toInteger() + o.match_city = d['E'] + o.match_region = d['F'] + o.match_name = d['G'] + o.house_count = d['C'].toString().toDouble().toInteger() + o.source_type = 1 //1:成交参考价 2:参考价 + o.c_date = new Date() + rets << o + } + } + if(rets){ + DISTRICT_GUIDE_PRICE_COLL.insert(rets) + } + } + + def saveChengduoPrice(def filePath){ + def datas = ExcelUtils.read(filePath,2, 0) + def rets = [] + for(def d : datas){ + if(d['E']){ + def o = new BasicDBObject() + o.city = d['E'] + o.region = d['B'] + o.name = d['C'] + o.avg_price = d['D'].toString().toDouble().toInteger() + o.match_city = d['E'] + o.match_region = d['F'] + o.match_name = d['G'] + o.source_type = 1 //1:成交参考价 2:参考价 + o.c_date = new Date() + rets << o + } + } + if(rets){ + DISTRICT_GUIDE_PRICE_COLL.insert(rets) + } + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index d7b05ed1b..18dc9f380 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -59,7 +59,7 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw -def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807 - 副本.xlsx" +def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0903.xlsx" //def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) //def dealMap = [:] @@ -83,58 +83,58 @@ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() // if (city) { // def cityid = cityCodeDatas.get(city) // def total = findCount(cityid, sqlInstance) -// d['K'] = total +// d['M'] = total // } //},0) -//小区字段覆盖率 -ExcelUtils.multiThreadingModify(fileName,{d-> - def city = d['A'] - if(city){ - def cityid = cityCodeDatas.get(city) - def total = findCount(cityid, sqlInstance) - d['AA'] = total - - def a = findEstateType(cityid, sqlInstance) - d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) - - def b = findFiledCount(cityid, 'buildyear', sqlInstance) - d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) - - def c = findFiledCount(cityid, 'address', sqlInstance) - d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) - - def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) - d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) - - def e = findFiledCount(cityid, 'landarea', sqlInstance) - d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) - - def f = findFiledCount(cityid, 'buildarea', sqlInstance) - d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) - - def g = findFiledCount(cityid, 'plotratio', sqlInstance) - d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) - - def h = findFiledCount(cityid, 'greeningrate', sqlInstance) - d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) - - def i = findFiledCount(cityid, 'latitude', sqlInstance) - d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) - - def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) - d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) - println d - } -},1) - -//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807-1.xlsx" -//def dealMap = findDealCount(sqlInstance) +////小区字段覆盖率 //ExcelUtils.multiThreadingModify(fileName,{d-> -// if(d['C']){ -// d['Q'] = dealMap.get(d['C']) +// def city = d['A'] +// if(city){ +// def cityid = cityCodeDatas.get(city) +// def total = findCount(cityid, sqlInstance) +// d['AA'] = total +// +// def a = findEstateType(cityid, sqlInstance) +// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) +// +// def b = findFiledCount(cityid, 'buildyear', sqlInstance) +// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) +// +// def c = findFiledCount(cityid, 'address', sqlInstance) +// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) +// +// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) +// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) +// +// def e = findFiledCount(cityid, 'landarea', sqlInstance) +// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) +// +// def f = findFiledCount(cityid, 'buildarea', sqlInstance) +// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) +// +// def g = findFiledCount(cityid, 'plotratio', sqlInstance) +// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) +// +// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) +// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) +// +// def i = findFiledCount(cityid, 'latitude', sqlInstance) +// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) +// +// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) +// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) +// println d // } -//},2) +//},1) + +//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx" +def dealMap = findDealCount(sqlInstance) +ExcelUtils.multiThreadingModify(fileName,{d-> + if(d['C']){ + d['K'] = dealMap.get(d['C']) + } +},2) //def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" @@ -170,7 +170,7 @@ ExcelUtils.multiThreadingModify(fileName,{d-> def findDealCount(sqlInstance){ - def sql = "select cityname,count(*) as count from bigdata_deal where dealdate>=to_date('2021-06-01','yyyy-mm-dd') and dealdate @@ -194,7 +194,7 @@ def findEstateType(cityid, sqlInstance){ } def findCount(cityid, sqlInstance){ - def sql = "select count(*) as count from bigdata_community where cityid='${cityid}'"; + def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}'"; def count = 0 sqlInstance.eachRow(sql,{r-> @@ -204,7 +204,7 @@ def findCount(cityid, sqlInstance){ } def findFiledCount(cityid, field, sqlInstance){ - def sql = "select count(*) as count from bigdata_community where cityid='${cityid}' and "+field+" is not null"; + def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and "+field+" is not null"; def count = 0 sqlInstance.eachRow(sql,{r-> count = r.count diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index f14ace3a9..8fadf19e0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -11,6 +11,8 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import java.util.concurrent.ConcurrentHashMap + /** * 建行模型估值项目 * 第三方均价数据同步 @@ -18,11 +20,13 @@ import com.mongodb.DBCollection @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def sDate= 202107//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202105 + def endDate = 202107 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") - + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") static main(args) { runBatch(new ThirdPartyPriceBatch(), args); @@ -33,13 +37,15 @@ class ThirdPartyPriceBatch extends BaseBatch { def citys = CcbCommonUtils.citys; def rows = [] def index = 0 + def weekend = DateUtils.getFirstDayOfWeek(new Date()) for(def city : citys){ index++ + def statMap = getDistrictStatByCity(city, weekend) def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) - def thirdPriceMap = findThirdPartyPrice(city) + def thirdPriceMap = findThirdPartyPrice(city, statMap) def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap) rows.addAll(rets) println index+" "+city+" "+rets.size() @@ -47,13 +53,13 @@ class ThirdPartyPriceBatch extends BaseBatch { - def fileName = "/home/goojia/ccb/third_party_price.txt" + def fileName = "/home/goojia/ccb/third_party_price.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.txt" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.dat" } - def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); + def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); } @@ -166,10 +172,10 @@ class ThirdPartyPriceBatch extends BaseBatch { } - def findThirdPartyPrice(def city){ + def findThirdPartyPrice(def city, def statMap){ def q = new BasicDBObject() q.put("city", city) - q.put("s_date", new BasicDBObject('$gte', sDate)) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def map = [:] for(def db : dbcursor){ @@ -242,13 +248,46 @@ class ThirdPartyPriceBatch extends BaseBatch { def resultMap = [:] map.each {key, value-> if(value && value.size()>1){ - resultMap.put(key, value) + if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){ + def price = statMap.get(key) + if(price){ + def priceMap = [:] + def startPrice = (int)(price - price*0.3) + def endPrice = (int)(price*0.3 + price) + value.each{pk,pv-> + if(pk.contains("price")){ + if(pv>=startPrice && pv<=endPrice){ + priceMap.put(pk,pv) + } + }else{ + priceMap.put(pk,pv) + } + } + if(priceMap.size()>1){ + resultMap.put(key, priceMap) + } + } + }else{ + resultMap.put(key, value) + } + } } return resultMap } + def getDistrictStatByCity(city, weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def dbcursor = DIST_SELL_STAT.find(q) + def map = new ConcurrentHashMap() + for(def d : dbcursor){ + map.put(d.region+"##"+d.district_name, d.avg_price) + } + return map + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index c2c545471..8a1af7d25 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -52,7 +52,7 @@ class YouDaDataParser { static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser();//友达二手房存入mongo + task.esfParser();//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 @@ -299,14 +299,14 @@ class YouDaDataParser { for(def i=0;i + def thirdPriceMap = findNearThirdPartyPrice(city) + def retMaps = findCreditHightDistrict3(city, "2021-08-22", thirdPriceMap) + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias","estate_company", "developer","cubage_percent","household_count", "finish_building_date"]) + //基本情况合格的小区 + def distMap = [:] + for(def d : districts){ + // && d.household_count && d.household_count>=50 + def distKeys = retMaps.get(d.region) + if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){ + if(d.finish_building_date){ + def year = CcbCodeUtils.parseBuildYear(d.finish_building_date) + if(year){ + def mod = 2021-year + if(mod <=15){ + d.house_year = mod + }else{ + d.house_year = null + } + }else{ + d.house_year = null + } + + }else{ + d.house_year = null + } + distMap.put(d.city+"##"+d.region+"##"+d.name, d) + } +// if(d.finish_building_date){ +// def year = CcbCodeUtils.parseBuildYear(d.finish_building_date) +// if(year){ +// def mod = 2021-year +// if(mod <=15){ +// def distKeys = retMaps.get(d.region) +// if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){ +// d.house_year = mod +// distMap.put(d.city+"##"+d.region+"##"+d.name, d) +// } +// } +// } +// } + + } + println city +" 基本情况合格的小区:"+distMap.size() + + for(def r : regions){ + def names = retMaps.get(r) + if(names){ + def results = getRoomTypeAndTradeInfo(names, distMap, priceMap) + rets.addAll(results) + } + } + println "结束" + } + + println "开始写入Excel....."+rets.size() + ExcelUtils.write("d:\\数据\\广发\\楼盘(西安东莞杭州).xlsx", rets, "数据") + } + def getRoomTypeAndTradeInfo(def names, def distMap, def priceMap){ def rets = Collections.synchronizedList(new ArrayList()); @@ -332,4 +402,52 @@ class GuangfaBlankSelectDistrict { } return thirdPriceMap } + + + def findNearThirdPartyPrice(def city){ + def q = new BasicDBObject() + q.put("city", city) + q.put("s_date", new BasicDBObject('$gte', 202105)) + q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true)) + def dbcursor = AVERAGE_STATISTICS.find(q) + def thirdPriceMap = [:] + for(def d : dbcursor){ + def key = d.city+"#"+d.region+"#"+d.name + if(!thirdPriceMap.containsKey(key)){ + thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price) + } + } + return thirdPriceMap + } + + def findCreditHightDistrict3(def city, def regions, def weekend, def thirdPriceMap){ + def q = new BasicDBObject() + q.put("city", city) + q.put("value_type", 2) + q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd")) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1)) + def retMaps = [:] + for(def d : dbcursor){ + def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name) + if(thirdPrice){ + def precent = (thirdPrice-d.avg_price)*100/d.avg_price + if(Math.abs(precent)>=0 && Math.abs(precent)<=10){ + def rets = retMaps.get(d.region) + if(!rets){ + rets = [] + retMaps.put(d.region, rets) + } + rets << d.city+"##"+d.region+"##"+d.district_name + } + }else{ + def rets = retMaps.get(d.region) + if(!rets){ + rets = [] + retMaps.put(d.region, rets) + } + rets << d.city+"##"+d.region+"##"+d.district_name + } + } + return retMaps; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 2b8980257..d0ebd044c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -15,9 +15,9 @@ class SynchDistrictPrice { def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); -// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", -// "anjuke.com":"安居客","cityhouse.cn":"城市房产"] - def sourceMap = ["lianjia.com":"链家"] + def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", + "anjuke.com":"安居客","cityhouse.cn":"城市房产"] +// def sourceMap = ["cityhouse.cn":"城市房产"] static main(args) { def task = new SynchDistrictPrice() task.run() @@ -28,23 +28,22 @@ class SynchDistrictPrice { def citys = CommonUtils.findAllCity() sourceMap.each { k, v -> - for(def city : citys){ - if(!["上海","北京","海口"].contains(city)){ - def rets = findCrawlDistrictDetails(city, k, v) - println city +" "+rets.size() - synchData(city, rets, v) - } + def rets = findCrawlDistrictDetails(city, k, v) + println city +" "+rets.size() + synchData(city, rets, v) } } } def findCrawlDistrictDetails(city, root, source){ + + def curMonthFirstDay = DateUtils.getFirstDayOfMonth(new Date()) def q = new BasicDBObject() q.put("city", city) q.put("root", root) - q.put('m_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-07-01"))); + q.put('m_date', new BasicDBObject('$gte', curMonthFirstDay)); def keys = new BasicDBObject() keys.put("city", 1) keys.put("region", 1) @@ -53,6 +52,7 @@ class SynchDistrictPrice { keys.put("address", 1) keys.put("avg_price", 1) keys.put("root", 1) + keys.put("c_date", 1) keys.put("url", 1) def rets = [] def dbcursor = districtDetailColl.find(q, keys) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index f6887e506..10283440a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"] for(def source :sources){ - def count1 = findTotalCount(source, 202106); - def count2 = findTotalCount(source, 202107); + def count1 = findTotalCount(source, 202107); + def count2 = findTotalCount(source, 202108); println source +" "+count1 +" "+count2 } } From c217142d9ca01b35f6800407cb4dba4999f74310 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 13 Sep 2021 15:21:04 +0800 Subject: [PATCH 08/80] update --- .../ccbevaluation/UpdateHouseRegion.groovy | 150 ++++++++++++++++++ .../script/commData/CommonDatautils.groovy | 23 ++- 2 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy new file mode 100644 index 000000000..082d07eb9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy @@ -0,0 +1,150 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.commData.CommonDatautils +import net.sf.json.JSONObject +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class UpdateHouseRegion { + + + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + def task = new UpdateHouseRegion() + task.run() + } + def run(){ + def city = "杭州" +// findCount(city, 20210101); + def rets = findHouseSource(city, 20200701, 20201231) + batchUpdate(city, rets); + } + + def findCount(def city, def startDate, def endDate){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT count(*) as count FROM \"seawater-2020\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and cat = 'sell'") + sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <="+endDate) + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + + while (rs.next()) { + def count = rs.getInt("count") + println count + } + } + + def batchUpdate(def city, def houses){ + def updateCount = new AtomicLong() + def curCount = new AtomicLong() + def totalCount = houses.size(); + final CountDownLatch latch = new CountDownLatch(totalCount) + println city +" 房源总量:"+totalCount + for(def i=0;i="+startDate).append(" and p_date <"+endDate); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + + def houses = [] + while (rs.next()){ + houses << ["region":rs.getString("region"), "name":rs.getString("district_name"),"url":rs.getString("url")] + } + return houses; + } + + + + def updateHouseRegion2(def url, def region){ + def header = [:] + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString() + String seawaterUrl = "http://192.168.0.206:9200/seawater-2020/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false" + def body = """ + { + "script": { + "source": "ctx._source.region='${region}'", + "lang": "painless" + }, + "query": { + "term": { + "url": "${url}" + } + } + } + """ + HttpRequest request = new HttpRequest() + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") +// def response = JSONObject.fromObject(res) +// println(response) + } + + + def updateHouseRegion(def url, def region){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("update seawater-2021") + sqlBuffer.append(" set region = '"+region+"'") + sqlBuffer.append(" where url='"+url+"'") + def result = dataSource.connection.createStatement().executeUpdate(sqlBuffer.toString()) + println region +" "+result + } + + def getAllDistricts(def city){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias"]) + def map = [:] + for(def d : dists){ + + map.put() + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 2a1d89a8a..42d3641f3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -536,8 +536,8 @@ class CommonDatautils { if(name){ alias << name; } - - + + DBObject q = new BasicDBObject(); q.put("city", city); q.put("region", region); @@ -890,6 +890,25 @@ class CommonDatautils { } return null; } + + public static def findDistrictsByName(String city, String name, String cat){ + + DBObject q = new BasicDBObject(); + q.put("city", city); + q.put("cat", cat); + q.put("name", name); + q.put("visible", 0); + + def rets = [] + DSCursor cursor = getDataStoreConnection().find(q); + if(cursor){ + for(def obj : cursor){ + rets << obj; + } + } + return rets; + } + public static def getSimpleDistrictMap(def cat, def city){ DBObject q = new BasicDBObject(); q.put("visible", 0); From db93de0d6ad067d054d94aa293603143ef994a57 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 13 Sep 2021 15:38:04 +0800 Subject: [PATCH 09/80] =?UTF-8?q?=E5=89=8D=E5=8D=8A=E5=B9=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy index 082d07eb9..0478870bc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy @@ -33,7 +33,7 @@ class UpdateHouseRegion { def run(){ def city = "杭州" // findCount(city, 20210101); - def rets = findHouseSource(city, 20200701, 20201231) + def rets = findHouseSource(city, 20200101, 20200731) batchUpdate(city, rets); } From b58b339665eff93901963fe67ff1a4697dd09934 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 13 Sep 2021 16:01:42 +0800 Subject: [PATCH 10/80] =?UTF-8?q?19=E5=B9=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/UpdateHouseRegion.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy index 0478870bc..dbb4d19d3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy @@ -33,7 +33,7 @@ class UpdateHouseRegion { def run(){ def city = "杭州" // findCount(city, 20210101); - def rets = findHouseSource(city, 20200101, 20200731) + def rets = findHouseSource(city, 20190701, 20191231) batchUpdate(city, rets); } @@ -91,7 +91,7 @@ class UpdateHouseRegion { def findHouseSource(def city, def startDate, def endDate){ final StringBuffer sqlBuffer = new StringBuffer() - sqlBuffer.append("SELECT * FROM \"seawater-2020\" WHERE 1=1") + sqlBuffer.append("SELECT * FROM \"seawater-2019\" WHERE 1=1") sqlBuffer.append(" and city = '"+city+"'") sqlBuffer.append(" and cat = 'sell'") sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate); @@ -109,7 +109,7 @@ class UpdateHouseRegion { def updateHouseRegion2(def url, def region){ def header = [:] header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString() - String seawaterUrl = "http://192.168.0.206:9200/seawater-2020/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false" + String seawaterUrl = "http://192.168.0.206:9200/seawater-2019/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false" def body = """ { "script": { From 7543adbc37939677e17de2acf35707e46bb6ccfc Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 13 Sep 2021 16:17:26 +0800 Subject: [PATCH 11/80] 19 --- .../com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy index dbb4d19d3..15d2fe3d4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy @@ -32,7 +32,6 @@ class UpdateHouseRegion { } def run(){ def city = "杭州" -// findCount(city, 20210101); def rets = findHouseSource(city, 20190701, 20191231) batchUpdate(city, rets); } From 137092f15f99dd4605528e36a7e9a692c4d9a510 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 13 Sep 2021 16:33:48 +0800 Subject: [PATCH 12/80] =?UTF-8?q?=E6=9D=AD=E5=B7=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index d10aab75c..a254ae163 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -66,7 +66,7 @@ class BigdataForsaleBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = ["杭州"]//CcbCommonUtils.citys; def index = 0 def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ From be1ee855e97977331520e8531dba9dc45cd4a938 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 13 Sep 2021 18:30:17 +0800 Subject: [PATCH 13/80] =?UTF-8?q?5=E4=B8=AA=E5=9F=8E=E5=B8=82=E6=88=BF?= =?UTF-8?q?=E6=BA=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index a254ae163..797641971 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -66,7 +66,7 @@ class BigdataForsaleBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = ["杭州"]//CcbCommonUtils.citys; + def citys = ["东莞","中山","文昌","保亭","陵水"]//CcbCommonUtils.citys; def index = 0 def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ From 3f4d9a5b3f2f9d3a72df0262ace7bfe6899b7ac8 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 13 Sep 2021 19:35:40 +0800 Subject: [PATCH 14/80] update --- .../ccbevaluation/BigdataDealBatch.groovy | 14 ++- .../ccbevaluation/BigdataForsaleBatch.groovy | 2 +- .../ccbevaluation/CcbCommonUtils.groovy | 2 +- .../ccbevaluation/UpdateHouseRegion.groovy | 105 +++++++++++++++++- 4 files changed, 114 insertions(+), 9 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 8566cf085..f928400f6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -144,16 +144,17 @@ class BigdataDealBatch extends BaseBatch{ def street = "" def streetNumber = "" if(trade.address){ - if(trade.address.toString().endsWith("弄")){ + if(trade.address.toString().contains("弄")){ String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) if(number){ street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) streetNumber = number } - }else{ + } + if(!street){ def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name) if(jsonObj && jsonObj.searchAddress){ - if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.endsWith("弄")){ + if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){ String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) if(number){ street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) @@ -170,7 +171,7 @@ class BigdataDealBatch extends BaseBatch{ if(!street){ if(obj.address){ - if(obj.address.toString().endsWith("弄")){ + if(obj.address.toString().contains("弄")){ String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄")) if(number){ street = obj.address.toString().substring(0, obj.address.toString().indexOf(number)) @@ -282,7 +283,7 @@ class BigdataDealBatch extends BaseBatch{ } if(trade.address){ - if(trade.address.toString().endsWith("弄")){ + if(trade.address.toString().contains("弄")){ String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) if(number){ ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number)) @@ -420,6 +421,9 @@ class BigdataDealBatch extends BaseBatch{ if(dist.city=="上海"){ startPrice = (int)(price - price*0.1) endPrice = (int)(price*0.5 + price) + }else if(dist.city=="杭州"){ + startPrice = (int)(price - price*0.2) + endPrice = (int)(price*0.5 + price) }else{ startPrice = (int)(price - price*0.5) endPrice = (int)(price*0.5 + price) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 797641971..d10aab75c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -66,7 +66,7 @@ class BigdataForsaleBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = ["东莞","中山","文昌","保亭","陵水"]//CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys; def index = 0 def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 999d4b347..0b413551a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -187,7 +187,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name", "石泉六村") +// q.put("name", "莱兴公寓") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy index 15d2fe3d4..c71f088de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy @@ -2,6 +2,11 @@ package com.fangjia.script.ccbevaluation import com.fangjia.core.util.HttpRequest import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import net.sf.json.JSONObject import org.elasticsearch.xpack.sql.jdbc.EsDataSource @@ -26,16 +31,112 @@ class UpdateHouseRegion { } public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + + def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + static main(args) { def task = new UpdateHouseRegion() - task.run() +// task.updateSeawaterRegion() +// task.updateDealRegion() + task.updateThirdPartyRegion() } - def run(){ + + def updateThirdPartyRegion(){ + def city = "杭州" + def rets = findThirdPartyPrice(city) + batchThirdPartyUpdate(city, rets); + + } + def findThirdPartyPrice(city){ + def q = new BasicDBObject() + q.put("city",city) + q.put("s_date",new BasicDBObject('$gte', 202101)) + def dbcursor = thirdPartyColl.find(q) + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets; + } + + def updateDealRegion(){ + def city = "杭州" + def q = new BasicDBObject() + q.put("city", city) + q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-01-01"))); + def cursor = DEAL_PRICE_COLL.find(q) + def index = 0 + def updateIndex = 0 + for(def obj : cursor){ + index++ + def region = obj.region + def name = obj.district_name + def rets = CommonDatautils.findDistrictsByName(city, name, "district") + if (!rets) { + rets = CommonDatautils.findDistrictByAlias(city, null, name, "district") + } + + if (rets.size() == 1) { + def dist = rets[0] + if (region != dist.region) { + updateIndex++ + def udata = new BasicDBObject() + udata.put("region", dist.region) + DEAL_PRICE_COLL.update(new BasicDBObject('_id', obj.get("_id")), new BasicDBObject('$set', udata)) + println updateIndex+" "+index+" "+region+" "+dist.region + } + } + } + + + } + def updateSeawaterRegion(){ def city = "杭州" def rets = findHouseSource(city, 20190701, 20191231) batchUpdate(city, rets); } + + def batchThirdPartyUpdate(def city, def thirdDatas){ + def updateCount = new AtomicLong() + def curCount = new AtomicLong() + def totalCount = thirdDatas.size(); + final CountDownLatch latch = new CountDownLatch(totalCount) + println city +" 第三方总量:"+totalCount + for(def i=0;i Date: Wed, 15 Sep 2021 09:48:40 +0800 Subject: [PATCH 15/80] =?UTF-8?q?=E5=85=A8=E5=9B=BD=E6=9D=91=E5=BA=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/gj/GaoDeMapCrawl.groovy | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index 05f491b09..ddc92508f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -120,17 +120,19 @@ class GaoDeMapCrawl extends BaseBatch{ def run(){//,"西安","重庆" //def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] def citys =["上海","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"] -// def citys = CommonUtils.findAllCity() + def allcitys = CommonUtils.findAllCity() def totalCount = 0; - for(def city : citys){ - def stop = false - while(stop == false){ - def cityLations = findAllLation(city); - if(!cityLations){ - stop = true; - }else{ - println "开始处理["+city +"]:"+cityLations.size() - gaoDeCitySearch(cityLations); + for(def city : allcitys){ + if(!citys.contains(city)){ + def stop = false + while(stop == false){ + def cityLations = findAllLation(city); + if(!cityLations){ + stop = true; + }else{ + println "开始处理["+city +"]:"+cityLations.size() + gaoDeCitySearch(cityLations); + } } } } From 93de1d7aa0339ed1aeba5b84ce3055a483ebd53d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 16 Sep 2021 16:41:04 +0800 Subject: [PATCH 16/80] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=B8=A4=E6=B5=B7?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E6=89=80=E8=BE=96=E4=BA=BA=E5=91=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-bkoff/src/main/webapp/fjt/add.ftl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-bkoff/src/main/webapp/fjt/add.ftl b/projects/fangjia-bkoff/src/main/webapp/fjt/add.ftl index f08cc8a63..4a70b7978 100644 --- a/projects/fangjia-bkoff/src/main/webapp/fjt/add.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/fjt/add.ftl @@ -176,7 +176,7 @@ table{text-align: left;} } }) - if(role==12){ + if(role==12||role==17){ if(userStr.length>0){ console.log(userStr); $("input[name=roleUsers]").val(userStr); @@ -306,7 +306,7 @@ $("input[name=email]").blur(function(){ $("#role").change(function() { var role = $(this).val(); //鍙鍐滆鐢熸晥 - if (role == 12) { + if (role == 12 || role == 17) { $("#show_chk_all").removeAttr("style"); jQuery.ajax({ type: 'POST', @@ -340,12 +340,12 @@ $("input[name=email]").blur(function(){ } return false; } - <#if user?? && user.role == 12> + <#if user?? && (user.role == 12 || user.role == 17)> $("#show_chk_all").removeAttr("style"); jQuery.ajax({ type: 'POST', url: '../fjt/getUsersByRole', - data: {role:12}, + data: {role:${user.role}}, dataType: 'json', timeout: 100000, success: function(model) { From b3696862903ca60d4f208acdbfe7f691c32a792d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 22 Sep 2021 10:32:43 +0800 Subject: [PATCH 17/80] update --- .../ccbevaluation/UpdateHouseRegion.groovy | 88 +++++++++++++------ .../fangjia/script/gj/YouDaDataParser.groovy | 20 ++--- .../district/SeaweedDataMatchProcess.groovy | 7 +- 3 files changed, 74 insertions(+), 41 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy index c71f088de..a47028b79 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy @@ -37,13 +37,13 @@ class UpdateHouseRegion { static main(args) { def task = new UpdateHouseRegion() -// task.updateSeawaterRegion() + task.updateSeawaterRegion() // task.updateDealRegion() - task.updateThirdPartyRegion() +// task.updateThirdPartyRegion() } def updateThirdPartyRegion(){ - def city = "杭州" + def city = "东莞" def rets = findThirdPartyPrice(city) batchThirdPartyUpdate(city, rets); @@ -60,40 +60,70 @@ class UpdateHouseRegion { return rets; } - def updateDealRegion(){ - def city = "杭州" + def findDealData(def city){ def q = new BasicDBObject() q.put("city", city) q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-01-01"))); - def cursor = DEAL_PRICE_COLL.find(q) + def cursor = DEAL_PRICE_COLL.find(q).batchSize(2000) + def datas = [] def index = 0 - def updateIndex = 0 for(def obj : cursor){ index++ - def region = obj.region - def name = obj.district_name - def rets = CommonDatautils.findDistrictsByName(city, name, "district") - if (!rets) { - rets = CommonDatautils.findDistrictByAlias(city, null, name, "district") - } - - if (rets.size() == 1) { - def dist = rets[0] - if (region != dist.region) { - updateIndex++ - def udata = new BasicDBObject() - udata.put("region", dist.region) - DEAL_PRICE_COLL.update(new BasicDBObject('_id', obj.get("_id")), new BasicDBObject('$set', udata)) - println updateIndex+" "+index+" "+region+" "+dist.region - } - } + datas << obj + println index } - - + return datas } + + def updateDealRegion(){ + def city = "东莞" + def datas = findDealData(city) + batchDealRegion(city, datas) + } + + def batchDealRegion(def city, def datas){ + def updateCount = new AtomicLong() + def curCount = new AtomicLong() + def totalCount = datas.size(); + final CountDownLatch latch = new CountDownLatch(totalCount) + println city +" 成交总量:"+totalCount + for(def i=0;i Date: Thu, 23 Sep 2021 10:07:54 +0800 Subject: [PATCH 18/80] update --- .../ccbevaluation/CcbCommonUtils.groovy | 40 ++++++++++++++++++- .../script/commData/CommonDatautils.groovy | 22 ++++++++++ .../fangjia/script/gj/GaoDeMapCrawl.groovy | 19 ++++++--- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 17 ++++---- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 5 ++- 5 files changed, 88 insertions(+), 15 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 0b413551a..f52ccdf19 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -103,6 +103,13 @@ class CcbCommonUtils { alias << dist.name } + if(dist.name.endsWith("弄小区")){ + def name = dist.name.replace("弄小区","弄") + if(!alias.contains(name)){ + alias << name + } + } + if(alias){ q.put("district_name", new BasicDBObject('$in', alias)) } @@ -131,6 +138,37 @@ class CcbCommonUtils { return lists } + public static def findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + q.put("area", new BasicDBObject('$gte', 20)) + q.put("district_name", dist.address) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcursor = DEAL_PRICE_COLL.find(q) + def total = dbcursor.count() + def rets = new HashSet() + def lists = [] + for(def d : dbcursor){ + if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){ + def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + if(!rets.contains(key)){ + rets << key + d.dist_id = dist._id.toString() + d.address = dist.address + d.master_name = dist.name + d.build_type = dist.build_type + d.build_structure = dist.build_structure + d.house_roperty = dist.house_roperty + d.property_life = dist.property_life + d.estate_type2 = dist.estate_type2 + lists << d + } + } + } + return lists + } + public static def findTrades(dist, startDate, endDate){ def q = new BasicDBObject() @@ -187,7 +225,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name", "莱兴公寓") +// q.put("name", "周家湾") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 42d3641f3..f98fd8db7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1231,6 +1231,28 @@ class CommonDatautils { return ret; } + + + public static def findDistrictByAddress2(def city, def region, def address, def cat){ + DBObject q = new BasicDBObject(); + if(region){ + q.put('region', region) + } + + q.put('city',city) + q.put('cat', cat) + q.put('visible',0) + q.put("address", address); + + + def dsCursor = getDataStoreConnection().find(q); + def ret = []; + for(def obj : dsCursor){ + ret << obj; + } + + return ret; + } public static def getCityRegions(city){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index ddc92508f..520a1bdd0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -119,10 +119,11 @@ class GaoDeMapCrawl extends BaseBatch{ def run(){//,"西安","重庆" //def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] - def citys =["上海","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"] + def citys = ["上海","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"] def allcitys = CommonUtils.findAllCity() def totalCount = 0; for(def city : allcitys){ + if(!citys.contains(city)){ def stop = false while(stop == false){ @@ -135,6 +136,7 @@ class GaoDeMapCrawl extends BaseBatch{ } } } + } } @@ -146,7 +148,11 @@ class GaoDeMapCrawl extends BaseBatch{ q.put(prefix+"status", new BasicDBObject(QueryOperators.EXISTS, false)) // q.put("status", "失败") // q.put("traffic_status", new BasicDBObject(QueryOperators.EXISTS, false)) - def results = BOUNDS_COLL.find(q).limit(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def dbcursor = BOUNDS_COLL.find(q).batchSize(3000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def results = [] + for(def d : dbcursor){ + results << d + } return results; } @@ -187,14 +193,15 @@ class GaoDeMapCrawl extends BaseBatch{ } def searching(i, location){ - // def bound_gd = location.bound_gd +// def bound_gd = location.bound_gd // def lats = [bound_gd[1], bound_gd[3]] // def lngs = [bound_gd[0], bound_gd[2]] // lats.sort() // lngs.sort() -// println index -// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){ - search(i, location) + println i+" "+ location +// 120.019556,30.313701 + //if(lngs[0]<=120.019556 && lngs[1]>=120.019556 && lats[0]<=30.313701 && lats[1]>=30.313701){ + search(i, location) // } if(updateLists.size() > 0 && updateLists.size()>1000){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 53823ee85..d69846af1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -45,15 +45,18 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - for(def i=0;i<5;i++){ - def city = "福州" - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); - distMap.put(city, districts) + def citys = CommonDatautils.findCitys() + for(def city : citys){ + for(def i=0;i<1;i++){ +// def city = "福州" + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); + distMap.put(city, districts) - addCrawlKeys(city); - addDetailContent(city, "https://fz.58.com/xiaoqu/"); + addCrawlKeys(city); + addDetailContent(city, "https://fz.58.com/xiaoqu/"); - Thread.sleep(1000*60*10) + Thread.sleep(1000*60*10) + } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index d0ebd044c..a8b9a0427 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -17,7 +17,7 @@ class SynchDistrictPrice { def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", "anjuke.com":"安居客","cityhouse.cn":"城市房产"] -// def sourceMap = ["cityhouse.cn":"城市房产"] +// def sourceMap = ["lianjia.com":"链家"] static main(args) { def task = new SynchDistrictPrice() task.run() @@ -43,6 +43,7 @@ class SynchDistrictPrice { def q = new BasicDBObject() q.put("city", city) q.put("root", root) +// q.put("name","兰州万达城") q.put('m_date', new BasicDBObject('$gte', curMonthFirstDay)); def keys = new BasicDBObject() keys.put("city", 1) @@ -95,6 +96,8 @@ class SynchDistrictPrice { thirdPartyColl.remove(new BasicDBObject('_id',obj._id)) addIndex++ thirdPartyColl.save(r) + }else{ + thirdPartyColl.save(r) } // if(!obj){ // addIndex++ From ba4642f3664294c96191ce6bb0f7bf0bec2bc521 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 23 Sep 2021 10:13:38 +0800 Subject: [PATCH 19/80] =?UTF-8?q?=E6=88=90=E4=BA=A4=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 49 ++++++++++++++++--- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index f928400f6..0fb1a6a97 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -13,6 +13,7 @@ import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import groovy.sql.Sql import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch @@ -37,6 +38,10 @@ class BigdataDealBatch extends BaseBatch{ DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static main(args) { runBatch(new BigdataDealBatch(), args); } @@ -45,15 +50,19 @@ class BigdataDealBatch extends BaseBatch{ public void process(Object args) { def weekend = DateUtils.getFirstDayOfWeek(new Date()) def citys = CcbCommonUtils.citys; + println citys.unique().size() def rows = [] def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - + def ppeDists = [:] + if(["贵阳","佛山","杭州","沈阳","天津","中山","徐州","绍兴","南通","哈尔滨","惠州","长沙","青岛","长春","烟台","温州","太原","台州","石家庄","宁波","常州","大连","嘉兴","金华","昆明"].contains(city)){ + ppeDists = getPpeGTE15Datas(city) + } def statMap = getDistrictStatByCity(city, weekend) - def trades = getCityData(city, statMap) + def trades = getCityData(city, statMap, ppeDists) def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") rows.addAll(rets) println city +" 成交共:"+rets.size() @@ -391,7 +400,7 @@ class BigdataDealBatch extends BaseBatch{ return s; } - def getCityData(def city, def statMap){ + def getCityData(def city, def statMap, def ppeDistsMap){ def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"]) def tradesList = Collections.synchronizedList(new ArrayList()); @@ -407,7 +416,7 @@ class BigdataDealBatch extends BaseBatch{ // startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // } - def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) for(def i=0;i Date: Thu, 23 Sep 2021 10:26:19 +0800 Subject: [PATCH 20/80] bug update --- .../groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index d69846af1..85943b397 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -45,7 +45,7 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - def citys = CommonDatautils.findCitys() + def citys = CommonDatautils.findAllCity() for(def city : citys){ for(def i=0;i<1;i++){ // def city = "福州" From 994cf4e86cb62212ea586385881d0e0d2edeae8e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 23 Sep 2021 11:09:48 +0800 Subject: [PATCH 21/80] =?UTF-8?q?58=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 163 +++++++++++++++++- 1 file changed, 157 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 85943b397..99cdc2a86 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -38,6 +38,152 @@ class Esf58Crawl extends BaseBatch{ def static useDistKeys = [] static def totalCount = new AtomicLong() + def cityList = [ + "合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319", + "安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","淮北":"huaibei|9357","滁州":"chuzhou|10266", + "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", + "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296", + "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232", + "福州":"fz|304","厦门":"xm|606","泉州":"qz|291","莆田":"pt|2429","漳州":"zhangzhou|710", + "宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761", + "石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757", + "福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133", + "漳浦":"zhangpu|717", + "深圳":"sz|4","广州":"gz|3","东莞":"dg|413","佛山":"fs|222","中山":"zs|771", + "珠海":"zh|910","惠州":"huizhou|722","江门":"jm|629","汕头":"st|783","湛江":"zhanjiang|791", + "肇庆":"zq|901","茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389","清远":"qingyuan|7303", + "阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449", + "潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725", + "博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456", + "南宁":"nn|845","柳州":"liuzhou|7133","桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046", + "北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340", + "来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774", + "北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119", + "贵阳":"gy|2015","遵义":"zunyi|7620","黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506", + "毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628", + "清镇":"qingzhen|12703", + "兰州":"lz|952","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154", + "酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428", + "陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390", + "海口":"haikou|2053","三亚":"sanya|2422","五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", + "文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184", + "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","保亭":"baoting|10367","白沙":"baish|10380", + "儋州":"danzhou|10394", + "郑州":"zz|342","洛阳":"luoyang|556","新乡":"xx|1016","南阳":"ny|592","许昌":"xc|977", + "平顶山":"pds|1005","安阳":"ay|1096","焦作":"jiaozuo|3266","商丘":"sq|1029","开封":"kaifeng|2342", + "濮阳":"puyang|2346","周口":"zk|933","信阳":"xy|8694","驻马店":"zmd|1067","漯河":"luohe|2347", + "三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123", + "禹州":"yuzhou|979","长葛":"changge|9344","灵宝":"lingbaoshi|9307","杞县":"qixianqu|7389","汝州":"ruzhou|1010", + "项城":"xiangchengshi|935","偃师":"yanshiqu|7121","长垣":"changyuan|5936","滑县":"huaxian|5405","林州":"linzhou|1101", + "沁阳":"qinyang|3268","孟州":"mengzhou|3267","温县":"wenxian|7312","尉氏":"weishixian|7391","兰考":"lankaoxian|7393", + "通许":"tongxuxian|7390","新安":"lyxinan|11217","伊川":"yichuan|11220","孟津":"mengjinqu|7122","宜阳":"lyyiyang|11219", + "舞钢":"wugang|1011","永城":"yongcheng|1032","睢县":"suixian|1038","鹿邑":"luyi|939","渑池":"yingchixian|9322", + "沈丘":"shenqiu|942","太康":"taikang|938","商水":"shangshui|936","淇县":"qixianq|9186","浚县":"junxian|9185", + "范县":"fanxian|7285","固始":"gushixian|8698","淮滨":"huaibinxian|8702","邓州":"dengzhou|595","新野":"xinye|603", + "哈尔滨":"hrb|202","大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", + "佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862", + "伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721", + "肇州":"zhaozhou|382", + "武汉":"wh|158","宜昌":"yc|858","襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", + "黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296", + "咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517", + "仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896", + "武穴":"wuxueshi|7362","钟祥":"zhongxiangshi|9119","京山":"jingshanxian|9117","沙洋":"shayangxian|9118","松滋":"songzi|3484", + "广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897", + "南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436", + "当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660", + "长沙":"cs|414","株洲":"zhuzhou|1086","益阳":"yiyang|10198","常德":"changde|872","衡阳":"hy|914", + "湘潭":"xiangtan|2047","岳阳":"yy|821","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756", + "永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091", + "澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921", + "祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473", + "邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532", + "湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095", + "石家庄":"sjz|241","保定":"bd|424","唐山":"ts|276","廊坊":"lf|772","邯郸":"hd|572", + "秦皇岛":"qhd|1078","沧州":"cangzhou|652","邢台":"xt|751","衡水":"hs|993","张家口":"zjk|3328", + "承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048", + "正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577", + "雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657", + "沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395", + "固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066", + "沙河":"shaheshi|755", + "苏州":"su|5","南京":"nj|172","无锡":"wx|93","常州":"cz|463","徐州":"xz|471", + "南通":"nt|394","扬州":"yz|637","盐城":"yancheng|613","淮安":"ha|968","连云港":"lyg|2049", + "泰州":"taizhou|693","宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279", + "如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147", + "扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402", + "邳州":"pizhou|477","沛县":"xzpeixian|11349","靖江":"jingjiang|698","建湖":"jianhu|618","海安":"haian|401", + "东台":"dongtai|615","丹阳":"danyang|648","宝应县":"baoyingx|14451","灌南":"guannan|2150","灌云":"guanyun|2148", + "姜堰":"jiangyan|697","金坛":"jintan|468","昆山":"szkunshan|16","泗洪":"sihong|5958","泗阳":"siyang|5959", + "句容":"jurong|650","射阳":"sheyang|621","阜宁":"funingxian|620","响水":"xiangshui|619","盱眙":"xuyi|976", + "金湖":"jinhu|975","江阴":"jiangyins|34984", + "南昌":"nc|669","赣州":"ganzhou|2363","九江":"jj|2247","宜春":"yichun|5709","吉安":"ja|2364", + "上饶":"sr|10120","萍乡":"px|2248","抚州":"fuzhou|10134","景德镇":"jdz|2360","新余":"xinyu|10115", + "鹰潭":"yingtan|3209","永新":"yxx|11077","乐平":"lepingshi|9048","进贤":"jinxian|677","分宜":"fenyi|10118", + "丰城":"fengchengshi|5711","樟树":"zhangshu|5713","高安":"gaoan|5712","余江":"yujiang|3210","南城":"nanchengx|10137", + "浮梁":"fuliangxian|9071", + "长春":"cc|319","吉林":"jl|700","四平":"sp|10171","延边":"yanbian|3184","松原":"songyuan|2315", + "白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171", + "梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708", + "梨树县":"lishu|10176", + "沈阳":"sy|188","大连":"dl|147","鞍山":"as|523","锦州":"jinzhou|2354","抚顺":"fushun|5722", + "营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038", + "本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306", + "瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733", + "银川":"yinchuan|2054","吴忠":"wuzhong|9962","石嘴山":"szs|9971","中卫":"zw|9951","固原":"guyuan|2421", + "呼和浩特":"hu|811","包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015", + "呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976", + "乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043", + "西宁":"xn|2052","海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909", + "黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904", + "青岛":"qd|122","济南":"jn|265","烟台":"yt|228","潍坊":"wf|362","临沂":"linyi|505", + "淄博":"zb|385","济宁":"jining|450","泰安":"ta|686","聊城":"lc|882","威海":"weihai|518", + "枣庄":"zaozhuang|961","德州":"dz|728","日照":"rizhao|3177","东营":"dy|623","菏泽":"heze|5632", + "滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146", + "寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690", + "高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235", + "蓬莱":"penglai|237","青州":"qingzhou|367","荣成":"rongcheng|522","乳山":"rushan|520","滕州":"tengzhou|967", + "新泰":"xintai|689","招远":"zhaoyuan|3325","邹城":"zoucheng|455","邹平":"zouping|946","临清":"linqing|884", + "茌平":"chiping|887","郓城":"hzyc|5637","博兴":"boxing|949","东明":"dongming|5641","巨野":"juye|5640", + "无棣":"wudi|951","齐河":"qihe|734","微山":"weishan|459","禹城":"yuchengshi|731","临邑":"linyixianq|739", + "乐陵":"leling|730","莱阳":"laiyang|234","宁津":"ningjin|733","高唐":"gaotang|885","莘县":"shenxian|888", + "阳谷":"yanggu|886","冠县":"guanxian|890","平邑":"pingyi|514","郯城":"tancheng|510","沂源":"yiyuanxian|7334", + "汶上":"wenshang|460","梁山":"liangshanx|462","利津":"lijin|628","沂南":"yinanxian|7301","栖霞":"qixia|238", + "宁阳":"ningyang|691","东平":"dongping|692","昌邑":"changyishi|372","安丘":"anqiu|370","昌乐":"changle|373", + "临朐":"linqu|374","鄄城":"juancheng|5635", + "太原":"ty|740","临汾":"linfen|5669","大同":"dt|6964","运城":"yuncheng|5653","晋中":"jz|8854", + "长治":"changzhi|6921","晋城":"jincheng|3350","阳泉":"yq|8760","吕梁":"lvliang|3222","忻州":"xinzhou|3453", + "朔州":"shuozhou|9871","临猗":"linyixian|9193","清徐":"qingxu|10908","柳林":"liulin|3225","高平":"gaoping|3354", + "泽州":"zezhou|3353","襄垣":"xiangyuanxian|6928","孝义":"xiaoyi|3227", + "西安":"xa|483","咸阳":"xianyang|7453","宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163", + "榆林":"yl|5942","延安":"yanan|8973","安康":"ankang|3157","商洛":"sl|9854","铜川":"tc|9832", + "神木":"shenmu|5944","韩城":"hancheng|5735","府谷":"fugu|5945","靖边":"jingbian|5947","定边":"dingbian|5948", + "成都":"cd|102","绵阳":"mianyang|1057","德阳":"deyang|2373","南充":"nanchong|2378","宜宾":"yb|2380", + "自贡":"zg|6745","乐山":"ls|3237","泸州":"luzhou|2372","达州":"dazhou|9799","内江":"scnj|5928", + "遂宁":"suining|9688","攀枝花":"panzhihua|2371","眉山":"ms|9704","广安":"ga|2381","资阳":"zy|6803", + "凉山":"liangshan|9717","广元":"guangyuan|9749","雅安":"ya|9687","巴中":"bazhong|9811","阿坝":"ab|9817", + "甘孜":"ganzi|9764","安岳":"anyuexian|6806","广汉":"guanghanshi|8719","简阳":"jianyangshi|6805","仁寿":"renshouxian|9706", + "射洪":"shehongxian|9694","大竹":"dazu|9806","宣汉":"xuanhan|9804","渠县":"qux|9807","长宁":"changningx|7148", + "乌鲁木齐":"xj|984","昌吉":"changji|8582","巴音郭楞":"bygl|9530","伊犁":"yili|9472","阿克苏":"aks|9499", + "喀什":"ks|9326","哈密":"hami|7452","克拉玛依":"klmy|2042","博尔塔拉":"betl|9529","吐鲁番":"tlf|9475", + "和田":"ht|9489","石河子":"shz|9551","克孜勒苏":"kzls|9519","阿拉尔":"ale|9539","五家渠":"wjq|9562", + "图木舒克":"tmsk|9559","库尔勒":"kel|7168","阿勒泰":"alt|18837","塔城":"tac|18845", + "拉萨":"lasa|2055","日喀则":"rkz|9615","山南":"sn|9576","林芝":"linzhi|9646","昌都":"changdu|9648", + "那曲":"nq|9618","阿里":"al|9678","日土":"rituxian|9682","改则":"gaizexian|9684", + "昆明":"km|541","曲靖":"qj|2389","大理":"dali|2398","红河":"honghe|2394","玉溪":"yx|2040", + "丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409", + "德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432", + "怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533", + "杭州":"hz|79","宁波":"nb|135","温州":"wz|330","金华":"jh|531","嘉兴":"jx|497", + "台州":"tz|403","绍兴":"sx|355","湖州":"huzhou|831","丽水":"lishui|7921","衢州":"quzhou|6793", + "舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333", + "诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334", + "长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536", + "安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409", + "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361", + "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575" + ] + static main(args) { runBatch(new Esf58Crawl(), args) } @@ -45,21 +191,26 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - def citys = CommonDatautils.findAllCity() - for(def city : citys){ + cityList.each{city, str-> for(def i=0;i<1;i++){ // def city = "福州" def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); - distMap.put(city, districts) + if(districts){ + def arrs = str.split("\\|") + def url = "https://"+arrs[0]+".58.com/xiaoqu/" + distMap.put(city, districts) - addCrawlKeys(city); - addDetailContent(city, "https://fz.58.com/xiaoqu/"); + addCrawlKeys(city); + addDetailContent(city, url); - Thread.sleep(1000*60*10) +// Thread.sleep(1000*60*10) + } } } } + + def addCrawlKeys(city){ def districts = distMap.get(city); for(def i=0;i Date: Fri, 24 Sep 2021 18:36:08 +0800 Subject: [PATCH 22/80] update --- .../ccbevaluation/BigdataDealBatch.groovy | 4 +- .../thirdPriceCrawl/DataStandardBatch.groovy | 72 +++++++++---------- 2 files changed, 36 insertions(+), 40 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 0fb1a6a97..ec1b947cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -80,9 +80,9 @@ class BigdataDealBatch extends BaseBatch{ // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据") } - def fileName = "/home/goojia/ccb/bigdata_deal.dat" + def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.dat" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" } FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index cbe499054..5522782e9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -61,7 +61,7 @@ class DataStandardBatch extends BaseBatch{ private static def limit = 5000; def static final SHORT_DATE = Utils.getLastMonth() - def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6] + def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7] def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) @@ -83,14 +83,13 @@ class DataStandardBatch extends BaseBatch{ @Override void process(Object args) { - def flag = true; // ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl) cityMap.each{city,alias-> - println "开始加载【${city}】" - def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"]) - districtMap.putAll(detailMap) + println "开始加载【${city}】" + def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"]) + districtMap.putAll(detailMap) } while(flag){ @@ -122,7 +121,7 @@ class DataStandardBatch extends BaseBatch{ void run() { try{ println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】" -// if(dbo.avg_price){ + if(dbo.name){ //标准化城市(包含region) def data = standardCity(dbo); @@ -147,10 +146,10 @@ class DataStandardBatch extends BaseBatch{ } } -// }else{ -// saveLog(dbo, "无价格"); -// updateStatus(dbo,statusMap.get("无价格")) -// } + }else{ + saveLog(dbo, "无名称"); + updateStatus(dbo,statusMap.get("无名称")) + } count++; }catch (Exception e){ @@ -480,27 +479,7 @@ class DataStandardBatch extends BaseBatch{ type = 12;//区域不同 district = null; }else{ - def score1 = 0; - if(data.address && district.address){ - score1 = CosineSimilarAlgorithm.getSimilarityRatio(data.address, district.address); - } - if(score1 && score1 >=0.8){ - type = 13;//地址相似度在0.8以上 - }else{ - if(addr && (district.address && addr.indexOf(district.address)>-1||district.address.toString().indexOf(addr)>-1) && (data.name.indexOf(district.name)>-1 || district.name.toString().indexOf(data.name)>-1)){ - type = 13; - }else{ - if(!addr){ - def openDistName = district.name.replace("小区","") - def name = data.name.replace("小区","") - if(openDistName.contains(name) || name.contains(openDistName)){ - type = 15;//匹配接口通过名称配上,名称包含 - } - } - } - //type = 11; - district = null; - } + type=15 } } } @@ -607,11 +586,13 @@ class DataStandardBatch extends BaseBatch{ def getMatchNameFromMap(def city, def region, def disName, String address){ //查询房价网是否存在该小区 def dbo = findSameNameFromMap(city, region, disName); - def addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') - if(addrFlag){ - dbo = addressCheck(dbo, region, address); + def addrFlag = null + if(address){ + addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') + if(addrFlag){ + dbo = addressCheck(dbo, region, address); + } } - if(!dbo){ //别名 def distrs = findAliasNameFromMap(city, region, disName); @@ -683,10 +664,16 @@ class DataStandardBatch extends BaseBatch{ return dbo; }*/ def getMatchAddressFromMap(def city, def region, def address, def name){ - def dbo = findSameNameFromMap(city, region, address); + def dbo = null + if(address && isStandardRoad(address)){ + dbo = findSameNameFromMap(city, region, address); + } if(!dbo){ - def result = findAddressFromMap(city, region, address); - if(result.size()==1){ + def result = null + if(address && isStandardRoad(address)){ + result = findAddressFromMap(city, region, address); + } + if(result && result.size()==1){ dbo = result[0]; }else if(result && result.size()>1 && name){ def score = null; @@ -734,6 +721,15 @@ class DataStandardBatch extends BaseBatch{ return dbo; } + def isStandardRoad(def s){ + def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)"); + def matcher = pattern.matcher(s) + if(matcher.find()){ + return true; + } + return false + } + def addressCheck(def dbo, def region, def address){ if(dbo && !region && address && address.length()>3){ if(dbo.address.contains(address)||address.contains(dbo.address)||dbo.address==address){ From 0948920c2fcb70c273d242e0018adf5ddc789c4f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 26 Sep 2021 10:31:16 +0800 Subject: [PATCH 23/80] update --- .../script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 5f96db29c..3f598a077 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -19,7 +19,7 @@ import java.util.concurrent.Executors @BatchOwner(Watcher.GaoJuan) class ThirdDataToGatherBatch extends BaseBatch{ - def static final SHORT_DATE = Utils.getLastMonth() + def static final SHORT_DATE = 202107//Utils.getLastMonth() def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol(); //MongoUtils.getMongoDBByAuth().getCollection("data_process") def static final DBCollection AVERAGE_STATISTICS_OFFLINE = CrawalDBUtil.getAverage_statistics_col(); @@ -43,6 +43,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ void process(Object args) { def citys = CommonDatautils.findAllCity(); def weekend = DateUtils.getFirstDayOfWeek(new Date()); + //def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15")); def cityIndex = 0; final CountDownLatch latch = new CountDownLatch(citys.size()) citys.each { city -> From a3bb2916b3f291d28ec1a16955def96dfd7fca15 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 26 Sep 2021 10:32:24 +0800 Subject: [PATCH 24/80] update --- .../com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 5522782e9..75a441c06 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -782,7 +782,6 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", SHORT_DATE) -// qdata.put("name", "银河湾花园(别墅)") qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); def ret = [] try{ From a78088a8bdeab1ebad0b28f35fbe2bfbd396ffb3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 26 Sep 2021 18:40:56 +0800 Subject: [PATCH 25/80] =?UTF-8?q?=E6=88=BF=E6=BA=908=E6=9C=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/BigdataForsaleBatch.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index d10aab75c..fc6ec3c08 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210831); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); } } @@ -96,9 +96,9 @@ class BigdataForsaleBatch extends BaseBatch{ def commId = rMap.get("_id").toString() def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias) - if(ret && ret.size()<100){ - ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) - } +// if(ret && ret.size()<100){ +// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) +// } if(ret){ def areaData = distCodeMap.get(rMap.region) if(areaData){ @@ -145,9 +145,9 @@ class BigdataForsaleBatch extends BaseBatch{ } latch.await() - def fileName = "/home/goojia/ccb/bigdata_forsale.dat" + def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.dat" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat" } println city + " 房源总量:"+retMap.size() From 87ed68ed615946842310916d6a5a41dbe80f6bb0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 26 Sep 2021 19:27:18 +0800 Subject: [PATCH 26/80] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BigdataCommunityBatch.groovy | 30 +++++++++-- .../ccbevaluation/CcbCommonUtils.groovy | 50 ++++++++++++++++++- .../fangjia/script/jiangnanblank/SFTP.groovy | 12 ++++- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 6 +-- 4 files changed, 90 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index c2efd484b..ef175956f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -12,6 +12,7 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -46,16 +47,19 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) println city +" 小区量:"+dists.size() + + def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) + rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) println city +" "+rets.size() // FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); } - def fileName = "/home/goojia/ccb/bigdata_community.dat" + def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.dat" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat" } def allDatas = [] @@ -66,8 +70,27 @@ class BigdataCommunityBatch extends BaseBatch{ // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") } + def addDeleteDist(dists, ccbMaps, rets){ + def fangjiaDists = [] + for(def d : dists){ + fangjiaDists << d._id.toString() + } + println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size() + def ccbDists = ccbMaps.keySet() + for(def ccbId : ccbDists){ + if(!fangjiaDists.contains(ccbId)){ + def datas = ccbMaps.get(ccbId) + datas[datas.size()-1] = 1 + rets << datas + } + } + println "添加删除状态小区:"+rets.size() + return rets + } + def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){ - def rets = Collections.synchronizedList(new ArrayList()); + //def rets = Collections.synchronizedList(new ArrayList()); + def rets = new CopyOnWriteArrayList() def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) for(def i=0;i + def data = [] + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.DISTNAME + data << r.COMMNAME + data << r.ADDRESS + data << r.LATITUDE + data << r.LONGITUDE + data << r.LANDAREA + data << r.BUILDAREA + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << r.BUILDYEAR + data << r.HOUSEUSE + data << r.ZTS + data << r.WORKDATE + data << r.UPDATEFLAG + data << r.UPDATETIME + data << r.COMMTYPE + data << r.COMMALIAS + data << r.STREET + data << r.STREETNO + data << r.IFNEW + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.SOURCECODE + data << r.XZJDB + data << r.LJXZQHDM + data << r.LJXZQH + data << r.STATUS + map.put(r.COMMID, data) + }); + return map + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy index 98eca9153..4a8801a5e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy @@ -47,12 +47,22 @@ class SFTP { * @param sftp */ public void upload(String directory, String uploadFile, ChannelSftp sftp) { + FileInputStream stream = null try { sftp.cd(directory); File file=new File(uploadFile); - sftp.put(new FileInputStream(file), file.getName()); + stream = new FileInputStream(file) + sftp.put(stream, file.getName()); } catch (Exception e) { e.printStackTrace(); + } finally { + if(stream!=null){ + stream.close(); + } +// if (sftp != null) { +// sftp.disconnect(); +// sftp.exit(); +// } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index a8b9a0427..0ba834ad5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -15,9 +15,9 @@ class SynchDistrictPrice { def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); - def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", - "anjuke.com":"安居客","cityhouse.cn":"城市房产"] -// def sourceMap = ["lianjia.com":"链家"] +// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", +// "anjuke.com":"安居客","cityhouse.cn":"城市房产"] + def sourceMap = ["zhuge.com":"诸葛找房"] static main(args) { def task = new SynchDistrictPrice() task.run() From 5164545184e39ad131f18b9f8e7c869bd4b718c3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 28 Sep 2021 11:07:30 +0800 Subject: [PATCH 27/80] =?UTF-8?q?9=E6=9C=88=E6=88=BF=E6=BA=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index fc6ec3c08..64f9c5548 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20210930); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); } } @@ -157,7 +157,7 @@ class BigdataForsaleBatch extends BaseBatch{ if(index == 1){ headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] } - FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", false); println "txt文件生成 " From 5d72594e30c38ef30e32433a2094c66b8022c352 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 28 Sep 2021 12:36:50 +0800 Subject: [PATCH 28/80] update --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 64f9c5548..db708a6ca 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -157,7 +157,7 @@ class BigdataForsaleBatch extends BaseBatch{ if(index == 1){ headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] } - FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true); println "txt文件生成 " From 510e0275f4f0a5a91575fc082a9128a7257ab67d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 28 Sep 2021 19:33:31 +0800 Subject: [PATCH 29/80] =?UTF-8?q?=E5=AF=BC=E6=A0=B7=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleData.groovy | 730 ++++++++++++++++++ 1 file changed, 730 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy new file mode 100644 index 000000000..898be1b50 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -0,0 +1,730 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.ccbevaluation.CcbFtpUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql + +import java.sql.ResultSet + +class ExportOracleData { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new ExportOracleData() + task.run() + } + def run(){ +// exportEvaluationRateData() +// exportBigdataPoiData() +// exportCommunityStaging() +// exportCommunityBuileType() +// exportBigdataDistrictData() +// exportGuidedPrice() +// exportCommunityGuidedPrice() +// exportBigdataDeal() +// exportThirdPartyPrice() + +// exportBigdataCommunity() + exportBigdataForsale() + +// exportAvmEvaluationGJNB() +// exportAVMEvaluationItemGJNB() +// exportCommunityBasePrice() + } + + def exportEvaluationRateData(){ + def sql = "select ID,RATE_TYPE,RATE,RANK,CITY,REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from evaluate_rate_gj"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.RATE_TYPE + data << r.RATE + data << r.RANK + data << r.CITY + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/output/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + def dataCodeFileName = "/output/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.ok" + } + def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, "202108") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + println "上传Ftp完成......" + } + + + def exportBigdataPoiData(){ + def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj where cityId in (350200,150100)"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.ID + data << r.POI_NAME + data << r.TYPE + data << r.TYPE_CODE + data << r.LATITUDE + data << r.LONGITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.ok" + } + def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "BIGDATA_POI_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, "202108") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + println "BIGDATA_POI_GJ 上传Ftp完成......" + } + + def exportCommunityStaging(){ + def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.COMMID + data << r.MASTER_COMMID + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.ok" + } + def headers = []//["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "COMMUNITY_STAGING_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, "202108") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + println "COMMUNITY_STAGING_GJ 上传Ftp完成......" + } + + def exportCommunityBuileType(){ + def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID in (350200,150100)"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.ok" + } + def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, "202108") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......" + } + + def exportBigdataDistrictData(){ + + def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj where cityId in (350200,150100)"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.LJXZQH + data << r.LJXZQHDM + data << r.XZQHDM + data << r.WORKDATE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.ok" + } + def headers = []//["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "BIGDATA_DISTRICT_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, "202108") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + println "BIGDATA_DISTRICT_GJ 上传Ftp完成......" + } + + + def exportGuidedPrice(){ + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where cityId in (350200,150100)"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.DEALDATE + data << r.AVG_PRICE + data << r.GUIDING_PERSON + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok" + } + def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "GUIDED_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, "202107") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202107") + println "GUIDED_PRICE_GJ 上传Ftp完成......" + } + + def exportCommunityGuidedPrice(){ + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where cityname='厦门' or cityname='呼和浩特'"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.AVGPRICE + data << r.SOURCETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok" + } + def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "COMMUNITY_GUIDE_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, "202108") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......" + } + + def exportBigdataDeal(){ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where cityId in (350200,150100) and dealdate>=to_date('2021-08-01','yyyy-mm-dd')"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.CJJE + data << r.WORKDATE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.ok" + } + def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "BIGDATA_DEAL_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, "202108") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + println "BIGDATA_DEAL_GJ 上传Ftp完成......" + } + + + def exportThirdPartyPrice(){ + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, \n" + + "TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE\n" + + "from third_party_price_gj where cityId in (350200,150100) and year_month<202108"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.ZHUGE_AVG_PRICE + data << r.ANJUKE_AVG_PRICE + data << r.FANG_AVG_PRICE + data << r.LIANJIA_AVG_PRICE + data << r.CITYHOUSE_AVG_PRICE + data << r.MAITIAN_AVG_PRICE + data << r.KE_AVG_PRICE + data << r.TAOFANG_AVG_PRICE + data << r.ZHONGYUAN_AVG_PRICE + data << r.TBS_AVG_PRICE + data << r.QFANG_AVG_PRICE + data << r.FANGTAN_AVG_PRICE + data << r.FANGDUODUO_AVG_PRICE + data << r.WIWJ_AVG_PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.ok" + } + def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, "202107") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202107") + println "THIRD_PARTY_PRICE_GJ 上传Ftp完成......" + } + + def exportBigdataCommunity(){ + def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" + + "\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj ";//where cityId in (350200,150100) + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.DISTNAME + data << r.COMMNAME + data << r.ADDRESS + data << r.LATITUDE + data << r.LONGITUDE + data << r.LANDAREA + data << r.BUILDAREA + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << r.BUILDYEAR + data << r.HOUSEUSE + data << r.ZTS + data << r.WORKDATE + data << r.UPDATEFLAG + data << r.UPDATETIME + data << r.COMMTYPE + data << r.COMMALIAS + data << r.STREET + data << r.STREETNO + data << r.IFNEW + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.SOURCECODE + data << r.XZJDB + data << r.LJXZQHDM + data << r.LJXZQH + data << r.STATUS + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.ok" + } + def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", + //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", + //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "BIGDATA_COMMUNITY_GJ 数据文件输出完成......" +// CcbFtpUtils.ftpUpload(fileName, "202109") +// CcbFtpUtils.ftpUpload(dataCodeFileName, "202109") +// println "BIGDATA_COMMUNITY_GJ 上传Ftp完成......" + } + + def exportBigdataForsale(){ + def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where PUBLISHTIME>=to_date('2021-09-01','yyyy-mm-dd')";//(cityId=350200 or cityId=150100) and + def allRows = [] + def index = 0 + sqlInstance.eachRow(sql,{r-> + index = index+1 + println index + def data = [] + data << r.CITYID + data << r.FORSALEID + data << r.PROPCODE + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.UNITPRICE + data << r.TOTALPRICE + data << r.PAYTYPE + data << r.FLOORNUM + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.HOUSESTRU + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEASPECT + data << r.FITMENT + data << r.IFELEVATOR + data << r.ELEVATORHOUSE + data << r.PUBLISHTIME + data << r.EXPIRETIME + data << r.IFFIRST + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.PUBLISHER + data << r.PHONE + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.ATTACH + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.ok" + } + def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "BIGDATA_FORSALE_GJ 数据文件输出完成......" +// CcbFtpUtils.ftpUpload(fileName, "202107") +// CcbFtpUtils.ftpUpload(dataCodeFileName, "202107") +// println "BIGDATA_FORSALE_GJ 上传Ftp完成......" + } + + + def exportAvmEvaluationGJNB(){ + def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" + + "ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" + + "DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where (cityId=350200 or cityId=150100) and year_month<202108 "; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.COMMUNITY_NUM + data << r.HAS_PRICE_COMMUNITY_NUM + data << r.PRICE_COVERAGE + data << r.ARTIFICIAL_CALC_NUM + data << r.ARTIFICIAL_CALC_COVERAGE + data << r.SAMPLE_CALC_NUM + data << r.SAMPLE_CALC_COVERAGE + data << r.ANN_CALC_NUM + data << r.ANN_CALC_COVERAGE + data << r.TEST_NUM + data << r.TEST_COMM_NUM + data << r.TEST_COMM_COVERAGE + data << r.PPE15_COVERAGE + data << r.RATE_MEDIAN + data << r.COEFFICIENT + data << r.DEAL_TEST_CASE_NUM + data << r.DEAL_TEST_COMM_NUM + data << r.DEAL_TEST_COMM_COVERAGE + data << r.DEAL_PPE15_COVERAGE + data << r.DEAL_COEFFICIENT + data << r.DEAL_RATE_MEDIAN + data << r.THIRD_TEST_CASE_NUM + data << r.THIRD_TEST_COMM_NUM + data << r.THIRD_TEST_COMM_COVERAGE + data << r.THIRD_PPE15_COVERAGE + data << r.THIRD_COEFFICIENT + data << r.THIRD_RATE_MEDIAN + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.ok" + } + def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE", + //"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE", + //"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "AVM_EVALUATION_GJNB 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, "202107") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202107") + println "AVM_EVALUATION_GJNB 上传Ftp完成......" + } + + def exportAVMEvaluationItemGJNB(){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" + + "to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where (cityId=350200 or cityId=150100) and year_month>=202108"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.CONFIDENCE + data << r.TARGET_AVG_PRICE + data << r.DEALDATE + data << r.UNIT_PRICE + data << r.BUILDAREA + data << r.FITMENT + data << r.HOUSEASPECT + data << r.FLOORNUM + data << r.FLOORCNT + data << r.DATA_TYPE + data << r.RATE + data << r.PE + data << r.CREATETIME + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.ok" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, "202108") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......" + } + + def exportCommunityBasePrice(){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where (cityId=350200 or cityId=150100) and year_month<202108"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.ok" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, "202107") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202107") + println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......" + } + + +} From a45a919db84e8a897896d0773182e8bdb3d40263 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 28 Sep 2021 19:44:15 +0800 Subject: [PATCH 30/80] update --- .../script/ccbevaluation/CcbFtpUtils.groovy | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy new file mode 100644 index 000000000..5e4c8766e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy @@ -0,0 +1,58 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.jiangnanblank.SFTP +import com.jcraft.jsch.ChannelSftp +import org.apache.commons.codec.digest.DigestUtils + +class CcbFtpUtils { + + static def ftpServer = "192.168.0.45" + static def ftpPort = 17495 + static def ftpUser = "ccb" + static def ftpPassword = "0I1k8EAM+O8iA" + static def ftpPath = "/home/ccb/output/" + + static SFTP sftp = new SFTP(); + static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword); + + + static def ftpUpload(outputPath_dist, month){ + //上传到ftp + sftpChannel = reConnect(); + sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel); + } + + static def reConnect(){ + if(sftpChannel == null){ + println "sftpChannel 为空,需重连....." + if(sftp == null){ + sftp = new SFTP(); + } + sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword); + }else{ + println "sftpChannel 正常....." + } + return sftpChannel; + } + + static def uploadSha512Data(def sb, def dataCodeFileName, def charset){ + String sha512HexStr= DigestUtils.sha512Hex(sb.getBytes())//生成加密码码 + DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件 + ftpUpload(dataCodeFileName)//上传加密码文件到FTP + } + + static def uploadSha512Data2(def dataFileName, def dataCodeFileName, def charset){ + + def data = new FileInputStream(new File(dataFileName)) + String sha512HexStr= DigestUtils.sha512Hex(data)//生成加密码码 + DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件 + ftpUpload(dataCodeFileName)//上传加密码文件到FTP + } + + + static main(args) { + + + } +} From c19c765a5173f344ddf41fe3c6156f7af729b1ed Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 28 Sep 2021 20:00:27 +0800 Subject: [PATCH 31/80] update --- .../com/fangjia/script/ccbevaluation/ExportOracleData.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 898be1b50..4c16c191d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -554,8 +554,8 @@ class ExportOracleData { allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" - def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.ok" + def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + def dataCodeFileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.ok" From e64fa84ab808f73518bbbcab1865c1bc14d63bc7 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 13 Oct 2021 12:46:29 +0800 Subject: [PATCH 32/80] update --- .../script/ccbevaluation/BigdataForsaleBatch.groovy | 10 ++++------ .../fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 10 +++++----- .../thirdPriceCrawl/ThirdDataToGatherBatch.groovy | 3 ++- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index db708a6ca..ab705a2a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20210930); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200101, 20210930); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); } } @@ -96,9 +96,9 @@ class BigdataForsaleBatch extends BaseBatch{ def commId = rMap.get("_id").toString() def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias) -// if(ret && ret.size()<100){ -// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) -// } + if(ret && ret.size()<100){ + ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) + } if(ret){ def areaData = distCodeMap.get(rMap.region) if(areaData){ @@ -159,8 +159,6 @@ class BigdataForsaleBatch extends BaseBatch{ } FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true); println "txt文件生成 " - - } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index eb080803a..2124be80f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -124,9 +124,9 @@ class CcbCommonUtils { def lists = [] for(def d : dbcursor){ if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){ - def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") - if(!rets.contains(key)){ - rets << key + //def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + //if(!rets.contains(key)){ +// rets << key d.dist_id = dist._id.toString() d.address = dist.address d.master_name = dist.name @@ -136,7 +136,7 @@ class CcbCommonUtils { d.property_life = dist.property_life d.estate_type2 = dist.estate_type2 lists << d - } + //} } } return lists @@ -229,7 +229,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name", "周家湾") +// q.put("name", "海昌欣城") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 3f598a077..ddc282cb5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -19,7 +19,7 @@ import java.util.concurrent.Executors @BatchOwner(Watcher.GaoJuan) class ThirdDataToGatherBatch extends BaseBatch{ - def static final SHORT_DATE = 202107//Utils.getLastMonth() + def static final SHORT_DATE = Utils.getLastMonth() def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol(); //MongoUtils.getMongoDBByAuth().getCollection("data_process") def static final DBCollection AVERAGE_STATISTICS_OFFLINE = CrawalDBUtil.getAverage_statistics_col(); @@ -41,6 +41,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ @Override void process(Object args) { + println SHORT_DATE def citys = CommonDatautils.findAllCity(); def weekend = DateUtils.getFirstDayOfWeek(new Date()); //def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15")); From a68ba3536096e4ff165dd6730b5b61d45e272b0a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 13 Oct 2021 17:25:04 +0800 Subject: [PATCH 33/80] update --- .../ccbevaluation/BigdataDealBatch.groovy | 79 +++++++++++-------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index ec1b947cc..f665824f6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -29,7 +29,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def curDate = 20210401//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); + def curDate = 20210701//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] @@ -48,7 +48,7 @@ class BigdataDealBatch extends BaseBatch{ @Override public void process(Object args) { - def weekend = DateUtils.getFirstDayOfWeek(new Date()) + def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) def citys = CcbCommonUtils.citys; println citys.unique().size() def rows = [] @@ -58,21 +58,25 @@ class BigdataDealBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def ppeDists = [:] - if(["贵阳","佛山","杭州","沈阳","天津","中山","徐州","绍兴","南通","哈尔滨","惠州","长沙","青岛","长春","烟台","温州","太原","台州","石家庄","宁波","常州","大连","嘉兴","金华","昆明"].contains(city)){ - ppeDists = getPpeGTE15Datas(city) - } - def statMap = getDistrictStatByCity(city, weekend) - def trades = getCityData(city, statMap, ppeDists) - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") - rows.addAll(rets) - println city +" 成交共:"+rets.size() +// if(["贵阳","佛山","杭州","沈阳","天津","中山","徐州","绍兴","南通","哈尔滨","惠州","长沙","青岛","长春","烟台","温州","太原","台州","石家庄","宁波","常州","大连","嘉兴","金华","昆明"].contains(city)){ +// ppeDists = getPpeGTE15Datas(city) +// } + def statMap = [:]//getDistrictStatByCity(city, weekend) if(city=="上海"){ def youDataTrade = getYouDaData(statMap) def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") rows.addAll(youdaDatas) println city +" "+youdaDatas.size() } + + + def trades = getCityData(city, statMap, ppeDists) + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + rows.addAll(rets) + println city +" 成交共:"+rets.size() + + // FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false); def cityDatas = [] cityDatas << headers @@ -411,33 +415,33 @@ class BigdataDealBatch extends BaseBatch{ def startDate = null; // if(citys.contains(city)){ - startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd")) + startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // }else{ // startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // } - def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) for(def i=0;i=startPrice && obj.price<=endPrice){ rets << obj; + }else{ + index++ + println index + " "+obj } - } - } } return rets; From 48c505166d22479b67b3930f364b4e9f80c6b407 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 13 Oct 2021 17:29:01 +0800 Subject: [PATCH 34/80] =?UTF-8?q?=E5=8F=8B=E8=BE=BE=E5=8E=BB=E4=BB=B7?= =?UTF-8?q?=E6=A0=BC=E9=99=90=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index f665824f6..7ee409fc9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -486,25 +486,25 @@ class BigdataDealBatch extends BaseBatch{ for(def obj : dbcursor){ if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房","其它"].contains(obj.houseType)){ - - def price = statMap.get(obj.fjRegion+"@"+obj.fjName) - if(price){ - def startPrice = 0 - def endPrice= 0 - if(obj.fjCity=="上海"){ - startPrice = (int)(price - price*0.1) - endPrice = (int)(price*0.5 + price) - }else{ - startPrice = (int)(price - price*0.5) - endPrice = (int)(price*0.5 + price) - } - if(obj.price>=startPrice && obj.price<=endPrice){ - rets << obj; - }else{ - index++ - println index + " "+obj - } - } + rets << obj; +// def price = statMap.get(obj.fjRegion+"@"+obj.fjName) +// if(price){ +// def startPrice = 0 +// def endPrice= 0 +// if(obj.fjCity=="上海"){ +// startPrice = (int)(price - price*0.1) +// endPrice = (int)(price*0.5 + price) +// }else{ +// startPrice = (int)(price - price*0.5) +// endPrice = (int)(price*0.5 + price) +// } +// if(obj.price>=startPrice && obj.price<=endPrice){ +// rets << obj; +// }else{ +// index++ +// println index + " "+obj +// } +// } } } return rets; From 66460b027facd1f7f2d0adaa3b425b0b92ad433d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 13 Oct 2021 17:43:30 +0800 Subject: [PATCH 35/80] update --- .../fangjia/script/baidu/CityLatlngBox.groovy | 11 +- .../BigdataCommbuildtypeBatch.groovy | 2 +- .../ccbevaluation/BigdataDistrictBatch.groovy | 4 +- .../ccbevaluation/BigdataPoiBatch.groovy | 4 +- .../ccbevaluation/CcbCommonUtils.groovy | 18 +- .../ccbevaluation/DistrictDelete.groovy | 44 ++- .../DistrictGuidePriceBatch.groovy | 6 +- .../ccbevaluation/EvaluateRateBatch.groovy | 4 +- .../ccbevaluation/ExportOracleData.groovy | 113 +++++- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../ccbevaluation/LocationUpdate.groovy | 78 ++++ .../ccbevaluation/StagingDistrictBatch.groovy | 4 +- .../fangjia/script/ccbevaluation/Test.groovy | 46 ++- .../script/ccbevaluation/TestRepeat.groovy | 81 ++++ .../ccbevaluation/ThirdPartyPriceBatch.groovy | 15 +- .../script/ccbevaluation/TradeStatLog.groovy | 369 +++++++++++++++++- .../ccbevaluation/UpdateHouseRegion.groovy | 112 +++++- .../com/fangjia/script/fuzhou/Test1.groovy | 4 +- .../fangjia/script/gj/OpenUserBatch.groovy | 12 +- .../fangjia/script/gj/ccb/CcbXykData.groovy | 18 + .../fangjia/script/gj/ccb/FileUtils.groovy | 8 + .../gj/location/DistrictLocationUpdate.groovy | 64 +++ .../script/gj/paimai/GaoDeUtils.groovy | 4 + .../script/gj/test/MatchDistrict.groovy | 45 +++ .../com/fangjia/script/gj/test/Test.groovy | 48 ++- .../script/test/RegionalismCodeStat.groovy | 58 +++ .../thirdPriceCrawl/SynchDistrictPrice.groovy | 2 +- .../TestCrawlTotalCount.groovy | 4 +- 28 files changed, 1084 insertions(+), 96 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LocationUpdate.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbXykData.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MatchDistrict.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/test/RegionalismCodeStat.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/baidu/CityLatlngBox.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/baidu/CityLatlngBox.groovy index 55204f53b..631b76733 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/baidu/CityLatlngBox.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/baidu/CityLatlngBox.groovy @@ -13,8 +13,9 @@ import com.mongodb.DBObject */ class CityLatlngBox { - public static final DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.105", 27018) - public static final DBCollection LATLNG_COLL = MongoUtils.getCollectionByName("addrepo", "city_latlng_range", "192.168.0.219", 27017) + //public static final DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.105", 27018) + DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018) + public static final DBCollection LATLNG_COLL = MongoUtils.getCollectionByName("addrepo", "city_latlng_range", "192.168.0.206", 27020) public static void main(String[] args) { CityLatlngBox sd = new CityLatlngBox(); @@ -22,7 +23,7 @@ class CityLatlngBox { } void process() { - def cities = CommonUtils.findAllCity() + def cities = ["白沙","阿坝州","甘孜州","阿勒泰","临沧","那曲","三沙","喀什","昌吉","迪庆州","普洱","德宏州","湘西","塔城","凉山州","屯昌","山南","海西州","阿拉善盟","乐东","定安","怒江州","黄山","图木舒克","楚雄","临高","乌兰察布","延边","文山州","锡林郭勒盟","陵水","天门","淮北","黔东南","万宁","湘潭","海北州","琼中","吐鲁番","惠州","巴音郭楞","恩施","铜仁","黔南","西双版纳州","潜江","林芝","文昌","仙桃","神农架","和田","黔西南","果洛州","儋州","哈密","大理","东方","日喀则","兴安盟","昌江","澄迈","保亭","济源","五指山","阿拉尔","伊犁","临夏","海南州","琼海","石河子","昌都","五家渠","玉树州","博尔塔拉","东营","铜陵","黄南州","廊坊","嘉峪关","马鞍山","宁波","广安","中山","克孜勒苏","厦门","莱芜","鄂州","东莞","海口","鹤壁","六盘水","漯河","三亚","珠海","深圳","乌海","淮南","北海","双鸭山","襄阳","新余","汕头","克拉玛依","鹰潭","盘锦","镇江","石嘴山","嘉兴","铜川","焦作","芜湖","潮州","萍乡","红河","黄石","辽源","辽阳","枣庄","毕节","许昌","阳泉","自贡","遂宁","景德镇","营口","日照","莆田"]//CommonUtils.findAllCity() for (String city : cities) { println city BOUNDS_COLL.remove(new BasicDBObject('city', city)) @@ -70,13 +71,13 @@ class CityLatlngBox { int row = 0 int inx = 0 def buffer = [] - while(seedLatBD < cityBoundBD.getMaxLat()) { + while(seedLatGD < cityBoundGD.getMaxLat()) { int col = 0 Bounds boundsBD = null; Bounds boundsQQ = null; Bounds boundsGD = null; - while (seedLngBD < cityBoundBD.getMaxLng()) { + while (seedLngGD < cityBoundGD.getMaxLng()) { double[] espBD = GeoUtils.getLatLon(seedLngBD, seedLatBD, offset, 90) // 右侧OFFSET处的经纬度 double[] wnpBD = GeoUtils.getLatLon(seedLngBD, seedLatBD, offset, 360) // 上方OFFSET处的经纬度 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index e6b9defe3..920c0d4fd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -39,7 +39,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ println city +" "+rows.size() } def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] - FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.dat",headers, "utf-8", false); + FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat",headers, "utf-8", false); println "txt 生成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy index 08c478e6d..c3affa2bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy @@ -40,8 +40,8 @@ class BigdataDistrictBatch extends BaseBatch{ def rets = [] rets << headers rets.addAll(allDatas) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_district.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_district.dat",headers, "utf-8",false); + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据") + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false); } def createDatas(rows){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy index 8a6e106ca..39a26972b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy @@ -65,9 +65,9 @@ class BigdataPoiBatch extends BaseBatch{ def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"] - def fileName = "/home/goojia/ccb/bigdata_poi.txt" + def fileName = "/home/goojia/ccb/BIGDATA_POI_GJ.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_poi.txt" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_POI_GJ.dat" } FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); println "txt 生成" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 2124be80f..ad48d4304 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -107,8 +107,8 @@ class CcbCommonUtils { alias << dist.name } - if(dist.name.endsWith("弄小区")){ - def name = dist.name.replace("弄小区","弄") + if(dist.name.endsWith("弄小区") || dist.name.endsWith("小区") ){ + def name = dist.name.replace("弄小区","弄").replace("小区","") if(!alias.contains(name)){ alias << name } @@ -123,10 +123,10 @@ class CcbCommonUtils { def rets = new HashSet() def lists = [] for(def d : dbcursor){ - if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){ - //def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") - //if(!rets.contains(key)){ -// rets << key +// if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){ + def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + if(!rets.contains(key)){ + rets << key d.dist_id = dist._id.toString() d.address = dist.address d.master_name = dist.name @@ -136,8 +136,8 @@ class CcbCommonUtils { d.property_life = dist.property_life d.estate_type2 = dist.estate_type2 lists << d - //} - } + } +// } } return lists } @@ -229,7 +229,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name", "海昌欣城") +// q.put("name", "东特金星公馆") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy index f340f72ab..1dbc5cd4d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy @@ -43,7 +43,9 @@ class DistrictDelete { static main(args) { def task = new DistrictDelete() // task.run() - task.gzRun(); +// task.gzRun(); +// task.deleteDistrict() + task.deleteCommPrice() } def run(){ @@ -211,9 +213,9 @@ class DistrictDelete { def gzRun(){ def index = 0 - ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度有问题的小区.xlsx",{d-> - if(d['U'] && d['U'].toString().toDouble().toInteger()==1){ - def id = d['E'] + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\要删除小区 (5).xlsx",{d-> + if(d['G']){ + def id = d['G'] BasicDBObject q=new BasicDBObject('name_cat',1) q.put("m_date", new Date()) DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q)) @@ -223,4 +225,38 @@ class DistrictDelete { },0) } + def deleteDistrict(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d-> + if(d['G']){ + def id = d['G'] + BasicDBObject q=new BasicDBObject('name_cat',1) + q.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q)) + deleteAllRefDistrict(id) + } + },0) + } + + def deleteAllRefDistrict(id){ + def tables = ["bigdata_forsale_gj","avm_evaluation_item_gj","community_base_price_gj","community_poi_gj","community_sale_gj","community_sale_tmp","community_test_price_tmp"] + for(def tableName : tables){ + deleteByCommid(tableName, id) + } + } + + def deleteCommPrice(){ + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\均价删除.xlsx",{d-> + if(d['B']){ + def id = d['B'] + println id + deleteByCommid("community_base_price_gj",id) + } + },0) + } + + def deleteByCommid(tableName, commid){ + def sql = "delete from "+tableName+" where commid='${commid}'"; + sqlInstance.execute(sql) + println "删除:"+tableName+" "+commid + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy index 3f73a32d6..b64fdcd1a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy @@ -17,7 +17,7 @@ class DistrictGuidePriceBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = ["深圳"]//CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys; def rows = [] for (def city : citys) { @@ -34,8 +34,8 @@ class DistrictGuidePriceBatch extends BaseBatch{ } } -// def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"] -// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\district_guied_price.dat", headers, "utf-8", false); + def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_GUIDE_PRICE_GJ.dat", headers, "utf-8", false); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy index 7b40980e1..afae30ba4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy @@ -29,8 +29,8 @@ class EvaluateRateBatch extends BaseBatch{ def rets = [] rets << headers rets.addAll(allDatas) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\evaluate_rate.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.dat",headers, "utf-8", false); + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\EVALUATE_RATE_GJ.xlsx", rets, "数据") + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\EVALUATE_RATE_GJ.dat",headers, "utf-8", false); } def createDatas(def datas){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 4c16c191d..7f6cf3fd0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -25,18 +25,24 @@ class ExportOracleData { // exportCommunityBuileType() // exportBigdataDistrictData() // exportGuidedPrice() -// exportCommunityGuidedPrice() + exportCommunityGuidedPrice() // exportBigdataDeal() // exportThirdPartyPrice() // exportBigdataCommunity() - exportBigdataForsale() +// exportBigdataForsale() // exportAvmEvaluationGJNB() // exportAVMEvaluationItemGJNB() // exportCommunityBasePrice() + +// exportCommunityBasePriceGj() +// exportCityPrice() +// exportRegionPrice() +// exportLastFile() } + def exportEvaluationRateData(){ def sql = "select ID,RATE_TYPE,RATE,RANK,CITY,REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from evaluate_rate_gj"; def allRows = [] @@ -213,7 +219,7 @@ class ExportOracleData { def exportGuidedPrice(){ - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where cityId in (350200,150100)"; + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where dealdate>=TO_DATE('2021-08-01', 'yyyy-MM-dd') and dealdate def data = [] @@ -237,20 +243,20 @@ class ExportOracleData { def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" def dataCodeFileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok" } def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "GUIDED_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202107") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202107") + CcbFtpUtils.ftpUpload(fileName, "202108") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") println "GUIDED_PRICE_GJ 上传Ftp完成......" } def exportCommunityGuidedPrice(){ - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where cityname='厦门' or cityname='呼和浩特'"; + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where cityname='西安'"; def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -273,16 +279,16 @@ class ExportOracleData { def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok" } def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "COMMUNITY_GUIDE_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202108") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + CcbFtpUtils.ftpUpload(fileName, "202107") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202107") println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......" } @@ -726,5 +732,88 @@ class ExportOracleData { println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......" } + def exportCommunityBasePriceGj(){ + + } + + def exportCityPrice(){ + def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where (cityId=350200 or cityId=150100) and year_month>=202108"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_CITY_PRICE_GJ_ADD_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.ok" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "CITY_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, "202108") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + println "CITY_PRICE_GJ 上传Ftp完成......" + } + + def exportRegionPrice(){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where (cityId=350200 or cityId=150100) and year_month>=202108"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.ok" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "DISTRICT_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, "202108") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + println "DISTRICT_PRICE_GJ 上传Ftp完成......" + } + + def exportLastFile(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + + def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + def dataCodeFileName = "/ccb/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curMonth}\\${curDate}.ok" + } + CcbFtpUtils.ftpUpload(dataCodeFileName, "${curMonth}") + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 68eb41ed3..351473f24 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -71,7 +71,7 @@ class GuiDingPriceBatch extends BaseBatch{ } def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.dat", headers, "utf-8", false); + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false); // def rets = [] // rets << headers diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LocationUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LocationUpdate.groovy new file mode 100644 index 000000000..a34e7f2d7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LocationUpdate.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.core.util.GeocodeUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.bson.types.ObjectId + +class LocationUpdate { + + static main(args) { + def task = new LocationUpdate(); + task.run() + } + + def run(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度\\经纬度人工核实结果_0923.xlsx",{d-> + index++ + if(d['C']){ + + println index +" "+d['A']+" "+d['B']+" "+d['C'] + def region = RegionData.formatRegion(d['A'], d['B']) + if(d['H']){ + def status = d['H'].toString().toDouble().toInteger() + if(status==1){ + def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") + if(obj){ + def gmap = GeocodeUtils.getGaodeToBaidu(d['F'].toString().toDouble(), d['G'].toString().toDouble()) + def lng2 = null + def lat2 = null + if(gmap){ + lng2 = (gmap.get("lng")*10000000000).longValue() + lat2 = (gmap.get("lat")*10000000000).longValue() + } + + BasicDBObject q=new BasicDBObject() + q.put("m_date", new Date()) + q.put("lat2", lat2) + q.put("lng2", lng2) + q.put("lng", (d['F'].toString().toDouble()*10000000000).longValue()) + q.put("lat", (d['G'].toString().toDouble()*10000000000).longValue()) + DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q)) + + }else{ + println d['A']+" "+d['B']+" "+d['C']+" 没找到小区" + } + }else{ + def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") + if(obj){ + BasicDBObject q=new BasicDBObject('name_cat',1) + q.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q)) + }else{ + println d['A']+" "+d['B']+" "+d['C']+" 没找到小区" + } + } + + }else{ + def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") + if(obj){ + BasicDBObject q=new BasicDBObject('name_cat',1) + q.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q)) + }else{ + println d['A']+" "+d['B']+" "+d['C']+" 没找到小区" + } + + } + + } + + }) + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index 3c3af2392..26db4158d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -40,8 +40,8 @@ class StagingDistrictBatch extends BaseBatch{ results << headers results.addAll(rows) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING.xlsx", results, "数据") - FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING.dat",headers, "utf-8", false); + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据") + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING_GJ.dat",headers, "utf-8", false); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 18dc9f380..f9ac2e3fc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -3,6 +3,8 @@ package com.fangjia.script.ccbevaluation import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.looplineinfo.Point import com.fangjia.script.util.DbUtils @@ -14,6 +16,11 @@ import com.mongodb.DBCollection import groovy.sql.Sql import org.bson.types.ObjectId +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong import java.util.regex.*; @@ -42,7 +49,7 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw // println city +" "+count //} -//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx" +//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927.xlsx" //def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1) //def dealMap = [:] //for(def d : datas){ @@ -59,7 +66,7 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw -def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0903.xlsx" +//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0903.xlsx" //def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) //def dealMap = [:] @@ -83,7 +90,7 @@ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() // if (city) { // def cityid = cityCodeDatas.get(city) // def total = findCount(cityid, sqlInstance) -// d['M'] = total +// d['O'] = total // } //},0) @@ -128,13 +135,26 @@ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() // } //},1) -//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx" -def dealMap = findDealCount(sqlInstance) -ExcelUtils.multiThreadingModify(fileName,{d-> - if(d['C']){ - d['K'] = dealMap.get(d['C']) +//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx" +//def dealMap = findDealCount(sqlInstance) +//ExcelUtils.multiThreadingModify(fileName,{d-> +// if(d['A']){ +// d['U'] = dealMap.get(d['A']) +// } +//},0) + + +def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx" +def datas = ExcelUtils.read(fileName, 0, 1) +def dealMap = [:] +for(def d : datas){ + dealMap.put(d['A'], d['B']) +} +ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d-> + if(d['A']){ + d['W'] = dealMap.get(d['A']) } -},2) +},0) //def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" @@ -170,7 +190,7 @@ ExcelUtils.multiThreadingModify(fileName,{d-> def findDealCount(sqlInstance){ - def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('2021-06-01','yyyy-mm-dd') and dealdate @@ -204,7 +224,7 @@ def findCount(cityid, sqlInstance){ } def findFiledCount(cityid, field, sqlInstance){ - def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and "+field+" is not null"; + def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null"; def count = 0 sqlInstance.eachRow(sql,{r-> count = r.count @@ -353,3 +373,5 @@ def getStreetInfo(def city, def glocation){ + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy new file mode 100644 index 000000000..c0833369a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy @@ -0,0 +1,81 @@ +package com.fangjia.script.ccbevaluation + +import groovy.sql.Sql + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class TestRepeat { + + ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new TestRepeat() + task.run() + } + def run(){ + + } + + + def checkUpdateCount(){ + def newHouses = readTxt("C:\\Users\\gaojuan\\Desktop\\BIGDATA_FORSALE_GJ.dat") + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(newHouses.size()) + println " 房源量:"+newHouses.size() + + def retMap = new ConcurrentHashMap() + for(def i=0;i + obj = r + }); + return obj + } + + + def readTxt(def file){ + def lines = org.apache.commons.io.FileUtils.readLines(new File(file)) + def rets = []; + for(line in lines) { + def arrs = line.split("|@|") + def map = [:] + map.houseId = arrs[1] + map.commId = arrs[6] + map.pdate = arrs[10] + rets << map + } + return rets; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 8fadf19e0..1e8ca6a82 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -20,9 +20,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202107//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202105 - def endDate = 202107 + def sDate= 202108//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202106 + def endDate = 202108 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -53,13 +53,13 @@ class ThirdPartyPriceBatch extends BaseBatch { - def fileName = "/home/goojia/ccb/third_party_price.dat" + def fileName = "/home/goojia/ccb/THIRD_PARTY_PRICE_GJ.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.dat" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\THIRD_PARTY_PRICE_GJ.dat" } - def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); + def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); } @@ -67,6 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){ def rets = [] for(def d : dists){ + def third = thirdPriceMap.get(d.region+"##"+d.name) if(third){ def data = distCodeMap.get(d.region) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index c2378636d..a35a99a4e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -1,12 +1,22 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import groovy.sql.Sql +import org.bson.types.ObjectId + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong /*** * 建行成交抓取统计 @@ -21,23 +31,34 @@ class TradeStatLog { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + static main(args) { def task = new TradeStatLog() - task.run() +// task.run() // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) +// task.averageStat() + task.exportTrade() +// task.deleteDistrict() + } def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","5月总量","6月总量","7月总量","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","6月总量","7月总量","8月总量","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] - def startDate = "2021-05-01" - def date = "2021-06-01" - def endDate = "2021-07-01" - def newDate = "2021-08-01" + //def startDate = "2021-05-01" + def startDate = "2021-06-01" + def date = "2021-07-01" + def endDate = "2021-08-01" + def newDate = "2021-09-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) @@ -64,7 +85,7 @@ class TradeStatLog { ret << lastMonthCount ret << curMonthCount ret << nearMonthCount - if(lastMonthCount){ + if(curMonthCount){ def a = (nearMonthCount-curMonthCount)*100/curMonthCount def pre = DataHandleUtils.getDoubleRound(2, a) ret << pre @@ -87,7 +108,7 @@ class TradeStatLog { } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx", rets, "数据") println "表格生成完成" } @@ -103,7 +124,7 @@ class TradeStatLog { rets << [city, districts.size(), m.total, m.count1, m.count2, m.count3, m.count4, m.count5, m.count6, m.count7, m.count8, m.count9, m.count10, m.count11, m.count12, m.count13, m.count14] println rets.get(rets.size()-1) } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\第三方均价统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\第三方均价统计0927.xlsx", rets, "数据") println "表格生成完成" } @@ -129,7 +150,7 @@ class TradeStatLog { def getThirdPartyPricePrecent(def city){ - def sql = "SELECT * FROM third_party_price WHERE cityname = '"+city+"'"; + def sql = "SELECT * FROM third_party_price_gj WHERE cityname = '"+city+"' and year_month=202108"; def count0 = 0 def count1 = 0 @@ -299,5 +320,333 @@ class TradeStatLog { } + def averageStat(){ + def citys = ["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"] + + def map = [:] + for(def city : citys){ + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]); + def rets = findThirdPriceByDist(districts) + map.put(city, rets.size()) + map.put(city+"-count", districts.size()) + println districts.size()+" "+ rets.size() + } + println map + + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\估值测试数据覆盖率占比.xlsx",{d-> + def city = d['A'] + def count = map.get(city) + def distCount = map.get(city+"-count") + d['H'] = count + d['J'] = distCount + + def pre = DataHandleUtils.getDoubleRound(2, count*100.0/distCount) + d['I'] = pre + },0) + } + + + def findThirdPriceByDist(districts){ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + for(def i=0;iendPrice){ +// reason = "价格过滤" +// } + if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){ + reason = "小区过滤" + } + + if(!reason){ + def key=t.city+"##"+dist.region+"##"+dist.name+"##"+t.area+"##"+t.avg_price+"##"+DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd") + if(details.contains(key)){ + reason = "数据重复" + } + } + } + + + def tradeDate = DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd") + //rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason, t.estate_type] + rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason] + } + } + println city +" "+rets.size() + } + + ExcelUtils.write("d:\\数据\\成交未入库记录(7月全).xlsx", rets,"数据") + } + + def deleteDistrict(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d-> + if(d['G']){ + def id = d['G'] + BasicDBObject q=new BasicDBObject('name_cat',1) + q.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q)) + deleteAllRefDistrict(id) + } + },0) + } + + def deleteAllRefDistrict(id){ + def tables = ["bigdata_community_gj"] + for(def tableName : tables){ + deleteByCommid(tableName, id) + } + } + + def deleteByCommid(tableName, commid){ + def sql = "delete from "+tableName+" where commid='${commid}'"; + sqlInstance.execute(sql) + println "删除:"+tableName+" "+commid + } + + def getYouDaData() { + def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd")) + def endDate = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd") + def q = new BasicDBObject() +// q.put("premisesName","共富一村") + q.put("contractTime", new BasicDBObject('$gte', startDate));//.append('$lt', endDate) + def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000) + def trades = [] + for(def db : dbcursor){ + def map = [:] + map._id=db._id.toString() + map.city = "上海" + map.region = db.region + if(db.districtName && db.districtName!="null"){ + map.district_name=db.districtName + }else if(db.address){ + map.district_name=db.address + } + + map.area = db.area + map.trade_date = db.contractTime + map.avg_price= db.price + map.estate_type = db.houseType + trades << map + } + return trades + } + + def findDealIds(city){ + def sql = "select * from bigdata_deal_gj where dealdate>=to_date('2021-07-01','yyyy-mm-dd') and dealdate + if(!ids.contains(r.dealid)){ + ids << r.dealid + + def region = RegionData.formatRegion(r.cityname, r.distname) + def key=r.cityname+"##"+region+"##"+r.commname+"##"+r.buildarea.toDouble()+"##"+r.dealunitprice+"##"+DateUtils.formatDateByPattern(r.dealdate,"yyyy-MM-dd") + if(!detailInfos.contains(key)){ + detailInfos << key + } + } + }); + def rMap = [:] + rMap.ids = ids + rMap.detail = detailInfos + return rMap + } + + + public static def findTradesDetail(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) +// q.put("district_name","百合花园") + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcursor = DEAL_PRICE_COLL.find(q) + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets + } + + def getDistrictStatByCity(city, weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def dbcursor = DIST_SELL_STAT.find(q) + def map = new ConcurrentHashMap() + for(def d : dbcursor){ + map.put(d.region+"@"+d.district_name, d.avg_price) + } + return map + } + + def getAllDistrict(city){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name", "alias"]) + def map = [:] + for(def d : dists){ + map.put(d.city+"##"+d.region+"##"+d.name, d._id.toString()) + def alias = d.alias + if(alias){ + for(def name : alias){ + map.put(d.city+"##"+d.region+"##"+name, d._id.toString()) + } + } + } + return map + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy index a47028b79..cb189d420 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy @@ -1,13 +1,18 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.ExcelUtil import com.fangjia.core.util.HttpRequest import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import net.sf.json.JSONObject +import org.bson.types.ObjectId import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet @@ -16,6 +21,7 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern class UpdateHouseRegion { @@ -37,11 +43,89 @@ class UpdateHouseRegion { static main(args) { def task = new UpdateHouseRegion() - task.updateSeawaterRegion() +// task.updateSeawaterRegion() // task.updateDealRegion() // task.updateThirdPartyRegion() + task.updateNoMatchDealRegion() +// task.addAlias() +// task.addFenQi() } + def addFenQi(){ + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district",["city","region","name","alias"]) + addFenQiAlias(dists) + println city +" 处理完成" + } + } + + def addFenQiAlias(dists){ + for(def dist : dists){ + def s1 = DataUtil.match(dist.name, Pattern.compile("[一|二|三|四|五|六|七|八|九|十|十一|十二|十三|十四|十五|十六|十七|十八|\\w+](期|区)\$")) + if(s1){ + def newName = dist.name.replace("Ⅰ","1").replace("Ⅱ","2").replace("A区","a区").replace("B区","b区").replace("C区","c区").replace("D区","d区").replace("E区","e区").replace("F区","f区").replace("一期","1期").replace("二期","2期").replace("三期","3期").replace("四期","4期").replace("五期","5期").replace("六期","6期").replace("七期","7期").replace("八期","8期").replace("九期","9期").replace("十期","10期").replace("十一期","11期").replace("十二期","12期") + if(newName!=dist.name){ + def alias = dist.alias + if(!alias){ + alias = [] + } + if(!alias.contains(newName)){ + alias << newName + def udata = new BasicDBObject() + udata.put("alias", alias) + DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata)) + println dist.city +" "+dist.name+" "+dist.alias+" 新:"+alias + } + } + } + } + } + + def addAlias(){ + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district",["city","region","name","alias"]) + addDistricts(dists) + println city +" 处理完成" + } + } + + def addDistricts(dists){ + for(def dist : dists){ + def alias = dist.alias + if(alias){ + def oldCount = alias.size() + def newAlias = [] + for(def aliasName : alias){ + if(aliasName.contains("·") || aliasName.contains(".")){ + def newName = aliasName.replace("·","").replace(".","") + newAlias << aliasName + if(!newAlias.contains(newName)){ + if(newName!=dist.name){ + newAlias << newName + } + } + }else{ + if(!newAlias.contains(aliasName)) { + newAlias << aliasName + } + } + } + def newCount = newAlias.size() + if(oldCount && alias!=newAlias){ + def udata = new BasicDBObject() + udata.put("alias", newAlias) + DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata)) + println dist.city +" "+dist.name+" "+dist.alias+" 新:"+newAlias + } + + + } + } + } + + def updateThirdPartyRegion(){ def city = "东莞" def rets = findThirdPartyPrice(city) @@ -121,6 +205,32 @@ class UpdateHouseRegion { latch.await() } + def updateNoMatchDealRegion(){ + ExcelUtils.multiThreadingModify("D:\\数据\\成交未入库记录(7月全).xlsx",{d-> + if(!d['J']){ + def city = d['B'] + def region = d['C'] + def name = d['D'] + def rets = CommonDatautils.findDistrictsByName(city, name, "district") + if (!rets) { + rets = CommonDatautils.findDistrictByAlias(city, null, name, "district") + } + + if (rets.size() == 1) { + def dist = rets[0] + if (region != dist.region) { + def udata = new BasicDBObject() + udata.put("region", dist.region) + DEAL_PRICE_COLL.update(new BasicDBObject('_id', new ObjectId(d['A'])), new BasicDBObject('$set', udata)) + d['J'] = "原数据区域修改("+dist.region+")" + println d['A']+" "+region +" "+dist.region + } + } + } + + },0) + } + def updateSeawaterRegion(){ def city = "东莞" def rets = findHouseSource(city, 20190701, 20211001) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index c61861c99..ad5a8babc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202107.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202109.csv") def count0 = 0 def count1 = 0 def count2 = 0 @@ -135,7 +135,7 @@ rets << ["9 def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM") ExcelUtils.write("d:\\数据\\福州\\多方均价对比统计数据"+curMonth+".xlsx",rets,"统计") def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )" - +println s def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60" DingDingMessageUtil.sendTextMessage(access_token, s) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenUserBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenUserBatch.groovy index c322efc25..45c2f60a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenUserBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenUserBatch.groovy @@ -45,18 +45,20 @@ class OpenUserBatch { } def addShenZhenFjtUser(){ - def datas = ExcelUtils.read("D:\\数据\\农行\\肇庆农行支行房价通账号.xlsx",0,1) + def datas = ExcelUtils.read("D:\\数据\\渤海\\开通线上评估客户经理名单.xlsx",0,1) def i = 0; for(def data : datas){ i++; - def id = 252580 + i; - def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+data[2]+"', '"+data[3]+"', '16', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2021-12-10 00:00:00', '0', '', NULL)"; + def id = 252608 + i; + BigDecimal bd = new BigDecimal(data[3].toString()); + String mobile = bd.toPlainString(); + def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)"; onLineSqlInstance.execute(sql); - def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[1]+"', '"+data[0]+"', NULL, NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '王辰岑')"; + def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '姚静梦')"; onLineSqlInstance.execute(sql2) - def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'肇庆','肇庆:')" + def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'上海','上海:')" onLineSqlInstance.execute(sql3) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbXykData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbXykData.groovy new file mode 100644 index 000000000..c90064d97 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbXykData.groovy @@ -0,0 +1,18 @@ +package com.fangjia.script.gj.ccb + +import com.fangjia.script.commData.CommonDatautils + +class CcbXykData { + static main(args) { + def task = new CcbXykData() + task.run() + } + + def run(){ + def districts = CommonDatautils.findAllDistrictByCity("上海","district",["city","region","name","estate_charge"]) + for(def d : districts){ + + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index e7232a3c8..7d7f43a1c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -9,6 +9,14 @@ class FileUtils { // file.delete(); // } + if (!file.getParentFile().exists()) { + boolean result = file.getParentFile().mkdirs(); + if (!result) { + System.out.println("创建目录失败"); + } + } + + OutputStream out = new FileOutputStream(fileName, isAppend); def sb = new StringBuffer(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy new file mode 100644 index 000000000..0ad5dbfcf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy @@ -0,0 +1,64 @@ +package com.fangjia.script.gj.location + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.NameProcess +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet + +class DistrictLocationUpdate { + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static main(args) { + def task = new DistrictLocationUpdate() + task.run() + } + def run(){ + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"]) + + + updateLocation(city, dists); + + } + + } + + def queryPoiMap(def city){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1") + sqlBuffer.append(" and cityname = '" + city + "'") + sqlBuffer.append(" and source = '高德'") + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def retMap = [:] + while (rs.next()) { + def map = [:] + map.city = rs.getString("cityname") + map.region = rs.getString("adname") + map.name = rs.getString("name") + map.address = rs.getString("address") + map.type = rs.getString("type") + def location = rs.getObject("location") + if(location){ + map.lng = location.x + map.lat = location.y + } + + def name = NameProcess.nameStandard(map.city, map.region, map.name) + if(name){ + retMap.put(map.region+"##"+name, map) + } + } + return retMap; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy index 4313382e5..d60d5f02c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy @@ -521,6 +521,10 @@ class GaoDeUtils { return KeyList.get(count); } + static void main(def a){ + println search("如东县大豫镇豫东村委会","南通") + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MatchDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MatchDistrict.groovy new file mode 100644 index 000000000..7d52d6a3b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MatchDistrict.groovy @@ -0,0 +1,45 @@ +package com.fangjia.script.gj.test + +import com.fangjia.core.RegionData +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject + +class MatchDistrict { + static main(args) { + def task = new MatchDistrict() + task.run() + } + def run(){ + def filePath = "D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx" + def city = "深圳" + ExcelUtils.multiThreadingModify(filePath, { data -> + if(data['A']){ + def region = RegionData.formatRegion(city, data['A']) + if(region){ + def rMap = SeaweedDataMatchProcess.doMatch(city, data['A'], data['C'], null); + if(rMap){ + data['E'] = rMap.city + data['F'] = rMap.region + data['G'] = rMap.name + } + } + } + }, 0); + } +} + +//{ +// "_id" : ObjectId("607963dc5bb87e2a18f4dde6"), +// "guiding_type" : NumberInt(0), +// "estate_type" : "普通住宅", +// "house_type" : NumberInt(0), +// "c_date" : ISODate("2021-04-16T10:15:56.651+0000"), +// "guiding_person" : "深圳房管局", +// "guiding_date" : ISODate("2021-04-10T16:00:00.000+0000"), +// "category" : "district", +// "guiding_value" : NumberInt(48300), +// "name" : "中粮云景国际", +// "city" : "深圳", +// "region" : "光明" +//} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy index aa345e412..91498750a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -1,33 +1,55 @@ package com.fangjia.script.gj.test +import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.Mongo +import groovy.sql.Sql import java.text.SimpleDateFormat //SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); //Date date = new Date(1617211269868); //print dateFormat.format(date); -DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info'); -def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx" -ExcelUtils.modify(path,{d-> - def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"]) - d['G'] = districts.size() +//DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info'); +//def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx" +//ExcelUtils.modify(path,{d-> +// def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"]) +// d['G'] = districts.size() +// +// def q = new BasicDBObject() +// q.put("city", d['B']) +// q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08")) +// def obj = CITY_INFO_CONN.findOne(q); +// if(obj){ +// d['H'] = obj.price_coverage*100 +// } +//}) - def q = new BasicDBObject() - q.put("city", d['B']) - q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08")) - def obj = CITY_INFO_CONN.findOne(q); - if(obj){ - d['H'] = obj.price_coverage*100 +def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver"); + +def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'"; +def countMap = [:] +onLineSqlInstance.eachRow(sql,{r-> + def addTime = r.visit_time + def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM") + def ips = countMap.get(month) + if(!ips){ + ips = new HashSet(); + countMap.put(month, ips) } + ips << r.ip +}); + +countMap.each {k,v-> + println k+" "+v.size() +} + + - -}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/test/RegionalismCodeStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/test/RegionalismCodeStat.groovy new file mode 100644 index 000000000..7f49b80b0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/test/RegionalismCodeStat.groovy @@ -0,0 +1,58 @@ +package com.fangjia.script.test + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class RegionalismCodeStat { + + DBCollection REGIONALISM_CODE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "regionalism_code", "192.168.0.235", 27018) + + + static main(args) { + def task = new RegionalismCodeStat() + task.run() + } + + def run(){ + def provinces = ["上海市","北京市","天津市","重庆市","福建省","贵州省","湖南省","广东省","湖北省","安徽省","四川省","江苏省","吉林省","河北省","河南省","海南省","江西省","云南省","甘肃省","山东省","陕西省","浙江省","青海省","辽宁省","黑龙江省","山西省","西藏自治区","新疆维吾尔自治区","宁夏回族自治区","内蒙古自治区","广西壮族自治区"] +// def provinces = ["宁夏回族自治区"] + provinces.each {v-> + def sets = findCount(v) + println v +" "+ sets.size() + } + + } + + def findCount(v){ + def q = new BasicDBObject() + q.put("province", v) + def dbcursor = REGIONALISM_CODE_COLL.find(q).batchSize(5000) + def set = new HashSet() + for(def obj : dbcursor){ +// if(obj.city=="省直辖县级行政区划" || obj.city=="自治区直辖县级行政区划"){ +// //set << obj.region +// println "【"+obj.region+"】 过滤" +// continue +// }else{ +// if(obj.region && !set.contains(obj.region)){ +// set << obj.region +// println "【"+obj.region+"】 add" +// } +// } + + if(obj.juweihui){ + def k = obj.street+"##"+obj.juweihui + if(!set.contains(k)){ + set << k +// println "【"+k+"】 add" + } + } + } + +// set.each {k-> +// println k +// } + return set; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 0ba834ad5..7ae95777c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -17,7 +17,7 @@ class SynchDistrictPrice { def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); // def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产"] - def sourceMap = ["zhuge.com":"诸葛找房"] + def sourceMap = ["cityhouse.cn":"城市房产"]//["zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] static main(args) { def task = new SynchDistrictPrice() task.run() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 10283440a..e299c60f6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"] for(def source :sources){ - def count1 = findTotalCount(source, 202107); - def count2 = findTotalCount(source, 202108); + def count1 = findTotalCount(source, 202108); + def count2 = findTotalCount(source, 202109); println source +" "+count1 +" "+count2 } } From 0f889c715b46cd101424a30dd9535af5c726bdb6 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 13 Oct 2021 17:47:22 +0800 Subject: [PATCH 36/80] =?UTF-8?q?8=E6=9C=88=E6=88=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 7ee409fc9..d69cdaccf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -29,7 +29,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def curDate = 20210701//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); + def curDate = 20210801//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] From a3b3c31274cb3212350c8fadadc40ad9ec0bd0a5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 13 Oct 2021 18:51:13 +0800 Subject: [PATCH 37/80] =?UTF-8?q?7=E6=9C=88=E6=88=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index d69cdaccf..7ee409fc9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -29,7 +29,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def curDate = 20210801//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); + def curDate = 20210701//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] From 7b0c0e626afa5f59c9bf1a2a9af7e908cdf2e2a3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 13 Oct 2021 18:58:25 +0800 Subject: [PATCH 38/80] =?UTF-8?q?=E5=AF=BC=E5=87=BA=E7=AC=AC=E4=B8=89?= =?UTF-8?q?=E6=96=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ExportOracleData.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 7f6cf3fd0..2a3c7fa09 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -25,9 +25,9 @@ class ExportOracleData { // exportCommunityBuileType() // exportBigdataDistrictData() // exportGuidedPrice() - exportCommunityGuidedPrice() +// exportCommunityGuidedPrice() // exportBigdataDeal() -// exportThirdPartyPrice() + exportThirdPartyPrice() // exportBigdataCommunity() // exportBigdataForsale() @@ -389,7 +389,7 @@ class ExportOracleData { def exportThirdPartyPrice(){ def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, \n" + "TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE\n" + - "from third_party_price_gj where cityId in (350200,150100) and year_month<202108"; + "from third_party_price_gj"; def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -432,9 +432,9 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202107") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202107") - println "THIRD_PARTY_PRICE_GJ 上传Ftp完成......" +// CcbFtpUtils.ftpUpload(fileName, "202107") +// CcbFtpUtils.ftpUpload(dataCodeFileName, "202107") +// println "THIRD_PARTY_PRICE_GJ 上传Ftp完成......" } def exportBigdataCommunity(){ From b08475984a9b0182e75157865cd0ed3015375598 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 13 Oct 2021 19:03:10 +0800 Subject: [PATCH 39/80] =?UTF-8?q?=E7=BE=8E=E9=A3=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/GaoDeMapCrawl.groovy | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index 520a1bdd0..9221c8fea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -56,8 +56,12 @@ class GaoDeMapCrawl extends BaseBatch{ // def prefix = "road_" // def categorys = ["190301"]//道路相关 - def prefix = "village_" - def categorys = ["190108","190109"]//村庄 +// def prefix = "village_" +// def categorys = ["190108","190109"]//村庄 + + + def prefix = "food_" + def categorys = ["050000"] // def prefix = "company_" // def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"] @@ -119,8 +123,8 @@ class GaoDeMapCrawl extends BaseBatch{ def run(){//,"西安","重庆" //def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] - def citys = ["上海","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"] - def allcitys = CommonUtils.findAllCity() + def citys = ["南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"] + def allcitys = ["上海"]//CommonUtils.findAllCity() def totalCount = 0; for(def city : allcitys){ From f903ba468ca651354f92b44f1f75ad5314c86c45 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 14 Oct 2021 16:11:18 +0800 Subject: [PATCH 40/80] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=B0=8F=E5=8C=BA=E6=B7=BB=E5=8A=A0name=5Fca?= =?UTF-8?q?t,=E9=98=B2=E4=BF=AE=E6=94=B9=E9=80=A0=E6=88=90name=5Fcat?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkoff/controllers/DistrictController.java | 1 + .../com/fangjia/dal/seaweed/model/District.java | 16 +++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java index 4c6a22236..dd04da3dd 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java @@ -1265,6 +1265,7 @@ public class DistrictController { district.setAddrAlias(new ArrayList(addrAlias)); } district.setStatus(0); + district.setNameCat(0); district.setCategory("district"); district.setPublisherType(11); district.setCreationDate(new Date()); diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java index afba53e15..32e526e32 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java @@ -234,6 +234,8 @@ public class District implements SeaweedEntity { private String mainTypeArea; private String createYear; + + private Integer nameCat; @Column(name = "source_web") public String getSourceWeb() { @@ -1879,9 +1881,13 @@ public class District implements SeaweedEntity { public void setTownship(String township) { this.township = township; } - - - - - + + @Column(name = "name_cat") + public Integer getNameCat() { + return nameCat; + } + + public void setNameCat(Integer nameCat) { + this.nameCat = nameCat; + } } From 619b8ef89d743cdebb057d284d594514d622c96f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 14 Oct 2021 17:13:42 +0800 Subject: [PATCH 41/80] =?UTF-8?q?=E6=9D=AD=E5=B7=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- .../com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 2 +- .../src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index ab705a2a5..4e0d0d1f5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -66,7 +66,7 @@ class BigdataForsaleBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = ["杭州"]//CcbCommonUtils.citys; def index = 0 def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index ad48d4304..9c1979965 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -122,6 +122,7 @@ class CcbCommonUtils { def total = dbcursor.count() def rets = new HashSet() def lists = [] + println dist.name +" "+total for(def d : dbcursor){ // if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){ def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") @@ -229,7 +230,6 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name", "东特金星公馆") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index 91cb7d57e..c30a1353a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -80,7 +80,7 @@ class ElasticUtils { @Override RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { requestConfigBuilder.setConnectionRequestTimeout(1000 * 60); - requestConfigBuilder.setSocketTimeout(1000 * 60); + requestConfigBuilder.setSocketTimeout(1000 * 180); return requestConfigBuilder; } }) From afb2fe666685a829dfdfc914f3b5015ddc8cc96f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 14 Oct 2021 18:35:43 +0800 Subject: [PATCH 42/80] =?UTF-8?q?=E6=89=80=E6=9C=89=E6=88=BF=E6=BA=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/BigdataForsaleBatch.groovy | 7 +++++-- .../com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 4e0d0d1f5..f4f1a7761 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -55,7 +55,7 @@ class BigdataForsaleBatch extends BaseBatch{ } public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -66,7 +66,7 @@ class BigdataForsaleBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = ["杭州"]//CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys; def index = 0 def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ @@ -283,6 +283,7 @@ class BigdataForsaleBatch extends BaseBatch{ def rets = [] def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + if(houses){ for (int k=0;k Date: Thu, 14 Oct 2021 18:39:02 +0800 Subject: [PATCH 43/80] =?UTF-8?q?=E6=88=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/BigdataDealBatch.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 7ee409fc9..7fbedb2fc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -420,7 +420,8 @@ class BigdataDealBatch extends BaseBatch{ // startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // } - def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) +// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd")) for(def i=0;i Date: Mon, 18 Oct 2021 15:31:04 +0800 Subject: [PATCH 44/80] =?UTF-8?q?58=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 99cdc2a86..e3c688a7b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -191,6 +191,7 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); + def citys = ["成都","南京","福州","厦门","泉州","东莞","广州","广州","广州","西安","沈阳","大连","昆明","太原","郑州","乌鲁木齐","烟台","绵阳","邯郸","潍坊","嘉兴","惠州","珠海","海口"] cityList.each{city, str-> for(def i=0;i<1;i++){ // def city = "福州" From 5ecd1713efd3494db622ee72c675c6db738fc512 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 18 Oct 2021 15:40:02 +0800 Subject: [PATCH 45/80] =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index e3c688a7b..849fee72e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -195,16 +195,18 @@ class Esf58Crawl extends BaseBatch{ cityList.each{city, str-> for(def i=0;i<1;i++){ // def city = "福州" - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); - if(districts){ - def arrs = str.split("\\|") - def url = "https://"+arrs[0]+".58.com/xiaoqu/" - distMap.put(city, districts) + if(citys.contains(city)){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); + if(districts){ + def arrs = str.split("\\|") + def url = "https://"+arrs[0]+".58.com/xiaoqu/" + distMap.put(city, districts) - addCrawlKeys(city); - addDetailContent(city, url); + addCrawlKeys(city); + addDetailContent(city, url); // Thread.sleep(1000*60*10) + } } } } From 3315ec5d3654097dc31e4a46bcf2a5963f96aec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Mon, 18 Oct 2021 15:53:33 +0800 Subject: [PATCH 46/80] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=B3=BB=E7=BB=9Fbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddressController.java | 2 +- .../fangjia/script/wenyan/temp/Price.groovy | 44 +++++++++++++++---- .../fangjia/script/wenyan/temp/QinLin.groovy | 6 +-- .../wenyan/temp/SiChuanJingCheng.groovy | 4 +- .../script/wenyan/temp/Test1122.groovy | 25 ++++++----- .../sql/EvaluationRateServiceImpl.java | 1 - 6 files changed, 54 insertions(+), 28 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 953611a47..708786087 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -2773,7 +2773,7 @@ public class AddressController { if(json.getInt("code")==200){ JSONObject resultJson = json.getJSONObject("result"); JSONObject searchAddress = resultJson.getJSONObject("searchAddress"); - if(resultJson.getInt("credit")>0){ + if(resultJson.getDouble("credit")>0){ District district = districtService.findByNameAndAlias(resultJson.getString("city"), resultJson.getString("district"), resultJson.getString("name"), "district"); if(district != null){ resultMap.put("riskLevel","A"); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Price.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Price.groovy index b245cfec6..a60bd7b2b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Price.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Price.groovy @@ -1,16 +1,42 @@ package com.fangjia.script.wenyan.temp - +import com.fangjia.core.RegionData +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.common.ServerAreaData +import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject +import net.sf.json.JSONObject -def data = [["城市", "区域", "小区名", "别名", "地址", "地址别名"]] -["ID", "北京", "重庆", "哈尔滨"].each {def city-> - DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(city)).each {BasicDBObject district-> - def item = [district._id.toString(), district.city, district.region, district.name, district.alias?.join(","), district.address, district.addr_alias?.join(",")] - println(item) - data << item +//city_sell_stat = MongoUtils.getDbCollection("fangjia_stat", "city_sell_stat") +//data = [["城市", "202101", "202106"]] +//CommonUtils.findAllCity().each {def city-> +// BasicDBObject serverArea = ServerAreaData.getServerAreaByDisplay(city) +// if(!serverArea.ranking || !serverArea.ranking.toString().contains("县级行政单位")){ +// BasicDBObject pre = city_sell_stat.findOne(new BasicDBObject("city", city).append("date", 202101)) +// BasicDBObject now = city_sell_stat.findOne(new BasicDBObject("city", city).append("date", 202106)) +// println([serverArea.display, pre?.avg_price, now?.avg_price]) +// data << [serverArea.display, pre?.avg_price, now?.avg_price] +// } +//} +//ExcelUtils.write("D:\\城市均价数据.xlsx", data, "城市均价") + +//ExcelUtils.multiThreadingModify("C:\\Users\\wenyan\\Downloads\\经纬度错误数据修正.xlsx",{List item-> +// String city = item["B"] +// String region = RegionData.formatRegion(city, item["D"]) +// String sql = "select count(1) as count from latest_seawater where cat='sell' and city='${item["B"]}' and region='${region}' and district_name='${item["F"]}'" +// ElasticUtils.executeSqlQuery(sql,{Map result-> +// item["P"] = result.get("count") +// }) +// println(item) +//}) +ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\装修案例.xlsx",{List item-> + JSONObject json = OpenApiClient.addressMatch("西安", item["H"], "property", false, item["C"]) + if(json.name){ + item["I"] = json.district + item["J"] = json.name } -} -ExcelUtils.write("D:\\待抓取小区.xlsx", data, "数据") \ No newline at end of file +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index ca851bcb7..785addef5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20210501", "yyyyMMdd") -def endDate = DateUtil.parseDate("20210601", "yyyyMMdd") +def startDate = DateUtil.parseDate("20210901", "yyyyMMdd") +def endDate = DateUtil.parseDate("20211001", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -22,7 +22,7 @@ while (startDate.before(endDate)) { calendar.add(Calendar.DATE, 1) def nextDate = calendar.getTime() count++ - OPEN_URI_LOG.find(and(eq("user_id", "1007757"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> + OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> def result = JSONObject.fromObject(doc.res_map.result) if (!hashSet.contains(result)) { def item = ["60a3202c0b42a00001aad385", "qlkjgx", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy index 82cf47113..8243bde22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20210601", "yyyyMMdd") -def endDate = DateUtil.parseDate("20210701", "yyyyMMdd") +def startDate = DateUtil.parseDate("20210901", "yyyyMMdd") +def endDate = DateUtil.parseDate("20211001", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test1122.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test1122.groovy index 6909f23a1..ec3f9affa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test1122.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test1122.groovy @@ -261,15 +261,16 @@ import org.dom4j.io.SAXReader // data << [dbo.parameter_map.city, dbo.parameter_map.address, dbo.res_map.result.province, dbo.res_map.result.city, dbo.res_map.result.district, dbo.res_map.result.developmentZone, dbo.res_map.result.township, dbo.res_map.result.street,dbo.res_map.result.streetNumber,dbo.res_map.result.name,dbo.res_map.result.buildingNumber,dbo.res_map.result.unitNumber,dbo.res_map.result.roomNumber,dbo.res_map.result.formattedAddress] //} //ExcelUtils.write("D:\\建行测试地址.xlsx", data, "结果") -DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com") -def query = new BasicDBObject() -query.cat = "region" -query.city= "上海" -query.weekend = Date.parse("yyyyMMdd", "20210228") -def doc = [] -def prices = [] -SQMETRE_RENT_COLL.find(query, new BasicDBObject("region", 1).append("avg_price", 1)).each { - doc.add(new BasicDBObject("name", it.region+"区").append("value", it.avg_price)) - prices << it.avg_price -} -println(JSONArray.fromObject(doc).toString()) \ No newline at end of file +//DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com") +//def query = new BasicDBObject() +//query.cat = "region" +//query.city= "上海" +//query.weekend = Date.parse("yyyyMMdd", "20210228") +//def doc = [] +//def prices = [] +//SQMETRE_RENT_COLL.find(query, new BasicDBObject("region", 1).append("avg_price", 1)).each { +// doc.add(new BasicDBObject("name", it.region+"区").append("value", it.avg_price)) +// prices << it.avg_price +//} +//println(JSONArray.fromObject(doc).toString()) +DbUtils.seaweed.remove(new BasicDBObject("_id", DbUtils.seaweed.findById("6103bf1c64baf60001e7e1d0")._id)) \ No newline at end of file diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/sql/EvaluationRateServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/sql/EvaluationRateServiceImpl.java index 3310cce53..4c6147db6 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/sql/EvaluationRateServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/sql/EvaluationRateServiceImpl.java @@ -912,7 +912,6 @@ public class EvaluationRateServiceImpl implements EvaluationRateService { evaluatePrice = (long) (evaluatePrice * houseFloorRate); } }else{ - int mid = evaluation.getHeight()/2; double num = getFloorRate(mid, flag); evaluatePrice = (long) (evaluatePrice * num * houseFloorRate); From 62eed98ff3ce60078c80d3389cbb2634c54e5ba3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 20 Oct 2021 10:34:24 +0800 Subject: [PATCH 47/80] =?UTF-8?q?=E5=AF=BC=E6=88=BF=E6=BA=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/CcbCommonUtils.groovy | 49 ++- .../ccbevaluation/ExportOracleData.groovy | 312 ++++++++---------- .../script/commData/CommonDatautils.groovy | 23 ++ .../fangjia/script/gj/ccb/CcbCodeUtils.groovy | 93 +++++- .../fangjia/script/jiangnanblank/SFTP.groovy | 52 ++- 5 files changed, 338 insertions(+), 191 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 4ea74925a..0bcb9d1a7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -14,7 +14,7 @@ import groovy.sql.Sql class CcbCommonUtils { // - public static def citys = ["扬州", "上海","北京","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] + public static def citys = ["扬州", "上海","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] // public static def citys = ["武汉"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] @@ -224,6 +224,53 @@ class CcbCommonUtils { return dataStoreConnection } + public static def findAllDistrictIdsByOracle(def cityid){ + def sql = "select commid from bigdata_community_gj where cityId=${cityid} and status=0"; + def set = new HashSet() + sqlInstance.eachRow(sql,{r-> + set << r.COMMID + }); + return set + } + + public static def findAllDistrictsByOracle(def cityid){ + def sql = "select CITYID,COMMID,COMMNAME,COMMALIAS,DISTID,STREET,STREETNO,ADDRESS,LANDAREA,BUILDAREA,COMMTYPE,IFNEW,GREENINGRATE,PLOTRATIO,PARKINGCNT,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,WORKDATE,ZTS,LONGITUDE,LATITUDE from bigdata_community_gj where cityId=${cityid} and status=0"; + def rows = [] + try{ + sqlInstance.eachRow(sql,{r-> + def map = [:] + map.cityid = r.CITYID + map.commid = r.COMMID + map.commname = r.COMMNAME + map.distid = r.DISTID + map.street = r.STREET + map.streetno = r.STREETNO + map.address = r.ADDRESS + map.landarea = r.LANDAREA + map.buildarea = r.BUILDAREA + map.commtype = r.COMMTYPE + map.ifnew = r.IFNEW + map.greeningrate = r.GREENINGRATE + map.plotratio = r.PLOTRATIO + map.parkingcnt = r.PARKINGCNT + map.introduction = "" + map.remark = r.REMARK + map.createtime = r.CREATETIME + map.updatetime = r.UPDATETIME + map.sourcecode = r.SOURCECODE + map.workdate = r.WORKDATE + map.zts = r.ZTS + map.lng = r.LONGITUDE + map.lat = r.LATITUDE + rows << map + }); + }catch(Exception e){ + findAllDistrictsByOracle(cityid) + } + + return rows + } + public static def findAllDistrictByCity(def city, def cat, def fields){ DBObject q = new BasicDBObject(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 2a3c7fa09..5956d4a7d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -24,13 +24,15 @@ class ExportOracleData { // exportCommunityStaging() // exportCommunityBuileType() // exportBigdataDistrictData() +// exportBigdataCommunity() + exportBigdataForsale() + + // exportGuidedPrice() // exportCommunityGuidedPrice() // exportBigdataDeal() - exportThirdPartyPrice() +// exportThirdPartyPrice() -// exportBigdataCommunity() -// exportBigdataForsale() // exportAvmEvaluationGJNB() // exportAVMEvaluationItemGJNB() @@ -60,19 +62,15 @@ class ExportOracleData { allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/output/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" - def dataCodeFileName = "/output/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" } def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202108") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + CcbFtpUtils.ftpUpload(fileName, curDate) println "上传Ftp完成......" } @@ -97,20 +95,17 @@ class ExportOracleData { allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" - def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" } def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "BIGDATA_POI_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202108") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + CcbFtpUtils.ftpUpload(fileName, curDate) println "BIGDATA_POI_GJ 上传Ftp完成......" } @@ -128,20 +123,16 @@ class ExportOracleData { data << r.SOURCECODE allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" - def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" } def headers = []//["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "COMMUNITY_STAGING_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202108") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + CcbFtpUtils.ftpUpload(fileName, curDate) println "COMMUNITY_STAGING_GJ 上传Ftp完成......" } @@ -160,20 +151,16 @@ class ExportOracleData { data << r.WORKDATE allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" - def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" } def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202108") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + CcbFtpUtils.ftpUpload(fileName, curDate) println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......" } @@ -200,20 +187,16 @@ class ExportOracleData { data << r.WORKDATE allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" - def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" } def headers = []//["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "BIGDATA_DISTRICT_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202108") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + CcbFtpUtils.ftpUpload(fileName, curDate) println "BIGDATA_DISTRICT_GJ 上传Ftp完成......" } @@ -239,19 +222,15 @@ class ExportOracleData { data << r.SOURCECODE allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" - def dataCodeFileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "GUIDED_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202108") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + CcbFtpUtils.ftpUpload(fileName, curDate) println "GUIDED_PRICE_GJ 上传Ftp完成......" } @@ -275,20 +254,16 @@ class ExportOracleData { data << r.SOURCECODE allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" - def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "COMMUNITY_GUIDE_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202107") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202107") + CcbFtpUtils.ftpUpload(fileName, curDate) println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......" } @@ -368,20 +343,16 @@ class ExportOracleData { data << r.WORKDATE allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" - def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" } def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "BIGDATA_DEAL_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202108") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + CcbFtpUtils.ftpUpload(fileName, curDate) println "BIGDATA_DEAL_GJ 上传Ftp完成......" } @@ -420,21 +391,16 @@ class ExportOracleData { data << r.SOURCECODE allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" - def dataCodeFileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......" -// CcbFtpUtils.ftpUpload(fileName, "202107") -// CcbFtpUtils.ftpUpload(dataCodeFileName, "202107") -// println "THIRD_PARTY_PRICE_GJ 上传Ftp完成......" + CcbFtpUtils.ftpUpload(fileName, curDate) } def exportBigdataCommunity(){ @@ -477,102 +443,102 @@ class ExportOracleData { data << r.STATUS allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ.dat" - def dataCodeFileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" } def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "BIGDATA_COMMUNITY_GJ 数据文件输出完成......" -// CcbFtpUtils.ftpUpload(fileName, "202109") -// CcbFtpUtils.ftpUpload(dataCodeFileName, "202109") -// println "BIGDATA_COMMUNITY_GJ 上传Ftp完成......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "BIGDATA_COMMUNITY_GJ 上传Ftp完成......" } def exportBigdataForsale(){ - def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + - " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + - " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where PUBLISHTIME>=to_date('2021-09-01','yyyy-mm-dd')";//(cityId=350200 or cityId=150100) and - def allRows = [] - def index = 0 - sqlInstance.eachRow(sql,{r-> - index = index+1 - println index - def data = [] - data << r.CITYID - data << r.FORSALEID - data << r.PROPCODE - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.COMMID - data << r.COMMNAME - data << r.BUILDID - data << r.BUILDNO - data << r.BUILDNAME - data << r.UNITID - data << r.UNITNO - data << r.UNITNAME - data << r.HOUSEID - data << r.ADDRESS - data << r.BUILDYEAR - data << r.PROPTYPE - data << r.PROPYEARS - data << r.UNITPRICE - data << r.TOTALPRICE - data << r.PAYTYPE - data << r.FLOORNUM - data << r.FLOORCNT - data << r.BUILDTYPE - data << r.HOUSESTRU - data << r.BUILDSTRU - data << r.BUILDAREA - data << r.LIVINGAREA - data << r.HOUSESTYLE - data << r.ROOMCNT - data << r.LIVINGROOMCNT - data << r.KITCHENCNT - data << r.BATHROOMCNT - data << r.HOUSEASPECT - data << r.FITMENT - data << r.IFELEVATOR - data << r.ELEVATORHOUSE - data << r.PUBLISHTIME - data << r.EXPIRETIME - data << r.IFFIRST - data << r.DATASOURCE - data << r.AGENCYID - data << r.AGENCYNAME - data << r.PUBLISHER - data << r.PHONE - data << "" - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - data << r.ATTACH - allRows << data - }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + + def citys = CcbCommonUtils.citys; + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" - def dataCodeFileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202109\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + for(def city : citys){ + def cityCode = cityCodeDataMap.get(city) + + def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId=${cityCode}"; + def allRows = [] + def index = 0 + sqlInstance.eachRow(sql,{r-> + index = index+1 + println city +" "+index + def data = [] + data << r.CITYID + data << r.FORSALEID + data << r.PROPCODE + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.UNITPRICE + data << r.TOTALPRICE + data << r.PAYTYPE + data << r.FLOORNUM + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.HOUSESTRU + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEASPECT + data << r.FITMENT + data << r.IFELEVATOR + data << r.ELEVATORHOUSE + data << r.PUBLISHTIME + data << r.EXPIRETIME + data << r.IFFIRST + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.PUBLISHER + data << r.PHONE + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.ATTACH + allRows << data + }); + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } println "BIGDATA_FORSALE_GJ 数据文件输出完成......" -// CcbFtpUtils.ftpUpload(fileName, "202107") -// CcbFtpUtils.ftpUpload(dataCodeFileName, "202107") -// println "BIGDATA_FORSALE_GJ 上传Ftp完成......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "BIGDATA_FORSALE_GJ 上传Ftp完成......" } @@ -621,21 +587,17 @@ class ExportOracleData { allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" - def dataCodeFileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" } def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE", //"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE", //"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "AVM_EVALUATION_GJNB 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202107") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202107") + CcbFtpUtils.ftpUpload(fileName, curDate) println "AVM_EVALUATION_GJNB 上传Ftp完成......" } @@ -673,19 +635,15 @@ class ExportOracleData { allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" - def dataCodeFileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" } def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202108") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + CcbFtpUtils.ftpUpload(fileName, curDate) println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......" } @@ -716,19 +674,15 @@ class ExportOracleData { allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" - def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" } def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202107") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202107") + CcbFtpUtils.ftpUpload(fileName, curDate) println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......" } @@ -753,19 +707,15 @@ class ExportOracleData { allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" - def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_CITY_PRICE_GJ_ADD_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "CITY_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202108") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + CcbFtpUtils.ftpUpload(fileName, curDate) println "CITY_PRICE_GJ 上传Ftp完成......" } @@ -788,19 +738,15 @@ class ExportOracleData { allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" - def dataCodeFileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "DISTRICT_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202108") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + CcbFtpUtils.ftpUpload(fileName, curDate) println "DISTRICT_PRICE_GJ 上传Ftp完成......" } @@ -812,7 +758,7 @@ class ExportOracleData { if(CommonDatautils.isWindow()){ dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curMonth}\\${curDate}.ok" } - CcbFtpUtils.ftpUpload(dataCodeFileName, "${curMonth}") + CcbFtpUtils.ftpUpload(dataCodeFileName, curDate) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index f98fd8db7..b8c48895f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -760,6 +760,29 @@ class CommonDatautils { } return ret; } + + public static def findDistrictFieldById(String id, def fields){ + + if(!id){ + return null; + } + DBObject q = new BasicDBObject(); + q.put("_id", new ObjectId(id.toString().replace("[","").replace("]",""))); + + DBObject keys = new BasicDBObject(); + if(fields){ + for(def field : fields){ + keys.put(field, 1); + } + } + def cursor = getDataStoreConnection().find(q, keys).batchSize(2000) + if(cursor){ + for(def obj : cursor){ + return obj; + } + } + return null; + } public static def findDistrictById(String id){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy index 7a766b668..73e4f2426 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy @@ -1,7 +1,9 @@ package com.fangjia.script.gj.ccb import com.fangjia.core.util.DataUtil +import org.apache.commons.lang.StringUtils +import java.util.regex.Matcher import java.util.regex.Pattern class CcbCodeUtils { @@ -173,19 +175,36 @@ class CcbCodeUtils { } //钢混结构:1 砖混结构:2 钢结构:3 框架结构:4 框架剪力墙结构:5 剪力墙结构:6 筒体结构:7 框架筒体结构:8 筒中筒:9 砖木结构:10 其他:11 - //建筑结构 +// 钢、钢筋混凝土结构 002 +// 钢筋混凝土结构 003 +// 砖木结构 004 +// 混合结构 005 +// 其它结构 006 + static def getBuildStructureCode(def buildStructure){ if(buildStructure){ if(buildStructure.contains("钢结构")){ - return 3; + return "001"; + }else if(buildStructure.contains("钢、钢筋混凝土结构")){ + return "002"; }else if(buildStructure.contains("钢筋混凝土结构")){ - return 1; + return "003"; }else if(buildStructure.contains("砖木结构")){ - return 10; + return "004"; + }else if(buildStructure.contains("混合结构")){ + return "005"; } - return 11; } - return null; + return "006"; + } + + static def getDistHouseTypeCode(def estateType){ + if(estateType && estateType=="别墅"){ + return "08" + }else if(estateType && (estateType=="普通住宅" || estateType=="商住")){ + return "01" + } + return "99" } //室内结构 //平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23 @@ -221,4 +240,66 @@ class CcbCodeUtils { } return 99 } + + public static Integer getBuildYear(String buildDate){ + if(StringUtils.isNotBlank(buildDate)){ + Pattern p = Pattern.compile("\\d{4}") + Matcher m = p.matcher(buildDate); + while (m.find()) { + return m.group(0).toInteger(); + } + } + return null; + } + + static def processEstateCharge(def city, def estateCharege, def estateType2){ + if(estateCharege){ + try{ + estateCharege = trimChar(estateCharege); + estateCharege = estateCharege.replace("元/平米/月","").replace("元/平方米/月","").replace("元/?O?月","") + estateCharege = estateCharege.toDouble() + if(estateType2=="普通住宅"){ + if(city=="上海" || city=="北京" || city=="广州" || city=="杭州"){ + if(estateCharege>=0.6 && estateCharege<=3.5){ + return estateCharege + }else{ + return null; + } + }else{ + if(estateCharege>=0.2 && estateCharege<=3.5){ + return estateCharege + }else{ + return null; + } + } + }else if(estateType2=="别墅"){ + if(estateCharege>=3){ + return estateCharege; + }else{ + return null; + } + }else if(estateType2=="商住"){ + if(estateCharege>=4){ + return estateCharege; + }else{ + return null; + } + } + }catch (Exception e){ + println "estateCharege 转不了 "+estateCharege + } + } + return estateCharege; + } + + static def trimChar(String c) { + try { + if(c){ + c = new String(c.getBytes("GBK"), "GBK").replace("?", "/"); + } + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + return c; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy index 4a8801a5e..811de10a1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy @@ -2,6 +2,8 @@ package com.fangjia.script.jiangnanblank import com.jcraft.jsch.* +import javax.transaction.SystemException + /** * Created by gaojuan on 2017/12/6. */ @@ -48,7 +50,8 @@ class SFTP { */ public void upload(String directory, String uploadFile, ChannelSftp sftp) { FileInputStream stream = null - try { + try{ + sftp = createDir(directory, sftp) sftp.cd(directory); File file=new File(uploadFile); stream = new FileInputStream(file) @@ -110,4 +113,51 @@ class SFTP { } + /** + * 创建一个文件目录 + */ + public ChannelSftp createDir(String createpath, ChannelSftp sftp) { + try { + if (isDirExist(createpath, sftp)) { + sftp.cd(createpath); + } + def pathArry = createpath.split("/"); + StringBuffer filePath = new StringBuffer("/"); + for (String path : pathArry) { + if (path.equals("")) { + continue; + } + filePath.append(path + "/"); + if (isDirExist(filePath.toString(), sftp)) { + sftp.cd(filePath.toString()); + } else { + // 建立目录 + sftp.mkdir(filePath.toString()); + // 进入并设置为当前目录 + sftp.cd(filePath.toString()); + } + } + sftp.cd(createpath); + } catch (SftpException e) { + throw new SystemException("创建路径错误:" + createpath); + } + return sftp + } +/** + * 判断目录是否存在 + */ + public boolean isDirExist(String directory, ChannelSftp sftp) { + boolean isDirExistFlag = false; + try { + SftpATTRS sftpATTRS = sftp.lstat(directory); + isDirExistFlag = true; + return sftpATTRS.isDir(); + } catch (Exception e) { + if (e.getMessage().toLowerCase().equals("no such file")) { + isDirExistFlag = false; + } + } + return isDirExistFlag; + } + } From 7fc2dc7300bce056595112ff8a1a7ebb8e1b544c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 20 Oct 2021 14:01:20 +0800 Subject: [PATCH 48/80] =?UTF-8?q?=E6=88=BF=E6=BA=90=E5=86=99=E5=85=A5?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index f4f1a7761..b3b6666e8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -66,8 +66,8 @@ class BigdataForsaleBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; def index = 0 + def citys = CcbCommonUtils.citys; def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ From 11318b1ca9b329940c4b12b3b0efba62f08f0da0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 20 Oct 2021 16:59:59 +0800 Subject: [PATCH 49/80] update --- .../script/ccbevaluation/ExportOracleData.groovy | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 5956d4a7d..64f7e439c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -533,8 +533,16 @@ class ExportOracleData { data << r.SOURCECODE data << r.ATTACH allRows << data + + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() + } }); - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } println "BIGDATA_FORSALE_GJ 数据文件输出完成......" CcbFtpUtils.ftpUpload(fileName, curDate) From d12ad4e7b60b236a95d9ac24ac3e30e0856ae1b6 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 20 Oct 2021 18:46:55 +0800 Subject: [PATCH 50/80] =?UTF-8?q?58=20=E7=89=B9=E5=AE=9A=E5=9F=8E=E5=B8=82?= =?UTF-8?q?=E6=8A=93=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 849fee72e..bc1909c72 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -191,7 +191,7 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - def citys = ["成都","南京","福州","厦门","泉州","东莞","广州","广州","广州","西安","沈阳","大连","昆明","太原","郑州","乌鲁木齐","烟台","绵阳","邯郸","潍坊","嘉兴","惠州","珠海","海口"] + def citys = ["成都","广州","深圳","杭州","南京","郑州","苏州","武汉","青岛","石家庄","太原","济南","长沙","沈阳","宁波","佛山","珠海","昆明","温州","无锡","哈尔滨","南宁","东莞","大连","海口","潍坊","绵阳","乌鲁木齐","惠州"] cityList.each{city, str-> for(def i=0;i<1;i++){ // def city = "福州" @@ -220,10 +220,12 @@ class Esf58Crawl extends BaseBatch{ def districtName = districts[i].name println "${city}【${i}/${districts.size()}】 "+districtName for(def j=1;j<=districtName.length()-1;j++) { - def keyword = districtName.toString().substring(0, j); - def keyStr = city+"##"+keyword; - if(!useDistKeys.contains(keyStr)){ - useDistKeys << keyStr + if(j<=2){ + def keyword = districtName.toString().substring(0, j); + def keyStr = city+"##"+keyword; + if(!useDistKeys.contains(keyStr)){ + useDistKeys << keyStr + } } } } From cee1ea26df51936ba2de7bad9f422a0a596e3053 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 21 Oct 2021 10:11:22 +0800 Subject: [PATCH 51/80] =?UTF-8?q?=E5=AF=BC10.1-18=E5=8F=B7=E6=88=BF?= =?UTF-8?q?=E6=BA=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/BigdataForsaleBatch.groovy | 8 ++++---- .../fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index b3b6666e8..2cea78fa5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200101, 20210930); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20211018); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); } } @@ -96,9 +96,9 @@ class BigdataForsaleBatch extends BaseBatch{ def commId = rMap.get("_id").toString() def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias) - if(ret && ret.size()<100){ - ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) - } +// if(ret && ret.size()<100){ +// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) +// } if(ret){ def areaData = distCodeMap.get(rMap.region) if(areaData){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 0bcb9d1a7..59ffd572d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -97,6 +97,7 @@ class CcbCommonUtils { q.put("city", dist.city) q.put("region", dist.region) q.put("area", new BasicDBObject('$gte', 20)) + q.put("avg_price", new BasicDBObject('$gt', 0)) def alias = [] if(dist.alias){ if(!dist.alias.contains(dist.name)){ From 176923b86a246946ad12598dbd533c2d0025ff70 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 21 Oct 2021 15:53:06 +0800 Subject: [PATCH 52/80] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E8=9E=8D=E5=90=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/BigdataCommunityGJRHBatch.groovy | 357 ++++++++++++++++++ 1 file changed, 357 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy new file mode 100644 index 000000000..3c39c1092 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -0,0 +1,357 @@ +package com.fangjia.script.ccbevaluation.ronghe + +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.jusfoun.JusfounCommon +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.OpenApiClient +import groovy.sql.Sql +import org.bson.types.ObjectId + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern + + +/** + * 建行融合——小区表 + */ +@BatchOwner(Watcher.GaoJuan) +class BigdataCommunityGJRHBatch extends BaseBatch { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + public static def batchsize = 100 + static main(args) { + runBatch(new BigdataCommunityGJRHBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + citys.remove("厦门") + citys.remove("杭州") + def rows = [] + +// addBigdataDistrictFJW() + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + for (def city : citys) { + def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) + + for(def dist : districts){ + def obj = CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) + if(obj){ + if(obj.alias && obj.alias.size()>0){ + dist.commalias = obj.alias.get(0) + }else{ + dist.commalias = "" + } + dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure) + dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2) + dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date); + dist.houseuse = "11" + dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type) + dist.compid = new ObjectId() //物业公司id + dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) + } + } + println city +" 小区加载完成 "+districts.size() + addBigdataCommunityGJRH(districts) + println city +" 1.小区表完成" + addBigdataCommbuildstruGJ(districts) + println city +" 2.小区建筑结构表" + addBigdataCommhousetypeGJ(districts) + println city +" 3.小区房屋类型表" + addBigdataCommbuildyearGj(districts) + println city +" 4.小区建筑年代表" + addBigdataCommfeeGj(districts) + println city +" 6.小区费用表" + addBigdataCommhouseuseGJ(districts) + println city +" 7.小区房屋用途表" + addBigdataPositionGJ(districts) + println city + " 8.地理位置表" + } + } + + def addBigdataCommunityGJRH(def districts){ + + sqlInstance.withBatch(batchsize) { stmt -> + for(def d : districts){ + def sql = "INSERT INTO BIGDATA_COMMUNITY_FJW (CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,CREATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.distid}','${d.commid}','${d.commname}',${converValue(d.address)},${d.landarea},${d.buildarea},${d.greeningrate},${d.plotratio},${d.parkingcnt}," + if(d.updatetime){ + sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.commtype}','${d.commalias}'," + }else{ + sql = sql + "null,'${d.commtype}','${d.commalias}'," + } + if(d.street && d.street!='null'){ + sql = sql +"'${d.street}'," + }else{ + sql = sql +"null," + } + if(d.streetno && d.streetno!='null'){ + sql = sql + "'${d.streetno}'," + }else{ + sql = sql +"null," + } + + sql = sql + "'${d.ifnew}',null,${d.remark}," + if(d.createtime){ + sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')" + }else{ + sql = sql + "null,'${d.sourcecode}')" + } + + try{ + stmt.addBatch sql + }catch(Exception e){ + println d + println sql + } + + } + + } + } + + def converValue(def val){ + if(val && val!='null'){ + return "'"+val+"'"; + }else{ + return null + } + } + + def addBigdataCommbuildstruGJ(def districts){ + sqlInstance.withBatch(batchsize) { stmt -> + for(def d : districts){ + def sql = "INSERT INTO BIGDATA_COMMBUILDSTRU_FJW (CITYID,COMMID,BUILDSTRU,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.buildstru}',${d.remark}," + if(d.createtime){ + sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss')," + }else{ + sql = sql + "null," + } + if(d.updatetime){ + sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')" + }else{ + sql = sql + "null,'${d.sourcecode}')" + } + try{ + stmt.addBatch sql + }catch(Exception e){ + println d + println sql + } + } + } + } + + + def addBigdataCommhousetypeGJ(def districts){ + sqlInstance.withBatch(batchsize) { stmt -> + for(def d : districts){ + def sql = "INSERT INTO BIGDATA_COMMHOUSETYPE_FJW (CITYID,COMMID,HOUSETYPE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.housetype}',${d.remark}," + if(d.createtime){ + sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss')," + }else{ + sql = sql + "null," + } + if(d.updatetime){ + sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')" + }else{ + sql = sql + "null,'${d.sourcecode}')" + } + try{ + stmt.addBatch sql + }catch(Exception e){ + println d + println sql + } + } + } + } + + def addBigdataCommbuildyearGj(districts){ + sqlInstance.withBatch(batchsize) { stmt -> + for(def d : districts){ + if(d.buildyear){ + def sql = "INSERT INTO BIGDATA_COMMBUILDYEAR_FJW (CITYID,COMMID,BUILDYEAR,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}',${d.buildyear},${d.remark}," + if(d.createtime){ + sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss')," + }else{ + sql = sql + "null," + } + if(d.updatetime){ + sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')" + }else{ + sql = sql + "null,'${d.sourcecode}')" + } + try{ + stmt.addBatch sql + }catch(Exception e){ + println d + println sql + } + } + } + } + } + + def addBigdataCommfeeGj(districts){ + + def sqls = [] + for(def d : districts){ + if(d.estatecharge){ + def sql = "INSERT INTO BIGDATA_COMMFEE_FJW (CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.compid}',${d.estatecharge},'${d.houseuse}','${d.buildtype}','1',${d.note},${d.remark}," + if(d.createtime){ + sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss')," + }else{ + sql = sql + "null," + } + if(d.updatetime){ + sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')" + }else{ + sql = sql + "null,'${d.sourcecode}')" + } + sqls << sql + + if (sqls.size() >= batchsize) { + try { + sqlInstance.withBatch(batchsize, { stmt -> + sqls.each { def s -> + println s + stmt.addBatch s + } + }) + + } catch (Exception e) { + e.printStackTrace() + } + sqls.clear() + } + + } + } + + if (sqls.size() > 0) { + sqlInstance.withBatch(100, { stmt -> + sqls.each { def s -> + stmt.addBatch s + } + }) + } + } + + def addBigdataCommhouseuseGJ(districts){ + def sqls = [] + for(def d : districts){ + def sql = "INSERT INTO BIGDATA_COMMHOUSEUSE_FJW (CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.houseuse}',${d.zts},${d.remark}," + if(d.createtime){ + sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss')," + }else{ + sql = sql + "null," + } + if(d.updatetime){ + sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')" + }else{ + sql = sql + "null,'${d.sourcecode}')" + } + sqls << sql + + if (sqls.size() >= batchsize) { + try { + sqlInstance.withBatch(batchsize, { stmt -> + sqls.each { def s -> + stmt.addBatch s + } + }) + + } catch (Exception e) { + e.printStackTrace() + } + sqls.clear() + } + + } + + if (sqls.size() > 0) { + sqlInstance.withBatch(100, { stmt -> + sqls.each { def s -> + stmt.addBatch s + } + }) + } + } + + def addBigdataPositionGJ(districts){ + def sqls = [] + def index = 1 + for(def d : districts){ + def sql = "INSERT INTO BIGDATA_POSITION_FJW (CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}','5', '${d.commid}','1',${index},${d.lng},${d.lat},null,${d.remark}," + index++ + if(d.createtime){ + sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss')," + }else{ + sql = sql + "null," + } + if(d.updatetime){ + sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')" + }else{ + sql = sql + "null,'${d.sourcecode}')" + } + sqls << sql + + if (sqls.size() >= batchsize) { + try { + sqlInstance.withBatch(batchsize, { stmt -> + sqls.each { def s -> + stmt.addBatch s + } + }) + + } catch (Exception e) { + e.printStackTrace() + } + sqls.clear() + } + + } + + if (sqls.size() > 0) { + sqlInstance.withBatch(100, { stmt -> + sqls.each { def s -> + stmt.addBatch s + } + }) + } + } + + def addBigdataDistrictFJW(){ + def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from bigdata_district_gj"; + def rows = [] + try{ + sqlInstance.eachRow(sql,{r-> + def insertSql = "INSERT INTO bigdata_district_fjw(DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${r.distid}','${r.distname}','${r.distspell}','${r.cityid}','${r.distcode}','${r.seq}',${r.disttype},null,to_date('${r.createtime}','yyyy-mm-dd hh24:mi:ss'),to_date('${r.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${r.sourcecode}')" + sqlInstance.withBatch(batchsize, { stmt -> + stmt.addBatch insertSql + }) + }); + }catch(Exception e){ + + } + } + +} \ No newline at end of file From f60bb2d10df5e31cac02dafc7288119681510605 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 21 Oct 2021 17:40:42 +0800 Subject: [PATCH 53/80] =?UTF-8?q?=E6=96=B0=E5=A2=9E4=E4=B8=AA=E5=9F=8E?= =?UTF-8?q?=E5=B8=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index 3c39c1092..2323d31d7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -42,7 +42,8 @@ class BigdataCommunityGJRHBatch extends BaseBatch { @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + + def citys = ["郑州","长沙","西安","南京"]//CcbCommonUtils.citys; citys.remove("厦门") citys.remove("杭州") def rows = [] From 39ea0af0bdc3df6f40752f398e32948c3ce6db7f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 21 Oct 2021 18:49:41 +0800 Subject: [PATCH 54/80] update --- .../fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 3 ++- .../ronghe/BigdataCommunityGJRHBatch.groovy | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 59ffd572d..d7735dfe7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -235,7 +235,7 @@ class CcbCommonUtils { } public static def findAllDistrictsByOracle(def cityid){ - def sql = "select CITYID,COMMID,COMMNAME,COMMALIAS,DISTID,STREET,STREETNO,ADDRESS,LANDAREA,BUILDAREA,COMMTYPE,IFNEW,GREENINGRATE,PLOTRATIO,PARKINGCNT,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,WORKDATE,ZTS,LONGITUDE,LATITUDE from bigdata_community_gj where cityId=${cityid} and status=0"; + def sql = "select CITYID,COMMID,COMMNAME,COMMALIAS,DISTID,STREET,STREETNO,ADDRESS,LANDAREA,BUILDAREA,COMMTYPE,IFNEW,GREENINGRATE,PLOTRATIO,PARKINGCNT,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,WORKDATE,ZTS,LONGITUDE,LATITUDE from bigdata_community_gj where cityId='${cityid}' and status=0 "; def rows = [] try{ sqlInstance.eachRow(sql,{r-> @@ -267,6 +267,7 @@ class CcbCommonUtils { }); }catch(Exception e){ findAllDistrictsByOracle(cityid) + e.printStackTrace() } return rows diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index 2323d31d7..5808a5da4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -35,7 +35,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - public static def batchsize = 100 + public static def batchsize = 1000 static main(args) { runBatch(new BigdataCommunityGJRHBatch(), args); } @@ -43,9 +43,10 @@ class BigdataCommunityGJRHBatch extends BaseBatch { @Override public void process(Object args) { - def citys = ["郑州","长沙","西安","南京"]//CcbCommonUtils.citys; - citys.remove("厦门") - citys.remove("杭州") + //"郑州","长沙","西安", + def citys = ["长沙","西安"]//CcbCommonUtils.citys; +// citys.remove("厦门") +// citys.remove("杭州") def rows = [] // addBigdataDistrictFJW() From 057caf9312135d614ed49d3994c45bb501272cc9 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 21 Oct 2021 19:31:29 +0800 Subject: [PATCH 55/80] =?UTF-8?q?=E8=A5=BF=E5=AE=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/BigdataCommunityGJRHBatch.groovy | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index 5808a5da4..66cde3752 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -44,7 +44,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { public void process(Object args) { //"郑州","长沙","西安", - def citys = ["长沙","西安"]//CcbCommonUtils.citys; + def citys = ["西安"]//CcbCommonUtils.citys; // citys.remove("厦门") // citys.remove("杭州") def rows = [] @@ -71,15 +71,15 @@ class BigdataCommunityGJRHBatch extends BaseBatch { dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) } } - println city +" 小区加载完成 "+districts.size() - addBigdataCommunityGJRH(districts) - println city +" 1.小区表完成" - addBigdataCommbuildstruGJ(districts) - println city +" 2.小区建筑结构表" - addBigdataCommhousetypeGJ(districts) - println city +" 3.小区房屋类型表" - addBigdataCommbuildyearGj(districts) - println city +" 4.小区建筑年代表" +// println city +" 小区加载完成 "+districts.size() +// addBigdataCommunityGJRH(districts) +// println city +" 1.小区表完成" +// addBigdataCommbuildstruGJ(districts) +// println city +" 2.小区建筑结构表" +// addBigdataCommhousetypeGJ(districts) +// println city +" 3.小区房屋类型表" +// addBigdataCommbuildyearGj(districts) +// println city +" 4.小区建筑年代表" addBigdataCommfeeGj(districts) println city +" 6.小区费用表" addBigdataCommhouseuseGJ(districts) From 14a1efe3095500ce93c0ef921a0f846e1cbf7c33 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 21 Oct 2021 20:01:49 +0800 Subject: [PATCH 56/80] =?UTF-8?q?=E8=A5=BF=E5=AE=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/BigdataCommunityGJRHBatch.groovy | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index 66cde3752..2c6cdf6cb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -71,21 +71,21 @@ class BigdataCommunityGJRHBatch extends BaseBatch { dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) } } -// println city +" 小区加载完成 "+districts.size() -// addBigdataCommunityGJRH(districts) -// println city +" 1.小区表完成" -// addBigdataCommbuildstruGJ(districts) -// println city +" 2.小区建筑结构表" -// addBigdataCommhousetypeGJ(districts) -// println city +" 3.小区房屋类型表" -// addBigdataCommbuildyearGj(districts) -// println city +" 4.小区建筑年代表" + println city +" 小区加载完成 "+districts.size() + addBigdataCommunityGJRH(districts) + println city +" 1.小区表完成" + addBigdataCommbuildstruGJ(districts) + println city +" 2.小区建筑结构表" + addBigdataCommhousetypeGJ(districts) + println city +" 3.小区房屋类型表" + addBigdataCommbuildyearGj(districts) + println city +" 4.小区建筑年代表" addBigdataCommfeeGj(districts) println city +" 6.小区费用表" - addBigdataCommhouseuseGJ(districts) - println city +" 7.小区房屋用途表" - addBigdataPositionGJ(districts) - println city + " 8.地理位置表" +// addBigdataCommhouseuseGJ(districts) +// println city +" 7.小区房屋用途表" +// addBigdataPositionGJ(districts) +// println city + " 8.地理位置表" } } @@ -234,12 +234,12 @@ class BigdataCommunityGJRHBatch extends BaseBatch { try { sqlInstance.withBatch(batchsize, { stmt -> sqls.each { def s -> - println s stmt.addBatch s } }) } catch (Exception e) { + println sqls[0] e.printStackTrace() } sqls.clear() From 016d2e4239455c51fe29bba585e94b57af75ff2e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 21 Oct 2021 21:20:51 +0800 Subject: [PATCH 57/80] =?UTF-8?q?=E5=85=A8=E5=9B=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/BigdataCommunityGJRHBatch.groovy | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index 2c6cdf6cb..91c175264 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -44,9 +44,13 @@ class BigdataCommunityGJRHBatch extends BaseBatch { public void process(Object args) { //"郑州","长沙","西安", - def citys = ["西安"]//CcbCommonUtils.citys; -// citys.remove("厦门") -// citys.remove("杭州") + def citys = CcbCommonUtils.citys; + citys.remove("厦门") + citys.remove("杭州") + citys.remove("郑州") + citys.remove("长沙") + citys.remove("西安") + citys.remove("南京") def rows = [] // addBigdataDistrictFJW() @@ -82,10 +86,10 @@ class BigdataCommunityGJRHBatch extends BaseBatch { println city +" 4.小区建筑年代表" addBigdataCommfeeGj(districts) println city +" 6.小区费用表" -// addBigdataCommhouseuseGJ(districts) -// println city +" 7.小区房屋用途表" -// addBigdataPositionGJ(districts) -// println city + " 8.地理位置表" + addBigdataCommhouseuseGJ(districts) + println city +" 7.小区房屋用途表" + addBigdataPositionGJ(districts) + println city + " 8.地理位置表" } } From 9526324806ead6e597c43ef08a1cf814f3009fae Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 22 Oct 2021 09:43:52 +0800 Subject: [PATCH 58/80] =?UTF-8?q?=E6=88=BF=E6=BA=90=20=E5=8C=97=E4=B8=8A?= =?UTF-8?q?=E5=B9=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/ExportOracleData.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 64f7e439c..3380be5f0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -24,7 +24,7 @@ class ExportOracleData { // exportCommunityStaging() // exportCommunityBuileType() // exportBigdataDistrictData() -// exportBigdataCommunity() +// exportBigdataCommunity() exportBigdataForsale() @@ -459,7 +459,7 @@ class ExportOracleData { def exportBigdataForsale(){ - def citys = CcbCommonUtils.citys; + def citys = ["北京","上海","广州"]//CcbCommonUtils.citys; def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; From 11a39b2e9d37316563894a9c5e4c931e9fdc0966 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 22 Oct 2021 10:38:31 +0800 Subject: [PATCH 59/80] =?UTF-8?q?=E6=88=BF=E6=BA=90=20=E9=9D=9E=E5=8C=97?= =?UTF-8?q?=E4=B8=8A=E5=B9=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/ExportOracleData.groovy | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 3380be5f0..9855fb8f6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -459,13 +459,16 @@ class ExportOracleData { def exportBigdataForsale(){ - def citys = ["北京","上海","广州"]//CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys; + citys.remove("北京") + citys.remove("上海") + citys.remove("广州") def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0002.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0002.dat" } def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] for(def city : citys){ From 41ac972f3d1191528016459aa7368727f9e66a4d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 22 Oct 2021 10:55:52 +0800 Subject: [PATCH 60/80] =?UTF-8?q?7=E4=B8=AA=E5=9F=8E=E5=B8=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ExportOracleData.groovy | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 9855fb8f6..821c9ba1b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -459,16 +459,20 @@ class ExportOracleData { def exportBigdataForsale(){ - def citys = CcbCommonUtils.citys; - citys.remove("北京") - citys.remove("上海") - citys.remove("广州") + def citys =["北京","上海","广州","深圳","重庆","天津","成都"] CcbCommonUtils.citys; +// citys.remove("北京") +// citys.remove("上海") +// citys.remove("广州") +// citys.remove("深圳") +// citys.remove("重庆") +// citys.remove("天津") +// citys.remove("成都") def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0002.dat" + def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0002.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] for(def city : citys){ From 3a3242848813165d1be2217eedfc5c4f5f328a77 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 22 Oct 2021 11:01:23 +0800 Subject: [PATCH 61/80] update --- .../com/fangjia/script/ccbevaluation/ExportOracleData.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 821c9ba1b..08a7235c7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -459,7 +459,7 @@ class ExportOracleData { def exportBigdataForsale(){ - def citys =["北京","上海","广州","深圳","重庆","天津","成都"] CcbCommonUtils.citys; + def citys =["北京","上海","广州","深圳","重庆","天津","成都"] //CcbCommonUtils.citys; // citys.remove("北京") // citys.remove("上海") // citys.remove("广州") From d39b4b61a525d784eb529e0d8434eebd0ff9133f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 22 Oct 2021 11:58:24 +0800 Subject: [PATCH 62/80] =?UTF-8?q?85=E4=B8=AA=E5=9F=8E=E5=B8=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleData.groovy | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 08a7235c7..ff9109f69 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -459,20 +459,20 @@ class ExportOracleData { def exportBigdataForsale(){ - def citys =["北京","上海","广州","深圳","重庆","天津","成都"] //CcbCommonUtils.citys; -// citys.remove("北京") -// citys.remove("上海") -// citys.remove("广州") -// citys.remove("深圳") -// citys.remove("重庆") -// citys.remove("天津") -// citys.remove("成都") + def citys = CcbCommonUtils.citys; + citys.remove("北京") + citys.remove("上海") + citys.remove("广州") + citys.remove("深圳") + citys.remove("重庆") + citys.remove("天津") + citys.remove("成都") def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0002.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0002.dat" } def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] for(def city : citys){ From e5a0c6752ea73566dd2bbd10aa7a3c71c75c5444 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 22 Oct 2021 14:22:36 +0800 Subject: [PATCH 63/80] =?UTF-8?q?=E6=88=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/ExportOracleData.groovy | 6 +++--- .../groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index ff9109f69..158f33799 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -25,12 +25,12 @@ class ExportOracleData { // exportCommunityBuileType() // exportBigdataDistrictData() // exportBigdataCommunity() - exportBigdataForsale() +// exportBigdataForsale() // exportGuidedPrice() // exportCommunityGuidedPrice() -// exportBigdataDeal() + exportBigdataDeal() // exportThirdPartyPrice() @@ -272,7 +272,7 @@ class ExportOracleData { "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + - "from bigdata_deal_gj where cityId in (350200,150100) and dealdate>=to_date('2021-08-01','yyyy-mm-dd')"; + "from bigdata_deal_gj"; def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy index 73e4f2426..a418cebb9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy @@ -287,9 +287,10 @@ class CcbCodeUtils { } }catch (Exception e){ println "estateCharege 转不了 "+estateCharege + return null; } } - return estateCharege; + return null; } static def trimChar(String c) { From dec88f62ae7d3c6ba094535dd59c10cb90ab6b11 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 22 Oct 2021 16:37:59 +0800 Subject: [PATCH 64/80] update --- .../script/ccbevaluation/BigdataDealBatch.groovy | 10 +++++----- .../script/ccbevaluation/ExportOracleData.groovy | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 7fbedb2fc..56c4d73e2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -29,7 +29,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def curDate = 20210701//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); + def curDate = 20210901//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] @@ -420,8 +420,8 @@ class BigdataDealBatch extends BaseBatch{ // startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // } -// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) - def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd")) + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) +// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd")) for(def i=0;i Date: Fri, 22 Oct 2021 18:25:27 +0800 Subject: [PATCH 65/80] update --- .../script/ccbevaluation/ThirdPartyPriceBatch.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 1e8ca6a82..b813a155d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -20,9 +20,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202108//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202106 - def endDate = 202108 + def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202107 + def endDate = 202109 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") From e2d4aa2fe1d984666cd727fb8a3450d0090cadf5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 22 Oct 2021 20:11:56 +0800 Subject: [PATCH 66/80] =?UTF-8?q?=E5=AF=BC=E7=AC=AC=E4=B8=89=E6=96=B9?= =?UTF-8?q?=E5=9D=87=E4=BB=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleData.groovy | 92 ++++++++++--------- 1 file changed, 49 insertions(+), 43 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index a20ebee03..4a7233a51 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -18,7 +18,8 @@ class ExportOracleData { def task = new ExportOracleData() task.run() } - def run(){ + + def run() { // exportEvaluationRateData() // exportBigdataPoiData() // exportCommunityStaging() @@ -45,10 +46,10 @@ class ExportOracleData { } - def exportEvaluationRateData(){ + def exportEvaluationRateData() { def sql = "select ID,RATE_TYPE,RATE,RANK,CITY,REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from evaluate_rate_gj"; def allRows = [] - sqlInstance.eachRow(sql,{r-> + sqlInstance.eachRow(sql, { r -> def data = [] data << r.ID data << r.RATE_TYPE @@ -62,9 +63,9 @@ class ExportOracleData { allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/output/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ + if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" } def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] @@ -75,10 +76,10 @@ class ExportOracleData { } - def exportBigdataPoiData(){ + def exportBigdataPoiData() { def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj where cityId in (350200,150100)"; def allRows = [] - sqlInstance.eachRow(sql,{r-> + sqlInstance.eachRow(sql, { r -> def data = [] data << r.CITYID data << r.DISTID @@ -95,10 +96,10 @@ class ExportOracleData { allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ + if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" } def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"] @@ -109,10 +110,10 @@ class ExportOracleData { println "BIGDATA_POI_GJ 上传Ftp完成......" } - def exportCommunityStaging(){ + def exportCommunityStaging() { def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj"; def allRows = [] - sqlInstance.eachRow(sql,{r-> + sqlInstance.eachRow(sql, { r -> def data = [] data << r.ID data << r.COMMID @@ -123,9 +124,9 @@ class ExportOracleData { data << r.SOURCECODE allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ + if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" } def headers = []//["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"] @@ -136,10 +137,10 @@ class ExportOracleData { println "COMMUNITY_STAGING_GJ 上传Ftp完成......" } - def exportCommunityBuileType(){ + def exportCommunityBuileType() { def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID in (350200,150100)"; def allRows = [] - sqlInstance.eachRow(sql,{r-> + sqlInstance.eachRow(sql, { r -> def data = [] data << r.CITYID data << r.COMMID @@ -151,9 +152,9 @@ class ExportOracleData { data << r.WORKDATE allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ + if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" } def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] @@ -164,11 +165,11 @@ class ExportOracleData { println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......" } - def exportBigdataDistrictData(){ + def exportBigdataDistrictData() { def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj where cityId in (350200,150100)"; def allRows = [] - sqlInstance.eachRow(sql,{r-> + sqlInstance.eachRow(sql, { r -> def data = [] data << r.DISTID data << r.DISTNAME @@ -187,12 +188,13 @@ class ExportOracleData { data << r.WORKDATE allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ + if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" } - def headers = []//["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"] + def headers = [] +//["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_DISTRICT_GJ 数据文件输出完成......" @@ -201,10 +203,10 @@ class ExportOracleData { } - def exportGuidedPrice(){ + def exportGuidedPrice() { def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where dealdate>=TO_DATE('2021-08-01', 'yyyy-MM-dd') and dealdate + sqlInstance.eachRow(sql, { r -> def data = [] data << r.ID data << r.COMMID @@ -222,22 +224,23 @@ class ExportOracleData { data << r.SOURCECODE allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ + if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" } - def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] + def headers = [] +//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "GUIDED_PRICE_GJ 数据文件输出完成......" CcbFtpUtils.ftpUpload(fileName, curDate) println "GUIDED_PRICE_GJ 上传Ftp完成......" } - def exportCommunityGuidedPrice(){ + def exportCommunityGuidedPrice() { def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where cityname='西安'"; def allRows = [] - sqlInstance.eachRow(sql,{r-> + sqlInstance.eachRow(sql, { r -> def data = [] data << r.ID data << r.COMMID @@ -254,9 +257,9 @@ class ExportOracleData { data << r.SOURCECODE allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ + if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"] @@ -267,14 +270,14 @@ class ExportOracleData { println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......" } - def exportBigdataDeal(){ + def exportBigdataDeal() { def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + "from bigdata_deal_gj"; def allRows = [] - sqlInstance.eachRow(sql,{r-> + sqlInstance.eachRow(sql, { r -> def data = [] data << r.CITYID data << r.DEALID @@ -343,12 +346,13 @@ class ExportOracleData { data << r.WORKDATE allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ + if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" } - def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + def headers = [] +//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_DEAL_GJ 数据文件输出完成......" @@ -357,12 +361,12 @@ class ExportOracleData { } - def exportThirdPartyPrice(){ + def exportThirdPartyPrice() { def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, \n" + "TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE\n" + "from third_party_price_gj"; def allRows = [] - sqlInstance.eachRow(sql,{r-> + sqlInstance.eachRow(sql, { r -> def data = [] data << r.ID data << r.COMMID @@ -388,21 +392,23 @@ class ExportOracleData { data << r.WIWJ_AVG_PRICE data << r.CREATETIME data << r.UPDATETIME - data << r.SOURCECODE.replaceAll(~'\\r','') + data << r.SOURCECODE.replaceAll(~'\\r', '') allRows << data - }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + }) + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ + if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" } - def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + def headers = [] +//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......" CcbFtpUtils.ftpUpload(fileName, curDate) } + def exportBigdataCommunity(){ def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" + "\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj ";//where cityId in (350200,150100) From 70f1cb8106cb542c5dd86e7a295ecc227a976578 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 22 Oct 2021 20:40:55 +0800 Subject: [PATCH 67/80] =?UTF-8?q?10=E6=9C=88=E6=88=BF=E6=BA=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleData.groovy | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 4a7233a51..fac7f79f8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -26,13 +26,13 @@ class ExportOracleData { // exportCommunityBuileType() // exportBigdataDistrictData() // exportBigdataCommunity() -// exportBigdataForsale() + exportBigdataForsale() // exportGuidedPrice() // exportCommunityGuidedPrice() // exportBigdataDeal() - exportThirdPartyPrice() +// exportThirdPartyPrice() // exportAvmEvaluationGJNB() @@ -466,19 +466,12 @@ class ExportOracleData { def exportBigdataForsale(){ def citys = CcbCommonUtils.citys; - citys.remove("北京") - citys.remove("上海") - citys.remove("广州") - citys.remove("深圳") - citys.remove("重庆") - citys.remove("天津") - citys.remove("成都") def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0002.dat" + def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0003.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0002.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0003.dat" } def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] for(def city : citys){ @@ -486,7 +479,7 @@ class ExportOracleData { def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + - " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId=${cityCode}"; + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId=${cityCode} and PUBLISHTIME>=TO_DATE('2021-10-01', 'yyyy-MM-dd') and PUBLISHTIME From 3efed3b8037896ca8cfede5cb5072bbfcdce2c23 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 25 Oct 2021 14:02:19 +0800 Subject: [PATCH 68/80] =?UTF-8?q?15=E4=B8=AA=E5=9F=8E=E5=B8=82=E5=B0=8F?= =?UTF-8?q?=E5=8C=BA=E8=9E=8D=E5=90=88=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/BigdataCommunityGJRHBatch.groovy | 52 +++++++++++-------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index 91c175264..c96c02ae1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -44,13 +44,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { public void process(Object args) { //"郑州","长沙","西安", - def citys = CcbCommonUtils.citys; - citys.remove("厦门") - citys.remove("杭州") - citys.remove("郑州") - citys.remove("长沙") - citys.remove("西安") - citys.remove("南京") + def citys = ["厦门","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉"]//CcbCommonUtils.citys; def rows = [] // addBigdataDistrictFJW() @@ -76,24 +70,26 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } } println city +" 小区加载完成 "+districts.size() - addBigdataCommunityGJRH(districts) + addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) println city +" 1.小区表完成" - addBigdataCommbuildstruGJ(districts) + addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) println city +" 2.小区建筑结构表" - addBigdataCommhousetypeGJ(districts) + addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) println city +" 3.小区房屋类型表" - addBigdataCommbuildyearGj(districts) + addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) println city +" 4.小区建筑年代表" - addBigdataCommfeeGj(districts) + addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) println city +" 6.小区费用表" - addBigdataCommhouseuseGJ(districts) + addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) println city +" 7.小区房屋用途表" - addBigdataPositionGJ(districts) + addBigdataPositionGJ(districts, cityCodeDataMap.get(city)) println city + " 8.地理位置表" } } - def addBigdataCommunityGJRH(def districts){ + def addBigdataCommunityGJRH(def districts, def cityid){ + + sqlInstance.executeUpdate("delete from BIGDATA_COMMUNITY_FJW where CITYID='${cityid}'") sqlInstance.withBatch(batchsize) { stmt -> for(def d : districts){ @@ -141,7 +137,8 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } } - def addBigdataCommbuildstruGJ(def districts){ + def addBigdataCommbuildstruGJ(def districts, def cityid){ + sqlInstance.executeUpdate("delete from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityid}'") sqlInstance.withBatch(batchsize) { stmt -> for(def d : districts){ def sql = "INSERT INTO BIGDATA_COMMBUILDSTRU_FJW (CITYID,COMMID,BUILDSTRU,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.buildstru}',${d.remark}," @@ -166,7 +163,9 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } - def addBigdataCommhousetypeGJ(def districts){ + def addBigdataCommhousetypeGJ(def districts, def cityid){ + sqlInstance.executeUpdate("delete from BIGDATA_COMMHOUSETYPE_FJW where CITYID='${cityid}'") + sqlInstance.withBatch(batchsize) { stmt -> for(def d : districts){ def sql = "INSERT INTO BIGDATA_COMMHOUSETYPE_FJW (CITYID,COMMID,HOUSETYPE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.housetype}',${d.remark}," @@ -190,7 +189,10 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } } - def addBigdataCommbuildyearGj(districts){ + def addBigdataCommbuildyearGj(districts, def cityid){ + + sqlInstance.executeUpdate("delete from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityid}'") + sqlInstance.withBatch(batchsize) { stmt -> for(def d : districts){ if(d.buildyear){ @@ -216,7 +218,9 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } } - def addBigdataCommfeeGj(districts){ + def addBigdataCommfeeGj(districts, def cityid){ + + sqlInstance.executeUpdate("delete from BIGDATA_COMMFEE_FJW where CITYID='${cityid}'") def sqls = [] for(def d : districts){ @@ -261,7 +265,10 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } } - def addBigdataCommhouseuseGJ(districts){ + def addBigdataCommhouseuseGJ(districts, def cityid){ + + sqlInstance.executeUpdate("delete from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityid}'") + def sqls = [] for(def d : districts){ def sql = "INSERT INTO BIGDATA_COMMHOUSEUSE_FJW (CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.houseuse}',${d.zts},${d.remark}," @@ -302,7 +309,10 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } } - def addBigdataPositionGJ(districts){ + def addBigdataPositionGJ(districts, def cityid){ + + sqlInstance.executeUpdate("delete from BIGDATA_POSITION_FJW where CITYID='${cityid}'") + def sqls = [] def index = 1 for(def d : districts){ From 2f1d5803ecdc4025955e542b075c42b85f92212c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 25 Oct 2021 16:47:02 +0800 Subject: [PATCH 69/80] update --- .../ronghe/BigdataCommunityGJRHBatch.groovy | 8 ++++--- .../script/commData/CommonDatautils.groovy | 22 +++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index c96c02ae1..d778576a2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -43,8 +43,8 @@ class BigdataCommunityGJRHBatch extends BaseBatch { @Override public void process(Object args) { - //"郑州","长沙","西安", - def citys = ["厦门","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉"]//CcbCommonUtils.citys; + //"厦门","北京" + def citys = ["上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉"]//CcbCommonUtils.citys; def rows = [] // addBigdataDistrictFJW() @@ -52,8 +52,10 @@ class BigdataCommunityGJRHBatch extends BaseBatch { for (def city : citys) { def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) + def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) + for(def dist : districts){ - def obj = CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) + def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) if(obj){ if(obj.alias && obj.alias.size()>0){ dist.commalias = obj.alias.get(0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index b8c48895f..821cc9765 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -677,6 +677,28 @@ class CommonDatautils { } + public static def findAllDistricMaptByCity(def city, def cat, def fields){ + + DBObject q = new BasicDBObject(); + q.put("visible",0) + q.put("cat", cat); + q.put("city", city); + + DBObject keys = new BasicDBObject(); + if(fields){ + for(def field : fields){ + keys.put(field, 1); + } + } + def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) + def retMap = [:] + for(def d : dbsour){ + retMap.put(d._id.toString(), d) + } + return retMap; + } + + public static def findAllDistrictByCity(def city, def cat, def fields){ DBObject q = new BasicDBObject(); From fc724c6a2fc6ac847c80a49d7caeff57afb65e75 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 25 Oct 2021 20:06:30 +0800 Subject: [PATCH 70/80] update --- .../com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 4 +++- .../ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index d7735dfe7..147a34823 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -266,8 +266,10 @@ class CcbCommonUtils { rows << map }); }catch(Exception e){ + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") findAllDistrictsByOracle(cityid) - e.printStackTrace() + println "错误:"+e.getMessage() } return rows diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index d778576a2..12bbc9acc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -43,8 +43,8 @@ class BigdataCommunityGJRHBatch extends BaseBatch { @Override public void process(Object args) { - //"厦门","北京" - def citys = ["上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉"]//CcbCommonUtils.citys; + //"厦门","北京","上海", + def citys = ["广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉"]//CcbCommonUtils.citys; def rows = [] // addBigdataDistrictFJW() From 22b3d1249e8aa5e006e43458a595ce8b4e2c7ba0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 26 Oct 2021 10:03:31 +0800 Subject: [PATCH 71/80] =?UTF-8?q?=E8=9E=8D=E5=90=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 239 ++++++++++++++++++ .../fangjia/script/gj/YouDaDataParser.groovy | 6 +- .../fangjia/script/gj/ccb/FileUtils.groovy | 4 +- 3 files changed, 244 insertions(+), 5 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy new file mode 100644 index 000000000..8dbfd559f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -0,0 +1,239 @@ +package com.fangjia.script.ccbevaluation.ronghe + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql + +import java.sql.ResultSet +import java.sql.ResultSetMetaData + +@BatchOwner(Watcher.GaoJuan) +class ExportDistrictRongHe extends BaseBatch{ + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static main(args) { + runBatch(new ExportDistrictRongHe(), args); + } + + @Override + void process(Object args) { + exportBigdataCommunityGJRH() + exportBigdataCommbuildstruGJ() + exportBigdataCommhousetypeGJ() + exportBigdataCommbuildyearGj() + exportBigdataCommfeeGj() + exportBigdataCommhouseuseGJ() + exportBigdataPositionGJ() + exportBigdataDistrictFJW() + } + + def exportBigdataCommunityGJRH(){ + def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.COMMNAME + data << r.ADDRESS + data << r.LANDAREA + data << r.BUILDAREA + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << r.UPDATETIME + data << r.COMMTYPE + data << r.COMMALIAS + data << r.STREET + data << r.STREETNO + data << r.IFNEW + data << r.INTRODUCTION + data << r.REMARK + data << r.CREATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMUNITY_GJRH_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + def exportBigdataCommbuildstruGJ(){ + def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDSTRU + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMBUILDSTRU_GJ_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + + def exportBigdataCommhousetypeGJ(){ + def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.HOUSETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMHOUSETYPE_GJ_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + + def exportBigdataCommbuildyearGj(){ + def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDYEAR + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMBUILDYEAR_GJ_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + + def exportBigdataCommfeeGj(){ + def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.COMPID + data << r.FEE + data << r.HOUSEUSE + data << r.BUILDTYPE + data << r.FEETYPE + data << r.NOTE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMFEE_FJW_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMFEE_GJ_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + + + def exportBigdataCommhouseuseGJ(){ + def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.HOUSEUSE + data << r.HOUSECNT + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMHOUSEUSE_GJ_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + + def exportBigdataPositionGJ(){ + def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.OBJECTTYPE + data << r.OBJECTID + data << r.POSITIONTYPE + data << r.SEQNO + data << r.LONGITUDE + data << r.LATITUDE + data << r.ALTITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POSITION_FJW_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_POSITION_GJ_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + + def exportBigdataDistrictFJW(){ + def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_FJW_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_DISTRICT_FJW_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index d9bf28e40..3a182be6a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-08-01" + static def statDate = "2021-09-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser();//友达二手房存入mongo + task.esfParser();//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2021年9月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2021年10月\\二手房\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 7d7f43a1c..3e4981856 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -22,7 +22,7 @@ class FileUtils { if(headers){ sb.append(headers.join("|@|")) - sb.append(System.getProperty("line.separator")); + sb.append("\n"); } out.write(sb.toString().getBytes(charset)); @@ -33,7 +33,7 @@ class FileUtils { if(row){ index++ sb.append(StringUtils.join(row.toArray(), "|@|")) - sb.append(System.getProperty("line.separator")); + sb.append("\n"); if(index%10000==0){ println index+"开始写入CSV "+new Date().toLocaleString() From 0a5b6e2854bd3b6ccca733345335cb35d2eb144a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 26 Oct 2021 10:51:06 +0800 Subject: [PATCH 72/80] =?UTF-8?q?=E5=AF=BC=E6=88=BF=E6=BA=90=E4=B8=80?= =?UTF-8?q?=E4=B8=AA=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ExportOracleData.groovy | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index fac7f79f8..7720d8f2b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -28,13 +28,11 @@ class ExportOracleData { // exportBigdataCommunity() exportBigdataForsale() - // exportGuidedPrice() // exportCommunityGuidedPrice() // exportBigdataDeal() // exportThirdPartyPrice() - // exportAvmEvaluationGJNB() // exportAVMEvaluationItemGJNB() // exportCommunityBasePrice() @@ -77,7 +75,7 @@ class ExportOracleData { def exportBigdataPoiData() { - def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj where cityId in (350200,150100)"; + def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj"; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -138,7 +136,7 @@ class ExportOracleData { } def exportCommunityBuileType() { - def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID in (350200,150100)"; + def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj"; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -167,7 +165,7 @@ class ExportOracleData { def exportBigdataDistrictData() { - def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj where cityId in (350200,150100)"; + def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj"; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -204,7 +202,7 @@ class ExportOracleData { def exportGuidedPrice() { - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where dealdate>=TO_DATE('2021-08-01', 'yyyy-MM-dd') and dealdate def data = [] @@ -469,9 +467,9 @@ class ExportOracleData { def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0003.dat" + def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0003.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] for(def city : citys){ From 8488257b10c08207bdd98a8180bd5dff36aaf086 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 26 Oct 2021 16:48:05 +0800 Subject: [PATCH 73/80] update --- .../script/ccbevaluation/ExportOracleData.groovy | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 7720d8f2b..8903b149b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -26,12 +26,13 @@ class ExportOracleData { // exportCommunityBuileType() // exportBigdataDistrictData() // exportBigdataCommunity() - exportBigdataForsale() - // exportGuidedPrice() -// exportCommunityGuidedPrice() -// exportBigdataDeal() + exportBigdataForsale() // exportThirdPartyPrice() +// exportBigdataDeal() + +// exportCommunityGuidedPrice() + // exportAvmEvaluationGJNB() // exportAVMEvaluationItemGJNB() @@ -477,7 +478,7 @@ class ExportOracleData { def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + - " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId=${cityCode} and PUBLISHTIME>=TO_DATE('2021-10-01', 'yyyy-MM-dd') and PUBLISHTIME From 1648e753bdeb65fb08547c43191dcf4979374ff4 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 27 Oct 2021 14:36:25 +0800 Subject: [PATCH 74/80] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=B9=B4=E6=9C=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleData.groovy | 35 ++++++++----------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 8903b149b..1401f34e8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -27,21 +27,18 @@ class ExportOracleData { // exportBigdataDistrictData() // exportBigdataCommunity() // exportGuidedPrice() - exportBigdataForsale() +// exportBigdataForsale() // exportThirdPartyPrice() // exportBigdataDeal() // exportCommunityGuidedPrice() - - -// exportAvmEvaluationGJNB() -// exportAVMEvaluationItemGJNB() -// exportCommunityBasePrice() - -// exportCommunityBasePriceGj() + exportAvmEvaluationGJNB() + exportAVMEvaluationItemGJNB() + exportCommunityBasePrice() + exportCommunityBasePriceGj() // exportCityPrice() // exportRegionPrice() -// exportLastFile() + exportLastFile() } @@ -237,7 +234,7 @@ class ExportOracleData { } def exportCommunityGuidedPrice() { - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where cityname='西安'"; + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj"; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -404,7 +401,7 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + //CcbFtpUtils.ftpUpload(fileName, curDate) } @@ -559,7 +556,7 @@ class ExportOracleData { def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" + "ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" + "DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" + - "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where (cityId=350200 or cityId=150100) and year_month<202108 "; + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH >= 202106"; def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -617,7 +614,7 @@ class ExportOracleData { def exportAVMEvaluationItemGJNB(){ def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" + "to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" + - "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where (cityId=350200 or cityId=150100) and year_month>=202108"; + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH >= 202106"; def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -661,7 +658,7 @@ class ExportOracleData { } def exportCommunityBasePrice(){ - def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where (cityId=350200 or cityId=150100) and year_month<202108"; + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj"; def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -704,7 +701,7 @@ class ExportOracleData { } def exportCityPrice(){ - def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where (cityId=350200 or cityId=150100) and year_month>=202108"; + def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj"; def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -733,7 +730,7 @@ class ExportOracleData { } def exportRegionPrice(){ - def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where (cityId=350200 or cityId=150100) and year_month>=202108"; + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ"; def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -764,12 +761,10 @@ class ExportOracleData { } def exportLastFile(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") - - def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def dataCodeFileName = "/ccb/${curDate}.ok" if(CommonDatautils.isWindow()){ - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curMonth}\\${curDate}.ok" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" } CcbFtpUtils.ftpUpload(dataCodeFileName, curDate) } From bc87a186c55c6cc05ee43e1c59777f9d57f57bc6 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 28 Oct 2021 19:21:01 +0800 Subject: [PATCH 75/80] =?UTF-8?q?=E5=AF=86=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/CcbFtpUtils.groovy | 2 +- .../script/ccbevaluation/ExportOracleData.groovy | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy index 5e4c8766e..329c4e425 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy @@ -10,7 +10,7 @@ class CcbFtpUtils { static def ftpServer = "192.168.0.45" static def ftpPort = 17495 static def ftpUser = "ccb" - static def ftpPassword = "0I1k8EAM+O8iA" + static def ftpPassword = "ccb!@#" static def ftpPath = "/home/ccb/output/" static SFTP sftp = new SFTP(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 1401f34e8..4f8f7d3b1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -29,16 +29,16 @@ class ExportOracleData { // exportGuidedPrice() // exportBigdataForsale() // exportThirdPartyPrice() -// exportBigdataDeal() + exportBigdataDeal() // exportCommunityGuidedPrice() - exportAvmEvaluationGJNB() - exportAVMEvaluationItemGJNB() - exportCommunityBasePrice() - exportCommunityBasePriceGj() +// exportAvmEvaluationGJNB() +// exportAVMEvaluationItemGJNB() +// exportCommunityBasePrice() +// exportCommunityBasePriceGj() // exportCityPrice() // exportRegionPrice() - exportLastFile() +// exportLastFile() } From 658c227d8c88980f689d82ab33044884e722b689 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 1 Nov 2021 10:46:07 +0800 Subject: [PATCH 76/80] =?UTF-8?q?=E5=85=AC=E5=8F=B8=E7=AE=80=E4=BB=8B?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-coresite/src/main/webapp/company/about.ftl | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/fangjia-coresite/src/main/webapp/company/about.ftl b/projects/fangjia-coresite/src/main/webapp/company/about.ftl index 0f386bf92..7772ed47d 100644 --- a/projects/fangjia-coresite/src/main/webapp/company/about.ftl +++ b/projects/fangjia-coresite/src/main/webapp/company/about.ftl @@ -54,7 +54,6 @@

鎴夸环缃戞垚绔嬩簬2006骞达紝闅跺睘浜庝笂娴蜂及瀹剁綉缁滅鎶�鏈夐檺鍏徃锛岃嚧鍔涗簬寤虹珛涓浗棰嗗厛鐨勬埧浜уぇ鏁版嵁搴揅PDB鍙婃帹鍔ㄥ熀浜庡湴鐞嗕綅缃殑浜哄伐鏅鸿兘鐨勫彂灞曞拰搴旂敤銆傚叕鍙镐互鈥滃彂鐜板湴鍧�浠峰�尖�濅负浣垮懡锛屽缁堢鎵垮瑙備弗璋ㄧ殑鎬佸害涓烘斂搴溿�侀摱琛屻�侀噾铻嶆満鏋勭瓑瀹㈡埛鎻愪緵澶ф暟鎹�佸缓妯°�佸挩璇㈢瓑鏈嶅姟銆傜洰鍓嶏紝鎵�绉疮鐨勬埧浜ф暟鎹凡瑕嗙洊鍏ㄥ浗367涓煄甯傚強鍦板尯锛屾妧鏈姏閲忓己澶э紝瀹㈡埛鏌ヨ閲忚涓氶鍏堛��

鍏徃鍦ㄨ捶鍓嶄及鍊笺�佸唴璇勭郴缁熴�佽捶鍚庢壒閲忓浼般�佷俊璐锋巿淇°�佸湴鍧�鍙嶆璇堛�佹暟鎹不鐞嗐�佺簿鍑嗚惀閿�绛夐鍩熼兘鏈夌潃涓板瘜鐨勭粡楠岋紝鎷ユ湁澶氶」涓撳埄锛岃璇勪负楂樻柊鎶�鏈紒涓氾紝璧㈠緱浜嗗浗瀹剁粺璁″眬銆佸缓璁鹃摱琛屻�佸啘涓氶摱琛屻�佸钩瀹夐泦鍥€�佸井浼楅摱琛岀瓑浼楀瀹㈡埛鐨勪俊璧栥��

-

2020骞达紝鍗椾含涓夋偊缃戠粶绉戞妧鏈夐檺鍏徃涓庢垜鍙歌揪鎴愬叏闈㈡垬鐣ュ悎浣滐紝骞舵帴鍙楁垜鍙稿鎵橈紝鍏ㄦ潈璐熻矗鎴戝徃浜у搧鎵嬫満鐗堚�滄埧浠风綉鈥濆叕浼楀彿鍜岀數鑴戠増鐨勨�滄埧浠烽�氣�濈殑甯傚満鎺ㄥ箍銆佸鎴峰紑鍙戜笌鍞悗鏈嶅姟锛涙湁鏉冪洿鎺ヤ笌瀹㈡埛绛捐涓婅堪浜у搧鐨勯攢鍞悎鍚岋紱鎴戝徃瀵瑰叾鎵�鏈夌敤鎴峰叏闈㈡彁渚涗骇鍝佸崌绾с�佹暟鎹洿鏂板強绾夸笂鍞悗绛夋湇鍔°��

鏁版嵁涓哄ぇ浼楁湇鍔�
From adef6f1ca8dfb79bb39d04e8f382b1b52f2430f8 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 1 Nov 2021 12:00:09 +0800 Subject: [PATCH 77/80] =?UTF-8?q?=E5=AF=BC=E5=B0=8F=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 2 +- .../com/fangjia/script/ccbevaluation/ExportOracleData.groovy | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 147a34823..3e5b20fbb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -281,7 +281,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","御海阳光") +// q.put("name","江南人家") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 4f8f7d3b1..30a10ae78 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -25,11 +25,11 @@ class ExportOracleData { // exportCommunityStaging() // exportCommunityBuileType() // exportBigdataDistrictData() -// exportBigdataCommunity() + exportBigdataCommunity() // exportGuidedPrice() // exportBigdataForsale() // exportThirdPartyPrice() - exportBigdataDeal() +// exportBigdataDeal() // exportCommunityGuidedPrice() // exportAvmEvaluationGJNB() From 4eb4476cfefff6ae5f04bf6261ea16681c8076ec Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 5 Nov 2021 18:03:40 +0800 Subject: [PATCH 78/80] =?UTF-8?q?=E6=88=BF=E6=BA=90=EF=BC=9A=E5=8C=97?= =?UTF-8?q?=E4=BA=AC=EF=BC=8C=E4=B8=8A=E6=B5=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleData.groovy | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 30a10ae78..7b4b6d11d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -25,10 +25,12 @@ class ExportOracleData { // exportCommunityStaging() // exportCommunityBuileType() // exportBigdataDistrictData() - exportBigdataCommunity() +// exportBigdataCommunity() // exportGuidedPrice() -// exportBigdataForsale() + exportBigdataForsale() // exportThirdPartyPrice() + +// exportThirdPartyPrice2() // exportBigdataDeal() // exportCommunityGuidedPrice() @@ -405,6 +407,39 @@ class ExportOracleData { } + def exportThirdPartyPrice2() { + def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from third_party_price"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.COMMNAME + data << r.DATA_SOURCE_ID + data << r.DATA_SOURCE + data << r.PRICE_DATE + data << r.COMMUNITY_PRICE + data << r.COMMUNITY_MAX_PRICE + data << r.COMMUNITY_MIN_PRICE + data << r.UPDATETIME + allRows << data + }) + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] +//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + //CcbFtpUtils.ftpUpload(fileName, curDate) + } + + def exportBigdataCommunity(){ def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" + "\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj ";//where cityId in (350200,150100) @@ -461,7 +496,7 @@ class ExportOracleData { def exportBigdataForsale(){ - def citys = CcbCommonUtils.citys; + def citys = ["北京","上海"]//CcbCommonUtils.citys; def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; From d26baea109c1b7e36e101dfca11ba06d12c640d2 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 5 Nov 2021 21:18:29 +0800 Subject: [PATCH 79/80] =?UTF-8?q?=E6=88=BF=E6=BA=90=20=2011=E4=B8=AA?= =?UTF-8?q?=E5=9F=8E=E5=B8=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/ExportOracleData.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 7b4b6d11d..792374cd7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -495,14 +495,14 @@ class ExportOracleData { } def exportBigdataForsale(){ - - def citys = ["北京","上海"]//CcbCommonUtils.citys; + //["北京","上海"] + def citys = ["重庆","天津","成都","沈阳","大连","深圳","杭州","合肥","无锡","武汉","广州"]//CcbCommonUtils.citys; def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0002.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0002.dat" } def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] for(def city : citys){ From c275971c7e1399b01b90c1d6d2205e9bf917987d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 11 Nov 2021 12:00:44 +0800 Subject: [PATCH 80/80] update --- projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl b/projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl index eab869c86..53fd8f659 100644 --- a/projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl @@ -1339,7 +1339,7 @@ 瀹ゅ彿 <#if evaluation.buildType!=3 && evaluation.buildType!=8 && evaluation.buildType!=0> - <#if evaluation.floorDesc!=""> + <#if evaluation.floorDesc?? && evaluation.floorDesc!=""> ${evaluation.floorDesc!}