From 110118291e34355de6d8f3205be1e943f34c970a Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 28 Oct 2022 18:45:01 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AD=A6=E6=A0=A1?= =?UTF-8?q?=E7=BB=8F=E7=BA=AC=E5=BA=A6=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/FillLngLat.groovy | 20 ++++---- .../temp/shenjian/util/BaiDuUtils.groovy | 48 +++++++++++++++++++ 2 files changed, 59 insertions(+), 9 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/BaiDuUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy index a0c3ec24f..3c5c2a52d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.temp.shenjian.poi import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.BaiDuUtils import com.fangjia.script.temp.shenjian.util.EsUtils import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.util.ElasticUtils @@ -15,25 +16,26 @@ import java.util.concurrent.Executors */ class FillLngLat { - static ExecutorService executorService = Executors.newFixedThreadPool(4) +// static ExecutorService executorService = Executors.newFixedThreadPool(4) static void main(String[] args) { - ElasticUtils.executeSqlQuery("select * from dws_poi where location is not null", { dataT -> + ElasticUtils.executeSqlQuery("select * from dws_poi where location is null", { dataT -> final def data = dataT - executorService.submit(new Runnable() { - @Override - void run() { +// executorService.submit(new Runnable() { +// @Override +// void run() { println(data) - def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address) + def lngLat = BaiDuUtils.parserByBaidu(data.city, data.region + data.name + data.address) + Thread.sleep(1000) if (lngLat != null && lngLat.size() == 2 ) { JSONObject jsonObject = new JSONObject() jsonObject.put("lon", Double.valueOf(lngLat['lng'])) jsonObject.put("lat", Double.valueOf(lngLat['lat'])) EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id)) } - } - }) +// } +// }) }) - executorService.shutdown() + //executorService.shutdown() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/BaiDuUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/BaiDuUtils.groovy new file mode 100644 index 000000000..e307bf28a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/BaiDuUtils.groovy @@ -0,0 +1,48 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.GeoUtils +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.DataUtil +import org.apache.commons.lang.StringUtils + +/** + * 百度工具类 + * + * @author shenjian + * @since 2022/10/28 + */ +class BaiDuUtils { + + def static final String KEY = "6w9SEHS3hnAFjmY2oEhqIDdgvLPKWXu5" + + /** + * 百度地址逆解析,返回高德坐标!!!!!!!!!!!!!!! + * @param city + * @param address + */ + def static parserByBaidu(def city, def address) { + Map params = new HashMap() + params.put("city", city) + if (StringUtils.isBlank(address)) { + return [] + } + params.put("address", address.replaceAll("null", "")) + params.put("output", "json") + params.put("ak", KEY) + params.put("callback", "showLocation") + + String content + content = new HttpRequest().get("https://api.map.baidu.com/geocoding/v3/", params, "utf-8") + //content = "showLocation&&showLocation({\"status\":0,\"result\":{\"location\":{\"lng\":131.04009309945693,\"lat\":45.78426770180344},\"precise\":1,\"confidence\":70,\"comprehension\":100,\"level\":\"地产小区\"}})" + content = content.replaceAll("showLocation&&showLocation\\(", "").replaceAll("\\)", "") + JSONObject json = JSONObject.parseObject(content) + if (json.status == 0 && json.result && json.result.location) { + def bd_lng = json.result.location.lng + def bd_lat = json.result.location.lat + Map locationMap = GeoUtils.bd09ToGcj02(bd_lat, bd_lng) + return ["lng": com.fangjia.core.util.DataUtil.getDoubleRound(6, locationMap['lng']), "lat": com.fangjia.core.util.DataUtil.getDoubleRound(6, locationMap['lat'])] + } + return [] + } +} From 321baf13d86be1c13a90c904619a8f502d496e95 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 5 Nov 2022 11:30:47 +0800 Subject: [PATCH 2/3] =?UTF-8?q?252=E5=9F=8E=E6=88=BF=E6=BA=90=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index ca517f31a..8a8b30846 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -68,14 +68,14 @@ class BigdataForsaleBatch extends BaseBatch{ public void process(Object args) { def index = 0 def citys = CcbCommonUtils.getRunningCitys(args) - statCity(citys) def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019); +// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221019); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 From f2a969b7e0cc113a66d2fa3303d360bae3ee4d3c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 5 Nov 2022 11:32:14 +0800 Subject: [PATCH 3/3] update --- projects/fangjia-coresite/pom.xml | 8 +++ .../src/main/webapp/header/v5/_footer.ftl | 4 +- .../ccbevaluation/ExportOracleData.groovy | 40 +++++++-------- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 10 ++-- .../fuzhou/DistrictPriceExportBatch.groovy | 49 ++++++++++++++----- 5 files changed, 73 insertions(+), 38 deletions(-) diff --git a/projects/fangjia-coresite/pom.xml b/projects/fangjia-coresite/pom.xml index 05fa7f275..72cefc4b9 100644 --- a/projects/fangjia-coresite/pom.xml +++ b/projects/fangjia-coresite/pom.xml @@ -242,5 +242,13 @@ 1.0.0 jdk18 + + + + + + + + diff --git a/projects/fangjia-coresite/src/main/webapp/header/v5/_footer.ftl b/projects/fangjia-coresite/src/main/webapp/header/v5/_footer.ftl index 323aeb051..b1a1c4956 100644 --- a/projects/fangjia-coresite/src/main/webapp/header/v5/_footer.ftl +++ b/projects/fangjia-coresite/src/main/webapp/header/v5/_footer.ftl @@ -38,7 +38,9 @@ 琛屼笟鍔ㄦ�� 鏁版嵁鍒嗘瀽 鎴愬姛妗堜緥 - 鍏徃鍔ㄦ�� + 鍚勮矾瀹� + 鎴夸环閫� + 鍦版暟 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index fed4c92bd..06e78b250 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -25,29 +25,29 @@ class ExportOracleData { def run(args) { citys = CcbCommonUtils.getRunningCitys(args) -// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] //不会变 - exportEvaluationRateData() - exportBigdataPoiData()// - exportBigdataDistrictData()// +// exportEvaluationRateData() +// exportBigdataPoiData()// +// exportBigdataDistrictData()// +// +// exportGuidedPrice()//随人工改变// +// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// +// +//// //处理小区情况 +// exportBigdataCommunity()// +// exportCommunityStaging()//没cityid +// exportCommunityBuileType()// - exportGuidedPrice()//随人工改变// - exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// - -// //处理小区情况 - exportBigdataCommunity()// - exportCommunityStaging()//没cityid - exportCommunityBuileType()// - - exportThirdPartyPrice2(lastMonth, new_dists)// - exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 +// exportThirdPartyPrice2(lastMonth, new_dists)// +// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 //第三方网站权重表 - exportWebsiteRate(); +// exportWebsiteRate(); //计算结果相关 @@ -63,11 +63,11 @@ class ExportOracleData { // exportCommunityPdescribeGj(curMonth) // exportCommCaseDetailGj(curMonth) -// def curDate = curMonth.toString()+"28" -// SSHHelper jschCommand = new SSHHelper(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); + def curDate = curMonth.toString()+"28" + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); //ok文件 -// exportLastFile() + exportLastFile() } @@ -377,7 +377,7 @@ class ExportOracleData { "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; - allRows = dealData(sql) + allRows.addAll(dealData(sql)) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 6b1009e6c..0b40eb938 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202107 - def endDate = 202109 + def sDate= 202209//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202207 + def endDate = 202209 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -502,8 +502,8 @@ class ThirdPartyPriceBatch extends BaseBatch { def findThirdPartyPrice(def city, def statMap){ def q = new BasicDBObject() q.put("city", city) - - q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + q.put("s_date", endDate) + // q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def map = [:] for(def db : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 45f3cbe35..517bfef5c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -41,10 +41,18 @@ class DistrictPriceExportBatch extends BaseBatch{ def headers = ["id","comm_id","city","district","name","avg_price","year_month"] - def ret = createDatas("福州") - - saveToDB(ret) + def citys = CommonDatautils.findAllCity() + citys.remove("福州"); + citys.add(0, "福州") + def fz_datas = [] + for(def city : citys){ + def ret = createDatas("福州") + saveToDB(ret) + if(city == "福州"){ + fz_datas = ret + } + } def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv" def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512" @@ -52,7 +60,7 @@ class DistrictPriceExportBatch extends BaseBatch{ fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv" dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512" } - def sb = appendResult(headers, ret) + def sb = appendResult(headers, fz_datas) DataHandleUtils.outPutFile(fileName, sb, charset); FzFtpUtils.ftpUpload(fileName) //生成加密数据文件并上传 @@ -102,7 +110,12 @@ class DistrictPriceExportBatch extends BaseBatch{ // def districts = CommonDatautils.findAllDistricts(city, "district", null); // def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); // districts.addAll(removeDists) - def districts = getFzDistricts(city) + def districts = [] + if(city == "福州"){ + districts = getFzDistricts(city) + }else{ + districts = CommonDatautils.findAllDistricts(city, "district", null); + } def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10" def dataStr = DateUtils.add(endDateStr, -30) @@ -127,14 +140,26 @@ class DistrictPriceExportBatch extends BaseBatch{ curPrice = dbo.get("avg_price") if(lastMonthPrice && curPrice && dbo.value_type!=1){ def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice - if(Math.abs(markup)>3){ - def randMarkup = nextDouble(0, 3) - if(markup < 0){ - markup = randMarkup*(-1) - }else{ - markup = randMarkup + if(city == "福州"){ + if(Math.abs(markup)>3){ + def randMarkup = nextDouble(0, 3) + if(markup < 0){ + markup = randMarkup*(-1) + }else{ + markup = randMarkup + } + curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) + } + }else{ + if(Math.abs(markup)>10){ + def randMarkup = nextDouble(0, 10) + if(markup < 0){ + markup = randMarkup*(-10) + }else{ + markup = randMarkup + } + curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) } - curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) } } }else{