From c363a8d0caad1a16e31baebad27bbb78c250b429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Tue, 18 May 2021 12:51:48 +0800 Subject: [PATCH 01/21] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=A1=97=E9=81=93?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/PropertyController.java | 2 +- .../com/fangjia/script/wenyan/Test4.groovy | 103 +++++++++++------- .../fangjia/script/wenyan/test/Test1.groovy | 5 + 3 files changed, 71 insertions(+), 39 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 93172390a..37b785212 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -671,7 +671,7 @@ public class PropertyController { && !"1007486".equals(userId) && !"1007533".equals(userId) && !"1007630".equals(userId) && !"1007705".equals(userId) && !"1007739".equals(userId) && !"1007736".equals(userId) && !"1007741".equals(userId) && !"1007742".equals(userId) && !"1007743".equals(userId) - && !"1007747".equals(userId) + && !"1007747".equals(userId) && !"1007757".equals(userId) && model.getDetailLevel() != null && model.getDetailLevel() != 0) {// 临时权限控制 mav.setModel(BasicModel.buildUnAuthorizedModel()); return mav; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy index 95d41237b..235253d60 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy @@ -4,6 +4,7 @@ import com.fangjia.core.ProvinceData import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils @@ -11,45 +12,71 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils -DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018) -def count = 0 -List> data = Collections.synchronizedList([]) -data << ["ID","城市","区域","小区名","经度","纬度","街道"] - -CommonUtils.findAllCity().each {def city-> - DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> - if(StringUtils.isBlank(item.township) && item.lng){ - count++ - def lng = item.lng/10000000000d - def lat = item.lat/10000000000d - def location = "${lng},${lat}" - JSONObject resp = GaoDeApiUtils.geocode(location) - def row = [] - row << item._id.toString() - row << item.city - row << item.region - row << item.name - row << lng - row << lat - BasicDBObject dbo = new BasicDBObject() - dbo.id = item._id.toString() - dbo.city = item.city - dbo.region = item.region - dbo.name = item.name - dbo.lng = lng - dbo.lat = lat - if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){ - row << resp.regeocode.addressComponent.township - dbo.township = resp.regeocode.addressComponent.township - } - println(row) - data << row - SEAWEED_TOWNSHIP.insert(dbo) - } - }) -} -ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据") +//DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018) +//def count = 0 +//List> data = Collections.synchronizedList([]) +//data << ["ID","城市","区域","小区名","经度","纬度","街道"] +// +//CommonUtils.findAllCity().each {def city-> +// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> +// if(StringUtils.isBlank(item.township) && item.lng){ +// count++ +// def lng = item.lng/10000000000d +// def lat = item.lat/10000000000d +// def location = "${lng},${lat}" +// JSONObject resp = GaoDeApiUtils.geocode(location) +// def row = [] +// row << item._id.toString() +// row << item.city +// row << item.region +// row << item.name +// row << lng +// row << lat +// BasicDBObject dbo = new BasicDBObject() +// dbo.id = item._id.toString() +// dbo.city = item.city +// dbo.region = item.region +// dbo.name = item.name +// dbo.lng = lng +// dbo.lat = lat +// if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){ +// row << resp.regeocode.addressComponent.township +// dbo.township = resp.regeocode.addressComponent.township +// } +// println(row) +// data << row +// SEAWEED_TOWNSHIP.insert(dbo) +// } +// }) +//} +//ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据") +//ExcelUtils.modify("D:\\街道补充数据.xlsx",{List item -> +// if(item["G"] && item["H"] == null){ +// DbUtils.seaweed.update(new BasicDBObject("_id", item["A"]), new BasicDBObject('$set', new BasicDBObject('township', item["G"]))) +// } +// println(item) +//}) +//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\错误经纬度.xlsx",{List item-> +// def location = "${item["A"]},${item["B"]}" +// JSONObject resp = GaoDeApiUtils.geocode(location) +// if(resp.regeocode && resp.regeocode.formatted_address){ +// item['G'] = resp.regeocode.formatted_address +// println(item) +// } +//}) +//def count = 0 +//CommonUtils.findAllCity().each {def city-> +// DbUtils.executeQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> +// if(item.township){ +// count++ +// } +// }, new BasicDBObject("township", 1)) +//} +//println(count) +ElasticUtils.executeSqlQuery("select count(*), p_date from seawater where p_date > 20210501 group by p_date",{ + println(it) +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy index 401d647e4..ff32247d7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy @@ -9,6 +9,11 @@ import com.fangjia.core.util.HttpRequest import com.fangjia.script.util.ExcelUtils import net.sf.json.JSONObject +def strt(String addr){ + +} + + ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item-> def city = item["A"] def address1 = item["B"] From 0e354a525d28eb294ba6c7b6aacfdce4be701616 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 24 May 2021 09:54:57 +0800 Subject: [PATCH 02/21] =?UTF-8?q?=E7=A6=8F=E5=B7=9E=E7=BD=91=E7=AB=99?= =?UTF-8?q?=E5=8F=8A=E4=BD=BF=E7=94=A8=E6=9C=9F=E9=99=90=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fuzhou/DistrictInfoExportBatch.groovy | 3 +++ .../fuzhou/ThirdPriceExportBatch.groovy | 10 +++---- .../fangjia/script/gj/YouDaDataParser.groovy | 4 +-- .../gj/district/FangjiaClientUtils.groovy | 6 ++++- .../thirdPriceCrawl/CrawalDBUtil.groovy | 5 ++++ .../thirdPriceCrawl/DataStandardBatch.groovy | 2 +- .../ThirdDataToGatherBatch.groovy | 2 +- .../fangjia/script/util/OpenApiClient.groovy | 27 +++++++++++++++++++ .../yinjihaun/amap/GaoDeMapApiData.groovy | 26 ++++++++++++++++++ 9 files changed, 75 insertions(+), 10 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index 95e40fb9b..fdf2ce5f9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -165,6 +165,9 @@ class DistrictInfoExportBatch extends BaseBatch{ rets << "非钢混" } } + if(rets.size()==0){ + rets << "钢混" + } return rets.join(",") } return "钢混"; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy index c08426b9c..6a36f086d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy @@ -18,7 +18,7 @@ import org.apache.commons.codec.digest.DigestUtils */ class ThirdPriceExportBatch extends BaseBatch{ - def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")); + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -156,12 +156,12 @@ class ThirdPriceExportBatch extends BaseBatch{ if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){ retMap.cityhouse_esf_price = d.cityhouse_esf_price } - if(!retMap.fangtan_esf_price && d.fangtan_esf_price){ - retMap.fangtan_esf_price = d.fangtan_esf_price + if(!retMap.fangtan_esf_price && d.jiwu_esf_price){//使用吉屋替换房探 待抓取 + retMap.fangtan_esf_price = d.jiwu_esf_price } - if(!retMap.fangdoudou_esf_price && d.fangdoudou_esf_price){ - retMap.fangdoudou_esf_price = d.fangdoudou_esf_price + if(!retMap.fangdoudou_esf_price && d.get("365house_esf_price")){//使用淘房365替换房多多 + retMap.fangdoudou_esf_price = d.get("365house_esf_price") } if(!retMap.maitian_esf_price && d.maitian_esf_price){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 32b251e77..25c631832 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -47,7 +47,7 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-03-01" + static def statDate = "2021-04-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2021年5月\\二手房\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index e0f5df506..033464eb5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -9,7 +9,11 @@ class FangjiaClientUtils { public static def getMapData(def city, def region, def name, def address, def types, def mapCode){ JSONObject jsonObj = null try { - def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode + def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode + //def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode + if(address){ + url = url + "&address="+address + } // def url = "http://localhost:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode def content = new HttpRequest().newGet(url, null, "utf-8") jsonObj = JSONObject.fromObject(content) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalDBUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalDBUtil.groovy index 4f6bc253e..c0cc1f468 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalDBUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalDBUtil.groovy @@ -25,6 +25,7 @@ class CrawalDBUtil { static DBCollection data_process_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "data_process") static DBCollection average_statistics_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "average_statistics") static DBCollection map_district_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "map_district_process") + static DBCollection districtDetailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail") public static getSeedsCol() { return seedsColl; @@ -67,6 +68,10 @@ class CrawalDBUtil { return map_district_col; } + public static getDistrictDetailColl(){ + return districtDetailColl; + } + static def updateSeedsCount(def id, def count) { DBObject uquery = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 3a8ee77b2..c8ab1b6d8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -61,7 +61,7 @@ class DataStandardBatch extends BaseBatch{ def static final SHORT_DATE = Utils.getLastMonth() def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6] - def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"] + def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) static { thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 67e67e520..5f96db29c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -30,7 +30,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ "房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf", "链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new", "乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf", - "房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf" + "房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf" ] public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index f35379b65..a23842026 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -309,6 +309,31 @@ public class OpenApiClient { return null } + + public static JSONObject addressStandard(String city, String address) { + if (StringUtils.isBlank(city) || StringUtils.isBlank(address)) { + throw new IllegalArgumentException(); + } + + try { + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8") + JSONObject result = JSONObject.fromObject(content) + if (result.getInt("code") == 200) { + return result.getJSONObject("result"); + } else if (result.getInt("code") == 500) { + println("${city} ${address} ${result}") + } + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + public static JSONObject evaluateByGeo(String city, def lat, def lng, String buildDate, String estateType, String geoType) { try { Map params = new HashMap(); @@ -1214,4 +1239,6 @@ public class OpenApiClient { return null; } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeMapApiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeMapApiData.groovy index 4d472474c..c9e44b3e0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeMapApiData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeMapApiData.groovy @@ -223,6 +223,32 @@ class GaoDeMapApiData { doSearchByGaoDe(type, lng, lat, radius, page) } } + + def static doSearchByGaoDe(String type, Double lng, Double lat, int radius, int page, int pageSize) { + Map params = new HashMap() + params.put("key", GAO_DE_AK) + params.put("types", type) + params.put("location", Double.toString(lng)+ ","+ Double.toString(lat)) + params.put("radius",radius) + params.put("offset",pageSize) + params.put("page",page) + String content + try { + content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8") + } catch (Exception e) { + content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8") + } + JSONObject jsonObj = JSONObject.fromObject(content) + + if(jsonObj.get("infocode").equals("10000")){ + JSONArray pois = jsonObj.get("pois") + return pois + } else { + println jsonObj.get("infocode")+"exit........" + doSearchByGaoDe(type, lng, lat, radius, page, pageSize) + } + return null + } private synchronized static fzData(JSONArray pois) { pois.each { poi -> From f3220beef36f8af94c1ad1bfb5f0ed190473c6ee Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 2 Jun 2021 15:36:07 +0800 Subject: [PATCH 03/21] update --- .../com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java | 5 +++-- .../fangjia/open/model/PropertyEvaluationModel.java | 10 ++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java index 9ff57486c..10a5791e3 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java @@ -199,7 +199,8 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i } private JSONObject query(String id,String category, String city, String region, String name, List alias, - List regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice, Integer room, String root,String sort, boolean order, int start, int limit,String table){ + List regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice, + Integer room, String root,String sort, boolean order, int start, int limit,String table){ long time1 = System.currentTimeMillis(); JSONObject reqBody = new JSONObject(); JSONObject q = new JSONObject(); @@ -313,7 +314,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i if(endPrice != null){ temp2.put("lte", endPrice); } - param.put("total_price", temp2); + param.put("avg_price", temp2); temp.put("range", param); jsonList.add(temp); } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java index 61f0618f7..5953fc3a3 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java @@ -284,6 +284,8 @@ public class PropertyEvaluationModel extends FangjiaApiModel { private String user; + private Integer avgPrice; + public void setDecorationAge(int decorationAge) { this.decorationAge = decorationAge; @@ -895,4 +897,12 @@ public class PropertyEvaluationModel extends FangjiaApiModel { public void setAtticSize(Double atticSize) { this.atticSize = atticSize; } + + public Integer getAvgPrice() { + return avgPrice; + } + + public void setAvgPrice(Integer avgPrice) { + this.avgPrice = avgPrice; + } } From f3ab53f6ebab12f780a2a93f091f95b8b5d2f265 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 2 Jun 2021 16:27:44 +0800 Subject: [PATCH 04/21] =?UTF-8?q?=E4=BA=BA=E5=B7=A5=E5=AE=A1=E6=9F=A5?= =?UTF-8?q?=E5=9C=B0=E5=9D=80=20=E6=B7=BB=E5=8A=A0=E5=8C=BA=E5=9F=9F?= =?UTF-8?q?=E5=8F=8A=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/fangjia/open/controllers/GuzhiController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java index 2f3621338..c2c3840e0 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java @@ -161,7 +161,7 @@ public class GuzhiController { RestfulModelAndView mv = new RestfulModelAndView(); mv.setModel(BasicModel.buildOKModel()); - String msg = "【房价网-人工审查地址】接收到新的人工审查地址信息,请求请及时处理!城市:" + model.getCity() + String msg = "【房价网-人工审查地址】接收到新的人工审查地址信息,请求请及时处理!城市:" + model.getCity()+" 区域:"+model.getDistrict() +" 小区名称:"+(model.getName()!=null?model.getName():"") + " 产证地址:" + model.getAddress(); DingDingMessageUtil.sendTextMessage("065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60", msg); return mv; From 5d09b016db9d59de62aa836c5a106ea0ce22b87e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 2 Jun 2021 16:30:45 +0800 Subject: [PATCH 05/21] log --- .../main/java/com/fangjia/open/controllers/GuzhiController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java index c2c3840e0..870dda5e3 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java @@ -156,7 +156,7 @@ public class GuzhiController { return mv; } - + //福州农商行人工录入 public Object checkAction(@Valid GuzhiCheckModel model){ RestfulModelAndView mv = new RestfulModelAndView(); From 851221de802a30bea5bae7ab39fedf720472b02a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 4 Jun 2021 11:28:23 +0800 Subject: [PATCH 06/21] =?UTF-8?q?=E6=96=B0=E5=A2=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/fuzhou/Test1.groovy | 133 ++++ .../script/gj/ccb/CcbCityPriceStat.groovy | 77 +++ .../script/gj/chongqingblank/DataMatch.groovy | 322 +++++++++ .../gj/chongqingblank/EvaluationPrice.groovy | 47 ++ .../gj/chongqingblank/TestMatchResult.groovy | 629 ++++++++++++++++++ .../script/gj/chongqingblank/TestPrice.groovy | 89 +++ .../script/gj/data/RegionExport.groovy | 29 + .../script/gj/es/EsMonthHouseStat.groovy | 236 +++++++ .../fangjia/script/gj/es/EsSelectTest.groovy | 66 ++ .../script/gj/location/LocationTest.groovy | 171 +++++ .../thirdPriceCrawl/SynchDistrictPrice.groovy | 106 +++ 11 files changed, 1905 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/EvaluationPrice.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestMatchResult.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestPrice.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy new file mode 100644 index 000000000..cf1e4950f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -0,0 +1,133 @@ +package com.fangjia.script.fuzhou + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) +def count0 = 0 +def count1 = 0 +def count2 = 0 +def count3 = 0 +def count4 = 0 +def count5 = 0 +def count6 = 0 +def count7 = 0 +def count8 = 0 +def total=4943 +for(def d : datas){ + def list = [] + if(d['G']){ + list << d['G'] + } + if(d['H']){ + list << d['H'] + } + if(d['I']){ + list << d['I'] + } + if(d['J']){ + list << d['J'] + } + if(d['K']){ + list << d['K'] + } + if(d['L']){ + list << d['L'] + } + if(d['M']){ + list << d['M'] + } + if(d['N']){ + list << d['N'] + } + if(d['O']){ + list << d['O'] + } + if(list.size()==0){ + count0++ + } + if(list.size()==1){ + count1++ + } + if(list.size()==2){ + count2++ + } + if(list.size()==3){ + count3++ + } + if(list.size()==4){ + count4++ + } + if(list.size()==5){ + count5++ + } + + if(list.size()==6){ + count6++ + } + if(list.size()==7){ + count7++ + } + if(list.size()==8){ + count8++ + } +} + +println("count0:"+ (DataHandleUtils.getDoubleRound(2, count0*100.0/total))) +println("count1:"+ (DataHandleUtils.getDoubleRound(2, count1*100.0/total))) +println("count2:"+ (DataHandleUtils.getDoubleRound(2, count2*100.0/total))) +println("count3:"+ (DataHandleUtils.getDoubleRound(2, count3*100.0/total))) +println("count4:"+ (DataHandleUtils.getDoubleRound(2, count4*100.0/total))) +println("count5:"+ (DataHandleUtils.getDoubleRound(2, count5*100.0/total))) +println("count6:"+ (DataHandleUtils.getDoubleRound(2, count6*100.0/total))) +println("count7:"+ (DataHandleUtils.getDoubleRound(2, count7*100.0/total))) +println("count8:"+ (DataHandleUtils.getDoubleRound(2, count8*100.0/total))) + + + +////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]); +//def total = 8018 +//def names = ["诸葛找房":"zhugezhaofang_esf_price","安居客":"anjuke_esf_price","搜房":"soufang_esf_price","链家":"lianjia_esf_price","城市房产":"cityhouse_esf_price","房探":"fangtan_esf_price","房多多":"fangdoudou_esf_price","麦田":"maitian_esf_price", "58同城":"58tc_esf_price"] +// +//def rets = [] +//def sDate = 202011 +//rets << ["时间","诸葛找房","安居客","搜房","链家","城市房产","房探","房多多","麦田", "58同城"] +//def ret = [] +//ret << sDate +//names.each{k,v-> +// def crawlCount = crawlCount(k, sDate) +// def count = count(v, sDate) +// println k +" "+DataHandleUtils.getDoubleRound(2, count*100.0/total)+"("+crawlCount+")" +// ret << DataHandleUtils.getDoubleRound(2,count*100.0/total)+"("+crawlCount+")" +//} +//rets.add(ret) +//ExcelUtils.write("d:\\数据\\第三方均价覆盖率1.xlsx",rets,"数据") +// +//def crawlCount(k, sDate){ +// def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); +// def q = new BasicDBObject() +// q.put("city","福州") +// q.put("s_date", sDate) +// q.put("source",k) +// return thirdPartyColl.count(q) +//} +// +// +//def count(name, sDate){ +// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") +// def q = new BasicDBObject() +// q.put("city","福州") +// q.put("s_date", sDate) +// q.put(name, new BasicDBObject('$gt', 0)); +// return AVERAGE_STATISTICS.count(q) +//} + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy new file mode 100644 index 000000000..c60a811b5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy @@ -0,0 +1,77 @@ +package com.fangjia.script.gj.ccb + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class CcbCityPriceStat { + + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + static main(args) { + def task = new CcbCityPriceStat() + task.run() + } + def run(){ + def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def rets = [] + for(def city : citys){ + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30")) + def cityRets = [] + for(def dist : districts){ + def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01") + def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name) + if(tradeAvgPrice && weekendPrice){ + def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()] + cityRets << row + println row.toString() + } + } + print city +" "+ cityRets.size() + rets.addAll(cityRets) + } + + println "总量:"+rets.size() + ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有") + } + + + def findTrades(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + q.put("district_name", new BasicDBObject('$in', dist.alias)) + }else{ + q.put("district_name", dist.name) + } + + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate))) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def avg_price = 0 + def total_price = 0 + def count = 0 + + for(def d : dbcorsor){ + if(d.avg_price){ + total_price = total_price + d.avg_price + count++ + } + } + if(count > 3){ + avg_price = total_price/count + } + return avg_price + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy new file mode 100644 index 000000000..9f84f6479 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy @@ -0,0 +1,322 @@ +package com.fangjia.script.gj.chongqingblank + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import com.fangjia.script.gj.* + +import java.util.regex.Matcher +import java.util.regex.Pattern + +class DataMatch { + + static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com") + + + + static main(args) { + def task = new DataMatch() +// task.run1() + task.run2() +// task.run3() +// println task.findAddress("昌州街道棠香路") + +// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") + } + + def run1(){ + def fileName = "D:\\数据\\重庆农商行\\估值2-V5.xlsx" + //matchRegionValue(fileName); + matchDistrict(fileName) +// ExcelUtils.multiThreadingModify(fileName,{d-> +// if(d['O']=="新增"){ +// +// d['L'] = d['P'] +// d['M'] = d['Q'] +// d['N'] = d['R'] +// } +// },0) + + } + + def parseAddress(def fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + + },0) + } + + def matchDistrict(fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + def region = d['Q'] + def city = d['P'] +// if(!d['I'] && !d['J']){ +// city = d['B'] +// region = d['C'] +// } + if(d['R'] && !d['G']){ + def results = null +// if(d['E'] && isStreet(d['E'])){//加上效果并没有想象中的好 +// results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, d['D']) +// } + if((results && results.precise<0) || !results){ + results = OpenApiClient.addressMatch(city, d['R'], "property", false, region, null) + } +// if((d['E'] && results && results.precise<0) || !results){ +// results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, d['D']) +// } + if(results && results.precise>=0){ + d['K'] = results.city + d['L'] = results.district + d['M'] = results.name + d['N'] = results.address + } + } + },0); + } + + def run2(){ + def fileName = "D:\\数据\\重庆农商行\\估值2-V2.xlsx" +// matchRegionValue(fileName); + matchDistrict2(fileName) + } + + + def matchDistrict2(fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + def region = d['I'] + def city = d['J'] + if(!d['I']){ + city = d['J'] + } + if(city && d['E'] && !d['O'] ){ + def addressResult = OpenApiClient.addressStandard(city, d['E']) + if(addressResult){ + if(addressResult.street && addressResult.streetNumber){ + d['H'] = addressResult.street+addressResult.streetNumber + } + } + def results = null + if(d['H']){ + results = OpenApiClient.addressMatch(city, d['H'], "property", false, region, null) + } + + if((d['E'] && results && results.precise<0) || !results){ + results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, null) + } + + if(results && results.precise>=0){ + d['K'] = results.city + d['L'] = results.district + d['M'] = results.name + d['N'] = results.address + } + } + },0); + } + + def run3(){ + def fileName = "D:\\数据\\重庆农商行\\估值3.xlsx" +// matchDistrict3(fileName) + ExcelUtils.multiThreadingModify(fileName,{d-> + if(d['I']){ + d['I'] = d['I'].toString().replace("省","").replace("市","") + } + if(d['J'] && d['I']){ + d['J'] = RegionData.formatRegion(d['I'], d['J']) + } + },0) + } + + + def matchDistrict3(fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + def address = d['B'] + if(address) { + def arrs = address.split("->") + def city = null + def region = null + def addr = null + if (arrs && arrs.size() == 4) { + city = arrs[0] + region = arrs[2] + addr = arrs[3] + d['I'] = city + d['J'] = region + } else if (arrs && arrs.size() == 2) { + def parseCity = CommonUtils.parseCityInAddress(arrs[1]) + city = parseCity + d['I'] = parseCity + addr = arrs[1] + } + if (addr && !d['O']) { + def addressResult = OpenApiClient.addressStandard(city, addr) + if (addressResult) { + if (!region) { + region = addressResult.district + d['J'] = region + } + if (addressResult.street && addressResult.streetNumber) { + d['H'] = addressResult.street + addressResult.streetNumber + } + } + def results = null + def name = null + if (d['C']) { + name = d['C'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "") + .replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "") + .replace("1期", "一期").replace("2期", "二期").replace("3期", "三期") + .replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区") + .replace("4区", "四区").replace("5区", "五区").replace("a区", "A区") + .replace("B区", "B区").replace("c区", "C区").replace("d区", "D区") + } + if (name) { + results = OpenApiClient.addressMatch(city, name, "property", false, region, null) + } + + if ((addr && results && results.precise < 0) || !results) { + results = OpenApiClient.addressMatch(city, addr, "property", false, region, name) + } + if (results && results.precise >= 0) { + d['K'] = results.city + d['L'] = results.district + d['M'] = results.name + d['N'] = results.address + } + } + } + + },0); + } + + + + def matchRegionValue(fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + def code = d['C'] + if(code && code.toString().isNumber()){ + if(code.toInteger().toString()=="500226"){ + d['I'] = "荣昌" + d['J'] = "重庆" + }else if(code.toInteger().toString()=="500223"){ + d['I'] = "潼南" + d['J'] = "重庆" + }else{ + def data = findRegionCodeValue(code.toInteger().toString()); + if(data){ + d['I'] = data.fangjia_district + d['J'] = data.fangjia_city + }else{ + data = findCityCodeValue(code.toInteger().toString()) + if(data){ + d['J'] = data.fangjia_city + } + } + } + + } + + },0) + } + + def findCityCodeValue(value){ + def q = new BasicDBObject() + q.put("city_code", value) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def count = dbcursor.count() + if(count >1){ + def data = null + for(def db : dbcursor){ + data = db + break + } + return data; + } + } + + def findRegionCodeValue(value){ + def q = new BasicDBObject() + q.put("district_code", value) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def count = dbcursor.count() + if(count == 1){ + def data = null + for(def db : dbcursor){ + data = db + break + } + return data; + } + } + + def findAddress(String address) { + //Pattern ROAD_PATTERN = Pattern.compile('.*[路|大道|巷|胡同|街]\\d+号'); + Pattern ROAD_PATTERN = Pattern.compile('.[里,路,道,街,巷,路段,胡同].?\\d+.?\\d+[号,弄]|.[里,路,道,街,巷,路段,胡同].?\\d+.?[号,弄]') + Matcher m = ROAD_PATTERN.matcher(address) + String newAddress + if (m.find()) { + newAddress = m.group() + } + return newAddress + } + + def run2MatchV2(fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + def region = d['I'] + def city = d['J'] + if(!d['I'] && !d['J']){ + city = d['B'] + region = d['C'] + } + if(d['D']) { + def name = d['D'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "") + .replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "") + .replace("1期", "一期").replace("2期", "二期").replace("3期", "三期") + .replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区") + .replace("4区", "四区").replace("5区", "五区").replace("a区", "A区") + .replace("B区", "B区").replace("c区", "C区").replace("d区", "D区") + + def result = OpenApiUtils.matchDetailDistrict(city, name, region, true) + if (!result && d['E']) { + if (findAddress(d['E'])) { + result = OpenApiUtils.matchDetailDistrict(city, d['E'], region, true) + } + } + if(result){ + def matchResult = result.matchResult + if (result.credit > 0.5 && result.matchResult && !result.matchResult.match) { + matchResult = amendMatch(result, d['E']) + } + if (matchResult.match) { + d['AQ'] = result.city + d['AR'] = result.district + d['AS'] = result.name + d['AT'] = result.address + } + } + } + + },0) + } + + def amendMatch(result, address){ + def matchResult = result.matchResult + if(result.credit > 0.65 && matchResult.distance < 2500){ + matchResult.match = true + }else if(result.credit>0.5 && address && result.address && findAddress(result.address)){ + if(result.address.contains(address) || address.contains(result.address)){ + matchResult.match = true + } + }else if(address && address.contains(result.district) && address.contains(result.name)){ + matchResult.match = true + } + return matchResult + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/EvaluationPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/EvaluationPrice.groovy new file mode 100644 index 000000000..90eaa4e13 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/EvaluationPrice.groovy @@ -0,0 +1,47 @@ +package com.fangjia.script.gj.chongqingblank + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import net.sf.json.JSONObject + +class EvaluationPrice { + + static main(args) { + def task = new EvaluationPrice() + task.run() + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\估值3-V3.xlsx",{ d-> + if(d['K'] && d['M']){ + def priceMap = getEvaluationPrice(d['J'], d['K'], d['L'], d['D'].toString()) + if(priceMap){ + d['E'] = priceMap.avgPrice + d['F'] = priceMap.totalPrice + } + } + },0); + } + + + def getEvaluationPrice(city, region, name, area){ + Map params = new HashMap(); + params.put("city", city); + params.put("district", region); + params.put("name", name); + params.put("size", area); + params.put("__refresh_cache","1") + + JSONObject result = OpenApiClient.evaluate(params); + if (result.getInt("code") == 200) { + if (result.containsKey("result")) { + return result.getJSONObject("result") + } + } else if (result.getInt("code") == 500) { + println("小区估价失败:${params.city} ${params.name} ${params.size} ${result}") + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestMatchResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestMatchResult.groovy new file mode 100644 index 000000000..9ff1149d5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestMatchResult.groovy @@ -0,0 +1,629 @@ +package com.fangjia.script.gj.chongqingblank + +import com.fangjia.core.RegionData +import com.fangjia.datastore.driver.DSCollection +import com.fangjia.datastore.driver.DataStore +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.gj.district.NameProcess +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.OpenApiClient +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.fangjia.script.yinjihaun.amap.GaoDeMapApiData +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +class TestMatchResult { + + private static final DSCollection SEAWEED = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") + + static main(args) { + def task = new TestMatchResult() +// task.run1("D:\\数据\\重庆农商行\\估值1.xlsx") + task.run2("D:\\数据\\重庆农商行\\估值2-V2.xlsx") +// task.run3("D:\\数据\\重庆农商行\\估值3.xlsx") + +// task.nearDistRun("D:\\数据\\重庆农商行\\估值2-V5.xlsx") +// task.run2V1("D:\\数据\\重庆农商行\\估值2-V2.xlsx") + +// task.similarScore("D:\\数据\\重庆农商行\\估值2-V5.xlsx") + +// task.mapRun("D:\\数据\\重庆农商行\\估值2-V4-map.xlsx"); + + } + + + + def run1(def fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + def region = d['I'] + def city = d['J'] + if (!d['I'] && !d['J']) { + city = d['B'] + region = d['C'] + } + def newName = null + if(d['D']=="大川菲尔小城"){ + if (d['D']) { + newName = d['D'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "") + .replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "") + .replace("1期", "一期").replace("2期", "二期").replace("3期", "三期") + .replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区") + .replace("4区", "四区").replace("5区", "五区").replace("a区", "A区") + .replace("B区", "B区").replace("c区", "C区").replace("d区", "D区") + } + if (!region || (region && region == d['L'])) { + if (d['D'] && !d['O'] && d['M']) { + if (d['M'] == newName) { + d['O'] = "相同" + } + if (!d['O']) { + def dbo = CommonDatautils.findDistrictByAlias(d['K'], d['L'], d['D'], "district") + if (dbo) { + d['O'] = "别名匹配" + } + } +// if (!d['O'] && d['M'] && d['E']) { +// if (d['E'].toString().contains(d['M'])) { +// d['O'] = "名称in地址" +// } +// } + + if (!d['O']) { + def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ") + def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d['M'], " ") + if (py1 == py2) { + d['O'] = "拼音相同" + } + } + + if (!d['O'] && d['D'] && d['M'] && newName) { + def word = null + if (newName.toString().length() >= d['M'].toString().length()) { + word = newName.toString().replace(d['M'], "") + } else if (newName.toString().length() < d['M'].toString().length()) { + word = d['M'].toString().replace(newName, "") + } + if (word) { + def words = ["重庆", "小区", "(高层)", "花园", "恒都", "(洋房)", "社区"] + for (def w : words) { + if (word == w) { + d['O'] = "去部分词后相同" + break + } + } + + if (!d['O'] && !word.endsWith("区") && !word.endsWith("期") && !word.endsWith("团") && !word.endsWith("东苑") && !word.endsWith("西苑") && !word.endsWith("南苑") && !word.endsWith("北苑")) { + if (d['N'] && d['N'].toString().endsWith("号") && d['E'] && d['E'].toString().contains(d['N'])) { + + def score = CosineSimilarAlgorithm.getSimilarity(d['M'], newName); + if (score >= 0.65) { + println newName + " " + d['M'] + " " + score + d['O'] = "地址相同" + } + } + } + } + } + + if (!d['O'] && newName && !newName.contains("期") && !newName.endsWith("组团") && (!newName.endsWith("区") || newName.endsWith("小区"))) { + + def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "高德") + if (!gaodeRest) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度") + } + if (gaodeRest && gaodeRest.name) { + def mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + if (mapName == d['M'] && gaodeRest.score >= 0.7) { + d['O'] = gaodeRest.source + "匹配" + println newName + " " + d['M'] + " " + gaodeRest.score + } + } + } + } + } + + + if (d['D'] && !d['O']) { + def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "高德") + def mapName = null + if (gaodeRest && gaodeRest.name) { + mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + } + if (!gaodeRest || (gaodeRest && mapName != newName)) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度") + if (gaodeRest && gaodeRest.name) { + mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + } + } + if (gaodeRest && mapName == newName) { + d['P'] = gaodeRest.district + d['Q'] = mapName + d['R'] = gaodeRest.address + d['S'] = gaodeRest.lng + d['T'] = gaodeRest.lat + d['U'] = gaodeRest.type + d['v'] = gaodeRest.source + } + } + } + + },0); + } + + def similarScore(def fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + if(d['D'] && d['M']){ + def score = CosineSimilarAlgorithm.getSimilarity(d['D'], d['M']); + d['Q'] = score + } + },0) + } + + + def run2(def fileName){ + ExcelUtils.multiThreadingModify(fileName,{ d-> + def region = d['I'] + def city = d['J'] + if(!d['I']){ + city = d['J'] + } + +// if(d['E']=="巴南区花溪镇花溪新街88号2单元3-4#") { +// println "ss" + + + if (!region || (region && region == d['L'])) { + if (d['N'] && !d['O'] && d['E']) { + if (d['N'].toString().endsWith("号") && d['E'] && d['E'].toString().contains(d['N'])) { + d['O'] = "地址匹配" + } + } + } + + if (!d['O'] && d['M'] && d['E']) { + def matchName = d['M'] + if(matchName.toString().endsWith("小区")){ + matchName = matchName.toString().replace("小区","") + } + if(matchName.toString().startsWith(d['L'])){ + matchName = matchName.toString().replace(d['L'],"") + } + def oldName= NameProcess.nameStandard(city,region, d['E']) + if (oldName.toString().contains(matchName)) { + d['O'] = "名称in地址" + } + } + + if (d['E'] && !d['O'] ) { + def gaodeRest = null + + if (!gaodeRest && d['E']) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, d['E'], null, "住宅", "高德") + } + if(!gaodeRest && d['H']){ + gaodeRest = FangjiaClientUtils.getMapData(city, region, d['H'], null, "住宅", "高德") + } + if (!gaodeRest && d['E']) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, d['E'], null, "住宅", "百度") + } + if (!gaodeRest && d['H']) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, d['H'], null, "住宅", "百度") + } + + if (gaodeRest && gaodeRest.score >= 0.5) { + d['P'] = gaodeRest.district + d['Q'] = gaodeRest.name + d['R'] = gaodeRest.address + d['S'] = gaodeRest.lng + d['T'] = gaodeRest.lat + d['U'] = gaodeRest.type + d['v'] = gaodeRest.source + } + } + + if(!d['O']){ + def locationMap = getLocation(city, d['E']) + if(locationMap){ + def mapRets = findNearestByMap(locationMap.lat, locationMap.lng) + if(mapRets){ + def o = mapRets.get(0) + d['W'] = o.cityname + d['X'] = o.adname + d['Y'] = o.name + if(o.address && o.address instanceof String){ + d['Z'] = o.address + } + d['AA'] = o.type + d['AB'] = o.distance + d['AC'] = o.location + } + def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10) + if(dbRets){ + def b = dbRets.get(0) + d['AD'] = b.region + d['AE'] = b.name + d['AF'] = b.address + d['AG'] = b.distance + } + } + } + +// } + + },0); + } + + def distanceRun(def fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + def city = d['J'] + if(!d['O'] && d['E'] && d['E'].toString().length()>6){ + def locationMap = getLocation(city, d['E']) + if(locationMap){ + def mapRets = findNearestByMap(locationMap.lat, locationMap.lng) + if(mapRets){ + def o = mapRets.get(0) + d['W'] = o.cityname + d['X'] = o.adname + d['Y'] = o.name + if(o.address && o.address instanceof String){ + d['Z'] = o.address + } + d['AA'] = o.type + d['AB'] = o.distance + d['AC'] = o.location + } + def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10) + if(dbRets){ + def b = dbRets.get(0) + d['AD'] = b.region + d['AE'] = b.name + d['AF'] = b.address + d['AG'] = b.distance + } + } + } + },0); + } + + def nearDistRun(def fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + def city = d['B'] +// if(!d['I'] && !d['J']){ +// city = d['B'] +// } + if(d['O'] && d['O']=="周边"){ + def locationMap = null + if(d['B'] && d['E']){ + locationMap = getLocation(city, d['E']) + } +// if(!locationMap && d['C']){ +// locationMap = getLocation(city, d['C']) +// } + + if(locationMap){ + def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10) + if(dbRets){ + def b = dbRets.get(0) + d['K'] = b.city + d['L'] = b.region + d['M'] = b.name + d['N'] = b.address + d['P'] = b.distance + } + } + } + },0); + } + + def run2V1(def fileName){ + ExcelUtils.multiThreadingModify(fileName,{ d-> + def city = d['J'] + if(d['O']){ + d['AH'] = d['L'] + d['AI'] = d['M'] + d['AJ'] = d['N'] + } + if(!d['O']){ + if(d['R'] && d['R'].contains("号")){ + def addr = d['R'].toString().substring(0, d['R'].indexOf("号")) + if((addr && d['E'] && d['E'].contains(addr))){ + def region = RegionData.formatRegion(city, d['P']) + if(d['U'] && d['U'].toString().contains("住宅")){ + d['AH'] = region + d['AI'] = d['Q'] + d['AJ'] = d['R'] + d['AK'] = d['U'] + d['AM'] = d['S']+","+d['T'] + d['O'] ="地图匹配" + }else if(!d['U']){ + d['AH'] = region + d['AI'] = d['Q'] + d['AJ'] = d['R'] + d['AK'] = d['U'] + d['AM'] = d['S']+","+d['T'] + d['O'] ="地图匹配" + } + } + } + } + + if(!d['O']){ + if(d['Y'] && d['AE']){ + def mapName = d['Y'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "") + .replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "") + .replace("1期", "一期").replace("2期", "二期").replace("3期", "三期") + .replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区") + .replace("4区", "四区").replace("5区", "五区").replace("a区", "A区") + .replace("B区", "B区").replace("c区", "C区").replace("d区", "D区") + if(mapName == d['AE']){ + def region = RegionData.formatRegion(city, d['AD']) + d['AH'] = region + d['AI'] = d['AE'] + d['AJ'] = d['AF'] + d['AK'] = d['DB'] + d['O'] ="名称相同" + }else{ + if(d['AB'].toString().toDouble()>=d['AG'].toString().toDouble()){ + def region = RegionData.formatRegion(city, d['AD']) + d['AH'] = region + d['AI'] = d['AE'] + d['AJ'] = d['AF'] + d['AK'] = "DB库近" + d['AL'] = d['AG'] + d['O'] ="距离比较" + }else{ + def region = RegionData.formatRegion(city, d['X']) + d['AH'] = region + d['AI'] = d['Y'] + d['AJ'] = d['Z'] + d['AK'] = d['AA'] + d['AL'] = d['AB'] + d['AM'] = d['AC'] + d['O'] ="距离比较" + } + } + }else if(d['Y']){ + def region = RegionData.formatRegion(city, d['X']) + d['AH'] = region + d['AI'] = d['Y'] + d['AJ'] = d['Z'] + d['AK'] = d['AA'] + d['AL'] = d['AB'] + d['AM'] = d['AC'] + d['O'] ="采用地图相近" + }else if(d['AE']){ + def region = RegionData.formatRegion(city, d['AD']) + d['AH'] = region + d['AI'] = d['AE'] + d['AJ'] = d['AF'] + d['AK'] = "DB库近" + d['AL'] = d['AG'] + d['O'] ="采用DB相近" + } + } + },0); + } + + + def findNearestByMap(lng, lat){ + def rets = GaoDeMapApiData.doSearchByGaoDe("120000|120203|120300|120301|120302|120303|120304", lng, lat, 1000, 1, 10) + return rets + + } + + public List findNearestDistrict(def city, double lat, double lng, int radiusMeters, int limit) { + DBObject districtFields = new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("address", 1) + districtFields.put("estate_type2", 1) + districtFields.put("lat", 1) + districtFields.put("lng", 1) + + + DBObject sdq = new BasicDBObject(); + sdq.put("sd.point", lat + "," + lng); + sdq.put("sd.sfield", "lat_lng"); + sdq.put("sd.distance", String.valueOf((double) radiusMeters / 1000.0)); + + DBObject q = new BasicDBObject(); + q.put("cat", "district"); + q.put("city", city); + q.put("visible", 0); + q.put('$sd', sdq); + + List list = new ArrayList() + Iterator iter = SEAWEED.find(q, districtFields).limit(limit * 2).iterator() + while(iter.hasNext()) { + def o = iter.next() + o.distance = GeoHelper.GetDistance(o.lat/10000000000.0, o.lng/10000000000.0, lat, lng) + list.add(o) + } + + if (list.size() == 0) { + return Collections.EMPTY_LIST; + } + list.sort{o1,o2-> + return o1.distance <=> o2.distance + } + if (list.size() > limit) { + list = list.subList(0, limit); + } + return list; + } + + def getLocation(city, addr){ + + if(addr){ + def rMap = [:] + def reps = GaoDeApiUtils.geo(city, addr); + if(reps && reps.count.toInteger()>=1){ + def map = reps.geocodes.get(0) + def location = map.location + def level = map.level + if(location){ + def locMap = location.split(",") +// def lat = dist.lat/10000000000.0; +// def lng = dist.lng/10000000000.0; +// def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng) +// rMap.distance = distance +// rMap.level = level + rMap.lat = locMap[1].toDouble() + rMap.lng = locMap[0].toDouble() +// rMap.keyword = map.formatted_address +// rMap.location = location +// rMap.region = map.district + return rMap; + } + } + } + return null; + } + + + + def run3(def fileName){ + ExcelUtils.multiThreadingModify(fileName,{ d-> + def city = d['I'] + def region = d['J'] + + def newName = null + if(d['C']){ + if(d['C'].toString().length()<2){ + newName = null + }else{ + newName = d['C'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "") + .replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "") + .replace("1期", "一期").replace("2期", "二期").replace("3期", "三期") + .replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("a区", "A区") + .replace("B区", "B区").replace("c区", "C区").replace("d区", "D区") + } + + } + def address = null + if(d['H']){ + address = d['H'] + }else{ + def arrs = d['B'].split("->") + if(arrs && arrs.size()==4){ + address = arrs[3] + }else if(arrs && arrs.size()==2){ + address = arrs[1] + } + } + if(!region ||(region && d['L'] && region.contains(d['L']))) { + if (d['B'] && !d['O'] && d['M']) { + if (d['M'] == newName) { + d['O'] = "相同" + } + if (!d['O']) { + def dbo = CommonDatautils.findDistrictByAlias(d['K'], d['L'], newName, "district") + if (dbo) { + d['O'] = "别名匹配" + } + } + + if (!d['O']) { + def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ") + def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d['M'], " ") + if (py1 == py2) { + d['O'] = "拼音相同" + } + } + + if (!d['O'] && d['M'] && newName) { + def word = null + if (newName.toString().length() >= d['M'].toString().length()) { + word = newName.toString().replace(d['M'], "") + } else if (newName.toString().length() < d['M'].toString().length()) { + word = d['M'].toString().replace(newName, "") + } + if (word) { + def words = ["重庆", "小区", "(高层)", "花园", "恒都", "(洋房)", "社区"] + for (def w : words) { + if (word == w) { + d['O'] = "去部分词后相同" + break + } + } + + if (!d['O'] && !word.endsWith("区") && !word.endsWith("期") && !word.endsWith("团") && !word.endsWith("东苑") && !word.endsWith("西苑") && !word.endsWith("南苑") && !word.endsWith("北苑")) { + if (d['N'] && d['N'].toString().endsWith("号") && address.toString().contains(d['N'])) { + + def score = CosineSimilarAlgorithm.getSimilarity(d['M'], newName); + if (score >= 0.65) { + println newName + " " + d['M'] + " " + score + d['O'] = "地址相同" + } + } + } + } + } + + if (!d['O'] && newName && !newName.contains("期") && !newName.endsWith("组团") && (!newName.endsWith("区") || newName.endsWith("小区"))) { + + def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "高德") + if (!gaodeRest) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "百度") + } + if (gaodeRest && gaodeRest.name) { + def mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + if (mapName == d['M'] && gaodeRest.score >= 0.7) { + d['O'] = gaodeRest.source + "匹配" + println newName + " " + d['M'] + " " + gaodeRest.score + } + } + } + } + } + + + + if(newName && !d['O']){ + def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "高德") + def mapName = null + if(gaodeRest && gaodeRest.name){ + mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + } + if(!gaodeRest || (gaodeRest && mapName != newName)) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度") + if(gaodeRest && gaodeRest.name){ + mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + } + } + if(gaodeRest && mapName== newName){ + d['P'] = gaodeRest.district + d['Q'] = mapName + d['R'] = gaodeRest.address + d['S'] = gaodeRest.lng + d['T'] = gaodeRest.lat + d['U'] = gaodeRest.type + d['v'] = gaodeRest.source + } + } + + },0); + } + + def mapRun(fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + def city = d['J'] + def region = d['I'] + if(city && region && d['Q'] && d['O']!="距离比较" && d['O']!="地址匹配"){ + def gaodeRest = FangjiaClientUtils.getMapData(city, region, d['Q'], null, "住宅", "高德") + if(gaodeRest){ + d['V'] = gaodeRest.city + d['W'] = gaodeRest.district + d['X'] = gaodeRest.name + d['Y'] = gaodeRest.address + d['Z'] = gaodeRest.lng + d['AA'] = gaodeRest.lat + d['AB'] = gaodeRest.type + } + + d['AC'] = NameProcess.nameStandard(city,region, d['Q']) + } + },0) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestPrice.groovy new file mode 100644 index 000000000..e252654d5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestPrice.groovy @@ -0,0 +1,89 @@ +package com.fangjia.script.gj.chongqingblank + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators + +class TestPrice { + + def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + static main(args) { + def task = new TestPrice() + task.run() + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\估值1-V2.xlsx",{ d-> + if(d['K'] && d['M']){ + def dbo = findThirdPrice(d['K'], d['L'], d['M'], 202012, "lianjia_esf_price") + if(dbo){ + d['AU'] = dbo.lianjia_esf_price + } + + def district = CommonDatautils.findDistrict(d['K'], d['L'], d['M'], "district") + def startDate = 20201101 + def endDate = 20210601 + def trades = findTrades(district, startDate, endDate) + if(trades){ + d['AV'] = (int)(trades.sum()/trades.size()) + d['AW'] = trades.size() + } + } + },0); + } + + def findThirdPrice(def city, def region, def name, def s_date,def fieldName){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", new BasicDBObject(QueryOperators.GTE, s_date)) + q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true)) + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1)) + def obj = null; + for(def d : dbcursor){ + obj = d; + break; + } + return obj; + } + + def findTrades(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) +// q.put("source", "链家在线") + + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + if(alias){ + q.put("district_name", new BasicDBObject('$in', alias)) + } + + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def prices = [] + for(def d : dbcorsor){ + if(d.avg_price){ + prices << d.avg_price + } + } + + return prices + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionExport.groovy new file mode 100644 index 000000000..6e967d2bc --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionExport.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.gj.data + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.ExcelUtils + +class RegionExport { + + static main(args) { + def task = new RegionExport() + task.run() + } + + def run(){ + ExcelUtils.multiThreadingModify("d:\\数据\\亲邻\\小区城市列表.xls",{d-> + def city = d['B'] + def newCity = RegionData.formatCity(city) + if(newCity){ + def regions = CommonUtils.getAllRegions(newCity, "region") + def regionStr = "" + regions.each{k,v-> + regionStr = regionStr + k+"\n" + } + d['C'] = regionStr + } + },0) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy new file mode 100644 index 000000000..7860afad9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy @@ -0,0 +1,236 @@ +package com.fangjia.script.gj.es + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.lucene.queryparser.classic.QueryParser + +class EsMonthHouseStat { + + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + static main(args) { + def task = new EsMonthHouseStat() +// task.run() + task.tradeRun() +// task.houseRun() + } + + def run(){ + def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def rets = [] + def dates = ["2020-05","2020-06","2020-07","2020-08","2020-09","2020-10","2020-11","2020-12","2021-01","2021-02","2021-03","2021-04","2021-05"] + def headers = ["城市"] + headers.addAll(dates) + rets << headers + for(def city : citys){ + def datas = [] + datas << city + for(def d : dates){ + def date = DateUtils.parseFromTightYYYYMMDD(d+"-01") + def startDate = DateUtils.formatDateByPattern(date, "yyyyMMdd") + def endDate = DateUtils.addMonth(date, 1).replace("-","") + def count = findHouseCount(city, startDate.toInteger(), endDate.toInteger()) + datas << count + } + rets << datas + println datas + } + ExcelUtils.write("d:\\数据\\建行92个城市房源统计.xlsx", rets, "统计") + } + + def tradeRun(){ + def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def rets = [] + def rets2 = [] + //, + //"2021-01","2021-02","2021-03","2021-04","2021-05" + //def dates = ["2020-01","2020-02","2020-03","2020-04","2020-05","2020-06","2020-07","2020-08","2020-09","2020-10","2020-11","2020-12"] + def dates = ["2021-04"] + def headers = ["城市"] + headers.addAll(dates) + rets << headers + + + def headers2 = ["城市"] + headers2.addAll(dates) + rets2 << headers2 + for(def city : citys){ + def datas = [] + datas << city + + def datas2 = [] + datas2 << city + for(def d : dates){ + def startDate = DateUtils.parseFromTightYYYYMMDD(d+"-01") + def endDateStr = DateUtils.addMonth(startDate, 1).replace("-","") + def endDate = DateUtils.parseFromTightYYYYMMDD2(endDateStr) + //def count = findTrades(city, startDate, endDate) + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]); + def oldCount = 0 + def newCount = 0 + for(def dist : districts){ + def countMap = findTradesDetail(dist, startDate, endDate) + oldCount = oldCount + countMap.total_old + newCount = newCount + countMap.total_new + println dist.city + " " + dist.name +" "+countMap.total_old+" "+countMap.total_new + } + + datas << oldCount + datas2 << newCount + println city +" "+ oldCount +" "+newCount + } + rets << datas + + rets2 << datas2 + println datas + } + ExcelUtils.write("d:\\数据\\建行92个城市成交统计(去重前匹配).xlsx", rets, "统计") + + ExcelUtils.write("d:\\数据\\建行92个城市成交统计(去重后匹配).xlsx", rets2, "统计") + } + + def houseRun(){ + ExcelUtils.multiThreadingModify("d:\\数据\\小区案例数据覆盖率.xlsx",{d-> + def city = d['B'] + if(city == "湖州"){ + def count = findHouseCount(city, 20210301, 20210601) + d['E'] = count + + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]); + def disthouseCount = 0 + for(def dist : districts){ + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + def distCount = findHouseCount(dist.city, dist.region, alias, 20210301, 20210601) + if(distCount){ + disthouseCount++ + } + println dist.city+" "+dist.region+" "+dist.name+" "+distCount + } + + println "disthouseCount:"+disthouseCount + println districts.size() + def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size()) + d['D'] = precent+"%" + } + },0) + } + + + def findHouseCount(def city, def region, def alias, def startDate, def endDate){ + def aliasStr = null; + def total = 0 + try{ + aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')" + + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.c + }) + }catch(Exception e){ + e.printStackTrace() + } + + return total + } + + /** + * 转义字符串中的特殊字符 + * 仅过滤Lucene regexps, 其他的勿用 + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + + def findHouseCount(def city, def startDate, def endDate){ + + String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} " + def total = 0 + ElasticUtils.executeSqlQuery(sql,{ data-> + total = data.c + }) + return total + } + + + + + + def findTrades(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) +// q.put("region", dist.region) +// q.put("source", "链家在线") + +// def alias = [] +// if(dist.alias){ +// if(!dist.alias.contains(dist.name)){ +// dist.alias << dist.name +// } +// alias = dist.alias +// }else{ +// alias << dist.name +// } + +// if(alias){ +// q.put("district_name", new BasicDBObject('$in', alias)) +// } + + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def count = DEAL_PRICE_COLL.count(q) + return count + } + + + + def findTradesDetail(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + if(alias){ + q.put("district_name", new BasicDBObject('$in', alias)) + } + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcursor = DEAL_PRICE_COLL.find(q) + def total = dbcursor.count() + def rets = new HashSet() + for(def d : dbcursor){ + def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + if(!rets.contains(key)){ + rets << key + } + } + def rMap = [:] + rMap.total_old = total + rMap.total_new = rets.size() + return rMap + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy new file mode 100644 index 000000000..6f2307343 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy @@ -0,0 +1,66 @@ +package com.fangjia.script.gj.es + +import com.fangjia.core.util.DateUtil +import com.fangjia.script.util.ElasticUtils +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.Connection +import java.sql.ResultSet +import java.sql.Statement + +class EsSelectTest { + EsDataSource dataSource = new EsDataSource(); + String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; + + static main(args) { + def task = new EsSelectTest() + task.run() + } + + def run(){ + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("request", "/ws/czbank")) + //.must(QueryBuilders.termQuery("request_body", "findSample")) + + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder); + if(houses) { + for (int k = 0; k < houses.size(); k++) { + println k + } + } + + + def endDate = new Date() + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + dataSource.setProperties(connectionProperties) + StringBuffer sqlBuffer = new StringBuffer(); + String d = DateUtil.formatByPattern(new Date(), "yyyy.MM.dd") + long startD = DateUtil.getBeforeMinute(new Date(), -7).getTime(); + long endD = DateUtil.getBeforeMinute(endDate, 1).getTime(); + println(DateUtil.formatByPattern(new Date(startD), "yyyy-MM-dd HH:mm:ss")+"===="+DateUtil.formatByPattern(new Date(endD), "yyyy-MM-dd HH:mm:ss")); + + + sqlBuffer.append("select request_time FROM \"fangjia-log*\" WHERE 1=1"); + sqlBuffer.append(" and request='/ws/czbank' and request_body='findSample'") + + ResultSet rs = null + try { + Connection connection = dataSource.getConnection() + Statement statement = connection.createStatement() + rs = statement.executeQuery(sqlBuffer.toString()) + } catch (Exception e) { + e.printStackTrace() + } + StringBuffer msgBuffer = new StringBuffer(); + while (rs.next()) { + + def responsetime = rs.getLong("responsetime"); + def timestamp = DateUtil.formatByPattern(new Date(rs.getLong("time")), "yyyy-MM-dd HH:mm") + println responsetime + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy new file mode 100644 index 000000000..55e0943ab --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy @@ -0,0 +1,171 @@ +package com.fangjia.script.gj.location + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.looplineinfo.Point +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import java.text.SimpleDateFormat +import java.util.regex.Matcher +import java.util.regex.Pattern; + + +class LocationTest { + + static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + + static main(args) { + def task = new LocationTest() +// task.run() + task.run2() + } + + def run(){ + def regionPointsMap = findRegionRange() + def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "address", "lat", "lng"]); + def index = 0 + def rets = [] + for(def dist : districts){ + def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) + if(!flag){ + index++ + + def ret = [] + ret << dist.city + ret << dist.region + ret << dist.name + + def loactionMap = matchMap(dist) + if(loactionMap){ + ret << "地图" + ret << loactionMap.region + ret << loactionMap.mapName + def distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng) + ret << distance + if(dist.region==loactionMap.region){ + println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同" + }else{ + println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance + } + }else{ + ret << "" + println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到" + } + rets << ret + + } + } + ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误3.xlsx", rets, "区域不在行政范围内") + } + + def run2(){ + ExcelUtils.multiThreadingModify("d:\\数据\\福州\\福州小区.xlsx") + } + + def matchMap(def dist){ + def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德") + def locationMap = null + if(gaodeRest){ + locationMap = getMapLocation(dist, gaodeRest); + if(locationMap){ + return locationMap + } + } + + def baiduRest = null + if (!locationMap) { + baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度") + } + if(baiduRest){ + locationMap = getMapLocation(dist, baiduRest); + } + return locationMap + } + + def getMapLocation(def dist, def rets){ + def lat = null + def lng = null + def region = null + def mapName = null + for(def r : rets){ + if(!r.city){ + continue + } + if(!r.city.contains(dist.city)){ + continue + } + def name = r.name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim() + if(dist.name == name){ + lat = r.lat + lng = r.lng + mapName = r.name + region = r.district + }else{ + def word = null + if (name.length() >= dist.name.length()) { + word = name.replace(dist.name, "") + } else if (name.length() < dist.name.length()) { + word = dist.name.replace(name, "") + } + if (word) { + def words = ["商住楼", "小区","公寓","别墅"] + for (def w : words) { + if (word == w) { + lat = r.lat + lng = r.lng + region = r.district + mapName = r.name + break + } + } + if(!lng){ + Pattern pattern = Pattern.compile("\\d+号楼"); + Matcher matcher = pattern.matcher(word); + if(matcher.find()){ + lat = r.lat + lng = r.lng + region = r.district + mapName = r.name + break + } + } + } + } + if(lng){ + break + } + } + if(lng){ + return ["region":region, "lng":lng, "lat":lat, "mapName":mapName] + } + + + return null + } + + def isInArea(lng, lat, points){ + List pointList = InsidePolygon.createPoint2(points); + //Point point = new Point(lng, lat); + //def flag = InsidePolygon.aboutZone(point, pointList) + def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) + return flag; + } + + def findRegionRange(){ + def q = new BasicDBObject() + q.put("city","福州市") + def dbcursor = POLYLINECOLL.find(q) + def regionMap = [:] + for(def d : dbcursor){ + def region = RegionData.formatRegion("福州",d.region) + regionMap.put(region, d.polyline) + } + return regionMap + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy new file mode 100644 index 000000000..b43b98223 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -0,0 +1,106 @@ +package com.fangjia.script.thirdPriceCrawl + +import com.fangjia.script.util.CommonUtils +import com.mongodb.BasicDBObject +import org.apache.solr.common.util.Hash + +class SynchDistrictPrice { + + def static final SHORT_DATE = Utils.getLastMonth() + def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); + def sourceMap = ["jiwu.com":"吉屋"] + //"58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","jiwu.com":"吉屋" + + static main(args) { + def task = new SynchDistrictPrice() + task.run() + } + + + def run(){ + def citys = ["福州"]//CommonUtils.findAllCity() + + sourceMap.each { k, v -> + for(def city : citys){ + def rets = findCrawlDistrictDetails(city, k, v) + synchData(city, rets, v) + println city +" "+rets.size() + } + } + + } + + def findCrawlDistrictDetails(city, root, source){ + def q = new BasicDBObject() + q.put("city", city) + q.put("root", root) + def keys = new BasicDBObject() + keys.put("city", 1) + keys.put("region", 1) + keys.put("block", 1) + keys.put("name", 1) + keys.put("address", 1) + keys.put("avg_price", 1) + keys.put("root", 1) + keys.put("url", 1) + def rets = [] + def dbcursor = districtDetailColl.find(q, keys) + def names = new HashSet() + for(def d : dbcursor){ + def s = d.city +"#"+d.region+"#"+d.name + if(!names.contains(s)){ + names << s + d.put("district_id", d.url) + d.put("source", source) + d.put("s_date", SHORT_DATE) + d.removeField("root") + d.removeField("url") + d.removeField("_id") + rets << d + } + } + return rets + } + + def synchData(def city, def rets, def source){ + def q = new BasicDBObject() + q.put("source", source) + q.put("s_date", SHORT_DATE) + q.put("city", city) + def count = thirdPartyColl.count(q) + def addIndex = 0 + if(count==0){ + try{ + thirdPartyColl.insert(rets) + }catch(Exception e){ + println e.getMessage() + } + + }else{ + for(def r : rets){ + def flag = findIsExists(r.city, r.region, r.name, SHORT_DATE, source) + if(!flag){ + addIndex++ + thirdPartyColl.save(r) + } + } + } + + println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count + } + + def findIsExists(def city, def region, def name, def sDate, def source){ + def q = new BasicDBObject() + q.put("source", source) + q.put("s_date", sDate) + q.put("city", city) + q.put("region", region) + q.put("name", name) + def count = thirdPartyColl.count(q) + if(count){ + return true + } + return false + } +} From 5834aaf7e15d24bf9c3f0e912a9ad77752ac2a32 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 4 Jun 2021 11:28:52 +0800 Subject: [PATCH 07/21] =?UTF-8?q?=E6=9C=88=E5=9D=87=E4=BB=B7=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/avgprice/MonthlyDistrictStatCalcBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/MonthlyDistrictStatCalcBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/MonthlyDistrictStatCalcBatch.groovy index e6daa5e94..1d1396141 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/MonthlyDistrictStatCalcBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/MonthlyDistrictStatCalcBatch.groovy @@ -30,12 +30,12 @@ public class MonthlyDistrictStatCalcBatch extends BaseBatch { void process(Object args) { List cities = []; def allCity = CommonUtils.findAllCity() - allCity = ["苏州"] + def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1) cities.addAll(allCity) def districts = [] cities.each {def city-> - districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0).append("name", "吾悦广场"), keys).batchSize(2000).toList() + districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000).toList() println(city) } ExecutorService executorService = Executors.newFixedThreadPool(20) From cbbd463d838d22940214b4016c862beae2bafcc3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 4 Jun 2021 11:30:35 +0800 Subject: [PATCH 08/21] =?UTF-8?q?=E7=A6=8F=E5=B7=9E=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E6=B7=BB=E5=8A=A0=E5=9C=B0=E5=9D=80=E5=88=AB=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index fdf2ce5f9..1af46cb50 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -37,7 +37,7 @@ class DistrictInfoExportBatch extends BaseBatch{ def headers = ["id","city","district","name","lng","lat","property_type","build_type","households", "property_fee","car_site_count","property_life","house_property","open_date","finish_date", "delivery_date","floor_area","build_area","plot_ratio","landscaping_ratio","developer","property_company", - "build_structure","building_count","alias","address"] + "build_structure","building_count","alias","address", "addr_alias"] def ret = createDatas("福州") def fileName = "/fzbank/output/FJW_COMM_"+curDate+".csv" @@ -110,6 +110,8 @@ class DistrictInfoExportBatch extends BaseBatch{ }else{ row << rMap.address } + + row << processAlias(rMap.addr_alias) result << row } } From d667012ec9676d657305c207e4c2877bc574ae47 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 4 Jun 2021 11:31:26 +0800 Subject: [PATCH 09/21] update --- .../script/fangjiatong/AccessLogBatch.groovy | 34 ++++-- .../com/fangjia/script/gj/OpenApiUtils.groovy | 18 +++ .../fangjia/script/gj/data/FieldStat.groovy | 107 ++++++++++++++---- .../script/gj/district/ExportData.groovy | 6 +- .../gj/district/FangjiaClientUtils.groovy | 31 +++++ .../gj/test/CrawlDistrictIdSelect.groovy | 28 ++--- .../script/looplineinfo/InsidePolygon.groovy | 56 +++++++++ 7 files changed, 233 insertions(+), 47 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AccessLogBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AccessLogBatch.groovy index 0b4d0724b..06e84af84 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AccessLogBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AccessLogBatch.groovy @@ -211,23 +211,35 @@ class AccessLogBatch extends BaseBatch{ } def read(){ - String fileName = "C:\\Users\\gaojuan\\Desktop\\localhost_access_log.2019-06-01.txt"; + String fileName = "D:\\数据\\logs\\performance.open.log"; File file = new File(fileName); def rets = [] - file.eachLine{ + def index = 0 + def list =[] + def slowCount = 0 + file.eachLine{it-> - if(it.contains("/fangjiatong2/")){ - try{ - it = it.replace(it, URLDecoder.decode(URLDecoder.decode(it, "utf-8"), "UTF-8")) - }catch(IllegalArgumentException e){} + + if(it.contains("ws/czbank") && it.contains("entity:House")){ +// println it + index++ +// try{ + def s1 = it.substring(it.indexOf("time:")+5, it.length()) + s1 = s1.substring(0, s1.indexOf("\t")) + list << s1.toInteger() + if(s1.isNumber() && s1.toInteger()>=4000){ + println it + slowCount++ + } +// }catch(IllegalArgumentException e){} } - if(!it.contains("漕溪四村")){ - rets << it - } + } - - writeTxt(rets, "C:\\Users\\gaojuan\\Desktop\\fjt-2020-6-1.txt"); + println list.sum()/list.size() + println index + println "slowCount:"+slowCount +// writeTxt(rets, "C:\\Users\\gaojuan\\Desktop\\fjt-2020-6-1.txt"); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 76ef3729b..4ec26d7db 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -43,6 +43,24 @@ class OpenApiUtils { } + static def matchDetailDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){ + address = address.replace("(","(").replace(")",")").replace(" ","") + def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1" + if(preciseLevelflag){ + url = url +"&preciseLevel=strict" + } + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==200 && doc.result){ + def dto = doc.result + return dto + } + } + return null; + } + + static def matchDistrict2(def city, def address, def preciseLevelflag){ address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","") def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy index 72fd50274..eb7d4aadd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy @@ -1,39 +1,106 @@ package com.fangjia.script.gj.data import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection class FieldStat { + final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo); + static main(args) { def task = new FieldStat(); - task.run() +// task.run() + task.run1() +// task.run2() + } + + def run1(){ + def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def dist_total = 0 + + def addr_alias_total = 0 + def rets = [] + rets << ["城市","别名总量","有别名小区数量","小区总量","有别名小区占比"] + for(def city : citys){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias", "addr_alias"]); + def alias_index = 0 + def alias_total = 0 + + for(def d : districts){ + def alias = d.alias + if(alias){ + alias_index++ + alias_total = alias_total + alias.size() + } + } + def precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size()) + rets << [city, alias_total, alias_index, districts.size(), precent+"%"] + + } + ExcelUtils.write("d:\\数据\\92个城市别名统计.xlsx", rets, "统计") +// println dist_total +// println alias_total +// println addr_alias_total } def run(){ - ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\建行数据融合结果统计_0425.xlsx", { row -> - def city = row['A'] + ExcelUtils.multiThreadingModify("D:\\数据\\城市统计.xlsx", { row -> + def city = row['B'] if(city){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]); + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); def total = districts.size() + row['D'] = total - def aliasIndex = 0 - def aliasTotalCount = 0 - for(def d : districts){ - def alias = d.alias - if(alias){ - aliasIndex++ - aliasTotalCount = aliasTotalCount + alias.size() - } - } -// row['J'] = aliasTotalCount -// row['K'] = aliasIndex - def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total) - row['B'] = p+"%" -// row['M'] = total - println city +" "+p +// def aliasIndex = 0 +// def aliasTotalCount = 0 +// for(def d : districts){ +// def alias = d.alias +// if(alias){ +// aliasIndex++ +// aliasTotalCount = aliasTotalCount + alias.size() +// } +// } +//// row['J'] = aliasTotalCount +//// row['K'] = aliasIndex +// def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total) +// row['B'] = p+"%" +//// row['M'] = total +// println city +" "+p } - }, 1); + }, 0); println "finish" } + + def run2(){ + def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def rets = [] + rets << ["城市","建筑年代覆盖率","物业类型覆盖率","建筑类型覆盖率","区域最高小区均价覆盖率","区域最低小区均价覆盖率"] + for(def c : citys){ + def obj = findCoverPrecent(c) + if(obj){ + def datePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.finishBuildingDate*100) + def estateTypePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.estateType2*100) + def buildTypePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.buildType*100) + rets << [c, datePrecent+"%", estateTypePrecent+"%", buildTypePrecent+"%", "100%", "100%"] + } + } + + ExcelUtils.write("d:\\数据\\92个城市覆盖率统计.xlsx", rets, "统计") + + } + + def findCoverPrecent(def city){ + def q = new BasicDBObject() + q.put("city", city) + def dbcursor = CITY_INFO_CONN.find(q).sort(new BasicDBObject("weekend", -1)).limit(1) + def obj = null + for(def d : dbcursor){ + obj = d + } + return obj + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy index fd2856af7..6a44d4139 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy @@ -24,8 +24,8 @@ class ExportData { static main(args) { def task = new ExportData(); -// task.exportThird() - task.exportMap(); + task.exportThird() +// task.exportMap(); } def exportMap(){ @@ -71,7 +71,7 @@ class ExportData { def exportThird(){ - def citys = ["淮安","昆山","连云港","南京","南通","宿迁","苏州","泰州","无锡","徐州","盐城","扬州","镇江"] + def citys = ["重庆"] for(def city : citys){ def rets = [] rets << ["城市","区域","名称","别名"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 033464eb5..a131a414c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.gj.district import com.fangjia.core.util.HttpRequest import net.sf.json.JSONArray import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils class FangjiaClientUtils { @@ -48,4 +49,34 @@ class FangjiaClientUtils { println content } + + + public static def searchMapData(def city, def keyword, def types, def mapCode){ + def jsonObj = null + try { + def url = "http://192.168.0.19:38080/FangJiaDataService/search?city="+city+"&keyword="+keyword+"&types="+types+"&mapCode="+mapCode+"&limit=10" + def content = new HttpRequest().newGet(url, null, "utf-8") + // def ret = content.replace("[","").replace("]","") + jsonObj = JSONArray.fromObject(content) + } catch (Exception e) { + return null; + } + return jsonObj; + } + + + public static Map JsonToMap(JSONObject json) { + Map columnValMap = new HashMap(); + Set jsonKeys = json.keySet(); + for (Object key : jsonKeys) { + Object JsonValObj = json.get(key); + if (key instanceof JSONObject) { + columnValMap.put((String) key, JsonToMap((JSONObject) JsonValObj)); + } else { + columnValMap.put((String) key, JsonValObj); + } + } + return columnValMap; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy index 8e8bba001..b26bd079b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy @@ -4,6 +4,7 @@ import com.fangjia.core.RegionData import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection class CrawlDistrictIdSelect { @@ -14,8 +15,8 @@ class CrawlDistrictIdSelect { static main(args) { def task = new CrawlDistrictIdSelect() - task.run() -// task.synch() +// task.run() + task.synch() } def synch(){ @@ -50,13 +51,13 @@ class CrawlDistrictIdSelect { def findCrawlData(){ def q = new BasicDBObject() - q.put("city","福州") - q.put("source", "安居客") - q.put("s_date",new BasicDBObject('$gte', 202001)) - def dbcursor = thirdPartyColl.find(q) +// q.put("city","福州") + q.put("source", "诸葛找房") + q.put("s_date",202010) + def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def retMap = [:] - def prefix = "/community/view/" - def keyList =[] + def prefix = "xiaoqu.zhuge.com/" + def keyList =new HashSet() for(def db : dbcursor){ def url = null if(!db.url){ @@ -67,12 +68,11 @@ class CrawlDistrictIdSelect { url = db.url } if(url && url.contains(prefix) ){ - def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size()) + def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1) if(district_id=="0"){ continue } - def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com")) - def region = RegionData.formatRegion(db.city, db.region) + def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".xiaoqu.zhuge.com")) def key = shortCity +"#"+district_id def ids = retMap.get(key) @@ -83,15 +83,17 @@ class CrawlDistrictIdSelect { if(!keyList.contains(key)){ def map = new BasicDBObject() - map.root = "anjuke.com" + map.root = "zhuge.com" map.city_short= shortCity map.city = db.city - map.region = region + map.region = db.region map.district_id = district_id map.source_url = url ids << map keyList << key + }else{ + println "重复:"+key } }else{ println url +" 链接不合要求" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy index f01c92679..5a8586fb8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy @@ -32,6 +32,22 @@ class InsidePolygon { return list; } + + + public static List createPoint2(String points){ + String[] array = points.split(";"); + List list = new ArrayList(); + for(int i =0 ;i> getPoints(String points){ String[] array = points.split("-"); @@ -167,4 +183,44 @@ class InsidePolygon { return false; return true; } + + + + public static boolean isPtInPoly (double ALon , double ALat , List ps) { + int iSum, iCount, iIndex; + double dLon1 = 0, dLon2 = 0, dLat1 = 0, dLat2 = 0, dLon; + if (ps.size() < 3) { + return false; + } + iSum = 0; + iCount = ps.size(); + for (iIndex = 0; iIndex= dLat1) && (ALat < dLat2)) || ((ALat >= dLat2) && (ALat < dLat1))) { + if (Math.abs(dLat1 - dLat2) > 0) { + //得到 A点向左射线与边的交点的x坐标: + dLon = dLon1 - ((dLon1 - dLon2) * (dLat1 - ALat) ) / (dLat1 - dLat2); + // 如果交点在A点左侧(说明是做射线与 边的交点),则射线与边的全部交点数加一: + if (dLon < ALon) { + iSum++; + } + } + } + } + if ((iSum % 2) != 0) { + return true; + } + return false; + } } From ab4bbbffcd1e36492105df80c2a04f9b2935f025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 7 Jun 2021 13:43:20 +0800 Subject: [PATCH 10/21] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=9B=91=E6=8E=A7?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deliverables/k8s/fangjia-bkoff.yaml | 10 ++++--- .../bkoff/controllers/K8sController.java | 28 +++++++++++++++++++ .../bkoff/interceptor/AuthInterceptor.java | 3 +- 3 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/K8sController.java diff --git a/deliverables/k8s/fangjia-bkoff.yaml b/deliverables/k8s/fangjia-bkoff.yaml index a6e01a7ab..768fd80cd 100644 --- a/deliverables/k8s/fangjia-bkoff.yaml +++ b/deliverables/k8s/fangjia-bkoff.yaml @@ -39,13 +39,15 @@ spec: name: web protocol: TCP readinessProbe: - tcpSocket: - port: 8080 + httpGet: + path: /k8s/readiness + port: web initialDelaySeconds: 5 periodSeconds: 10 livenessProbe: - tcpSocket: - port: 8080 + httpGet: + path: /k8s/liveness + port: web initialDelaySeconds: 10 periodSeconds: 5 volumeMounts: diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/K8sController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/K8sController.java new file mode 100644 index 000000000..11a80b8a2 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/K8sController.java @@ -0,0 +1,28 @@ +package com.fangjia.bkoff.controllers; + +/** + * 房价网后台存活监控 + * + * @author shenjian + * @since 2021/6/7 + */ +public class K8sController { + + /** + * 存活检查 + * + * @return + */ + public Object livenessAction() { + return "liveness"; + } + + /** + * 就绪检查 + * + * @return + */ + public Object readinessAction() { + return "readiness"; + } +} diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/interceptor/AuthInterceptor.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/interceptor/AuthInterceptor.java index 192767d85..111b3c515 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/interceptor/AuthInterceptor.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/interceptor/AuthInterceptor.java @@ -40,7 +40,8 @@ public class AuthInterceptor implements Interceptor { request.setAttribute("__resources", request.getSession().getAttribute(userId + "_resources")); if (invocation.getAction().getController().getName().equals("login") - || invocation.getAction().getController().getName().equals("")) { + || invocation.getAction().getController().getName().equals("") + || invocation.getAction().getController().getName().equals("k8s")) { return invocation.invoke(); } From bec697ac855da23c5d3e6c322ddfba31675a0fa1 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 8 Jun 2021 16:43:33 +0800 Subject: [PATCH 11/21] =?UTF-8?q?=E7=A6=8F=E5=B7=9E=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E5=88=AB=E5=90=8D=E6=94=BE=E5=85=A5=E5=88=AB=E5=90=8D=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fuzhou/DistrictInfoExportBatch.groovy | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index 1af46cb50..042f94b43 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -37,7 +37,7 @@ class DistrictInfoExportBatch extends BaseBatch{ def headers = ["id","city","district","name","lng","lat","property_type","build_type","households", "property_fee","car_site_count","property_life","house_property","open_date","finish_date", "delivery_date","floor_area","build_area","plot_ratio","landscaping_ratio","developer","property_company", - "build_structure","building_count","alias","address", "addr_alias"] + "build_structure","building_count","alias","address"] def ret = createDatas("福州") def fileName = "/fzbank/output/FJW_COMM_"+curDate+".csv" @@ -102,7 +102,7 @@ class DistrictInfoExportBatch extends BaseBatch{ row << rMap.estate_company row << getBuildStructure(rMap.build_structure)//为空的,默认钢混 row << rMap.loudong_count - row << processAlias(rMap.alias) + row << processAlias(rMap.alias, rMap.addr_alias) def addr = fzAddrMap.get(rMap.get("_id").toString()); if(addr){ @@ -110,22 +110,32 @@ class DistrictInfoExportBatch extends BaseBatch{ }else{ row << rMap.address } - - row << processAlias(rMap.addr_alias) + //row << processAlias(rMap.addr_alias) result << row } } return result } - def processAlias(alias){ + def processAlias(alias, def addr_alias){ + def newAlias = [] if(alias){ - def newAlias = [] for(def name : alias){ if(name && !name.contains("(") && !name.contains("、") && !name.contains("?") && !name.contains("(")){ newAlias << name } } + } + + if(addr_alias){ + for(def name : addr_alias){ + if(name && !name.contains("(") && !name.contains("、") && !name.contains("?") && !name.contains("(")){ + newAlias << name + } + } + } + + if(newAlias){ return newAlias.join(","); } return null; From facf6fffe56b307554339c4195d8dc0680fd7a3f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 8 Jun 2021 16:55:03 +0800 Subject: [PATCH 12/21] update --- .../com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index 042f94b43..9934cb1c7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -121,7 +121,7 @@ class DistrictInfoExportBatch extends BaseBatch{ def newAlias = [] if(alias){ for(def name : alias){ - if(name && !name.contains("(") && !name.contains("、") && !name.contains("?") && !name.contains("(")){ + if(name && !name.contains("(") && !name.contains("、") && !name.contains("?") && !name.contains("(") && !newAlias.contains(name)){ newAlias << name } } @@ -129,7 +129,7 @@ class DistrictInfoExportBatch extends BaseBatch{ if(addr_alias){ for(def name : addr_alias){ - if(name && !name.contains("(") && !name.contains("、") && !name.contains("?") && !name.contains("(")){ + if(name && !name.contains("(") && !name.contains("、") && !name.contains("?") && !name.contains("(") && !newAlias.contains(name)){ newAlias << name } } From f3fd6cdef6bc36048c9042a96f762d0ae0f91de5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 9 Jun 2021 14:17:22 +0800 Subject: [PATCH 13/21] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=9B=91=E6=8E=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/gj/GaoDeMapCrawl.groovy | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index 2ba2cdf53..e93f98720 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -3,6 +3,9 @@ package com.fangjia.script.gj import com.fangjia.core.ProvinceData import com.fangjia.core.RegionData import com.fangjia.httpclient.UserAgents +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher import com.fangjia.script.gj.paimai.GaoDeUtils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.JacksonUtil @@ -30,14 +33,15 @@ import java.util.regex.Matcher import java.util.regex.Pattern import java.util.zip.GZIPInputStream -class GaoDeMapCrawl { +@BatchOwner(Watcher.GaoJuan) +class GaoDeMapCrawl extends BaseBatch{ DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018) - static def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc" + def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc" - public static final def dbUrl = "mongodb://192.168.0.235:27018"; + final def dbUrl = "mongodb://192.168.0.235:27018"; - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) final AtomicInteger count = new AtomicInteger() def updateLists = [] def updateErrorLists = [] @@ -50,19 +54,13 @@ class GaoDeMapCrawl { def totalCount = new AtomicLong() static main(args) { - def task = new GaoDeMapCrawl(); - -// def citys = CommonUtils.findAllCity() -// def i =1 -// for(def city : citys){ -// i++ -// println i+" "+ city -// } - - task.run() -// task.allRun(); + runBatch(new GaoDeMapCrawl(), args) } + @Override + void process(Object args) { + run() + } def allRun(){ def cityLations = null; def stop = false; From 0517c8c731b1a514db7d4d26fddded9e88c4ed0a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 10 Jun 2021 10:55:21 +0800 Subject: [PATCH 14/21] =?UTF-8?q?=E6=97=B6=E9=97=B4=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/dealPrice/DealPriceMathBatch.groovy | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/DealPriceMathBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/DealPriceMathBatch.groovy index d9e3c9364..5c5aadd9e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/DealPriceMathBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/DealPriceMathBatch.groovy @@ -54,6 +54,10 @@ class DealPriceMathBatch extends BaseBatch{ def flag = true; + + def startDate = DateUtil.getFirstDayOfMonth(new Date(),-1) + def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1) + cityMap.each{city,alias-> println "开始加载【${city}】小区" if(city != "昆玉"){ @@ -61,10 +65,8 @@ class DealPriceMathBatch extends BaseBatch{ districtMap.putAll(detailMap) } } - def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd")) -// def startDate = DateUtil.getFirstDayOfMonth(new Date()) -// def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1) - def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd")) +// def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd")) +// def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd")) while(flag){ println "开始加载解析数据......" From 1be296f789ffd51cf3305c827ba4cff76e0cb949 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 10 Jun 2021 11:08:53 +0800 Subject: [PATCH 15/21] =?UTF-8?q?=E7=A6=8F=E5=B7=9E=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E5=9D=87=E4=BB=B7=E6=97=B6=E9=97=B4=E4=BF=AE=E6=94=B9=EF=BC=8C?= =?UTF-8?q?=E4=BB=8E=E5=90=8C=E6=AD=A5=E4=B8=8A=E4=B8=AA=E6=9C=88=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=EF=BC=8C=E6=9C=80=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index c2ecb8014..ddfdcb661 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -76,7 +76,7 @@ class DistrictPriceExportBatch extends BaseBatch{ def createDatas(def city){ def districts = CommonDatautils.findAllDistricts(city, "district", null); - def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1) + def lastDate = new Date() def startDate = DateUtils.getFirstDayOfMonth(lastDate); def result = [] From c5800012df538f016516a105c4f96e21de553950 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 10 Jun 2021 11:20:48 +0800 Subject: [PATCH 16/21] update --- .../com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index ddfdcb661..87a2d9962 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -77,7 +77,7 @@ class DistrictPriceExportBatch extends BaseBatch{ def districts = CommonDatautils.findAllDistricts(city, "district", null); def lastDate = new Date() - def startDate = DateUtils.getFirstDayOfMonth(lastDate); + def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1) def result = [] for(def rMap : districts){ From 809bfcb16f7c3593e6453b3866940d55ac44b1e7 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 10 Jun 2021 14:24:54 +0800 Subject: [PATCH 17/21] =?UTF-8?q?=E5=9C=B0=E5=9B=BE=E6=8A=93=E5=8F=96?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/GaoDeMapCrawl.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index e93f98720..3fcc8cd67 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -127,7 +127,7 @@ class GaoDeMapCrawl extends BaseBatch{ q.put(prefix+"status", new BasicDBObject(QueryOperators.EXISTS, false)) // q.put("status", "失败") // q.put("traffic_status", new BasicDBObject(QueryOperators.EXISTS, false)) - def results = BOUNDS_COLL.find(q).batchSize(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def results = BOUNDS_COLL.find(q).limit(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) return results; } @@ -140,7 +140,7 @@ class GaoDeMapCrawl extends BaseBatch{ // void run() { // try{ // totalCount.addAndGet(1) - searching(location) + searching(i, location) // }catch(Exception e){ // e.printStackTrace() // } finally { @@ -167,7 +167,7 @@ class GaoDeMapCrawl extends BaseBatch{ } } - def searching(location){ + def searching(i, location){ // def bound_gd = location.bound_gd // def lats = [bound_gd[1], bound_gd[3]] // def lngs = [bound_gd[0], bound_gd[2]] @@ -175,7 +175,7 @@ class GaoDeMapCrawl extends BaseBatch{ // lngs.sort() // println index // if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){ - search(location) + search(i, location) // } if(updateLists.size() > 0 && updateLists.size()>1000){ @@ -194,7 +194,7 @@ class GaoDeMapCrawl extends BaseBatch{ } - def search(location){ + def search(i, location){ def bound_gd = location.bound_gd def lats = [bound_gd[1], bound_gd[3]] def lngs = [bound_gd[0], bound_gd[2]] @@ -228,7 +228,7 @@ class GaoDeMapCrawl extends BaseBatch{ updateErrorLists.addAll(location) } // update(location); - println "[ "+totalCount.get()+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status + println "[ "+i+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status } def processData(rets){ From f61b4f786372dee76172035ef209c3aeabea9af4 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 10 Jun 2021 17:07:00 +0800 Subject: [PATCH 18/21] update --- .../com/fangjia/script/gj/GaoDeMapCrawl.groovy | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index 3fcc8cd67..129d15ed9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -108,14 +108,20 @@ class GaoDeMapCrawl extends BaseBatch{ def run(){//,"西安","重庆" -// def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"] - def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] -// def citys = CommonUtils.findAllCity() +// def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def citys = CommonUtils.findAllCity() def totalCount = 0; for(def city : citys){ - def cityLations = findAllLation(city); - println "开始处理["+city +"]:"+cityLations.size() - gaoDeCitySearch(cityLations); + def stop = false + while(stop == false){ + def cityLations = findAllLation(city); + if(!cityLations){ + stop = true; + }else{ + println "开始处理["+city +"]:"+cityLations.size() + gaoDeCitySearch(cityLations); + } + } } } From a81ede278ec515a8d5e59d23a21c141b48dbd671 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 18 Jun 2021 16:21:05 +0800 Subject: [PATCH 19/21] =?UTF-8?q?=E9=AB=98=E5=BE=B7=E5=9C=B0=E5=9B=BE?= =?UTF-8?q?=E6=8A=93=E5=8F=96=E6=9D=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/GaoDeMapCrawl.groovy | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index 129d15ed9..e07d6eff0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -47,10 +47,16 @@ class GaoDeMapCrawl extends BaseBatch{ def updateErrorLists = [] def flagLists = [] def index = 0 - def prefix = "district_" + // def categorys = ["141202", "141203"]//中学,小学 - def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关 + +// def prefix = "district_" +// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关 + // def categorys = ["190301"]//道路相关 + + def prefix = "village_" + def categorys = ["190108","190109"]//村庄 def totalCount = new AtomicLong() static main(args) { @@ -108,8 +114,8 @@ class GaoDeMapCrawl extends BaseBatch{ def run(){//,"西安","重庆" -// def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] - def citys = CommonUtils.findAllCity() + def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] +// def citys = CommonUtils.findAllCity() def totalCount = 0; for(def city : citys){ def stop = false From 6d6c7cab29c33628c611768f5c3ffc9bfa96a485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 18 Jun 2021 18:08:24 +0800 Subject: [PATCH 20/21] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=91=A8=E8=BE=B9POI?= =?UTF-8?q?=E6=90=9C=E7=B4=A2=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 7 +- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 185 ++++++++++++++++ .../temp/shenjian/util/GaoDeUtils.groovy | 181 +++++++++++++++ .../temp/shenjian/util/HttpUtils.groovy | 29 +++ .../temp/shenjian/util/SeaweedUtils.groovy | 208 ++++++++++++++++++ .../script/statistics/poi/AmapPoi.java | 71 ++++++ 6 files changed, 680 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index ee26c1830..75ca2b1ff 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -482,7 +482,12 @@ j2v8_win32_x86_64 4.6.0 - + + org.projectlombok + lombok + 1.18.20 + provided + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy new file mode 100644 index 000000000..00ee277ea --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -0,0 +1,185 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取 + * + * @author shenjian + * @since 2021/5/24 + */ +@BatchOwner(Watcher.ShenJian) +class SeaweedAroundPoi extends BaseBatch { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } + + /** + * 有限处理招标城市 + */ + static dealSpecialSeaweedAround(def types) { + def cities = ["重庆", "达州", "曲靖", "昆明", "大理", "迪庆州", "福安", "平潭", "三明", "石狮", "柳州", "苏州"] + for (def city : cities) { + DBObject q = new BasicDBObject(); + q.put("city", city); + if (city == "福安") { + city = "宁德" + q.put("city", city) + q.put("region", "福安") + } else if (city == "平潭") { + city = "福州" + q.put("city", city) + q.put("region", "平潭") + } else if (city == "石狮") { + city = "泉州" + q.put("city", city) + q.put("region", "石狮") + } + dealPoi(q, types) + } + } + + static dealPoi(DBObject query, def types) { + query.put("status", 1) + conn136.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + }) + } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List list = [] + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { + // 已存在小区,无需同步 + } else { + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + list << newObj + if (list.size() == 5000) { + conn136.insert(list) + list.clear() + } + } + } + conn136.insert(list) + list.clear() + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } + + public static final Map KEYWORDS_MAP = new HashMap(){{ + put("地铁", "150500"); + put("公交", "1507"); + put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); + put("写字楼", "120201"); + put("学校","141200|141201|141202|141203|141204"); + put("银行","1601|1603"); + put("大型超市","0604"); + put("商场","0601"); + put("餐饮","0501|0502|0503|0504"); + put("旅游景点","110000"); + put("图书馆","140500"); + }}; + + @Override + void process(Object args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy new file mode 100644 index 000000000..5adde74e3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -0,0 +1,181 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSONArray +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.DataUtil +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.HttpRequestUtil +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +import java.util.concurrent.atomic.AtomicInteger + +/** + * 高德工具类 + * + * @author shenjian + * @since 2021/5/15 + */ +class GaoDeUtils { + + // 新获取key + def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "35925efd7333f1df52145e51b8e4e56a"] + def static AtomicInteger keyUsed = new AtomicInteger(0) + + def static codeNameMap = new HashMap(){{ + put("150500", "地铁"); + put("1507", "公交"); + put("090100", "医院") + put("090101", "医院") + put("090200", "医院") + put("090202", "医院") + put("090202", "医院") + put("090204", "医院") + put("090205", "医院") + put("090206", "医院") + put("090206", "医院") + put("090208", "医院") + put("090209", "医院") + put("090210", "医院") + put("090211", "医院") + put("141200", "学校") + put("141201", "学校") + put("141202", "学校") + put("141203", "学校") + put("141204", "学校") + put("1601", "银行") + put("1603", "银行") + put("0601", "商场") + }}; + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|1603|0601"; @see com.fangjia.script.wenyan.auto.KeywordSearchUtils.KEYWORDS_MAP.get(kw) + * @return + */ + static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed] + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + result.addAll(DataUtil.toMongoDbObj(json.pois)) + } else if (json && json.infocode && json.infocode == 10003) { + throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + } else if (json && json.infocode && json.infocode == 10004) { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types + * @return + */ + static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed.intValue()] + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + result.addAll(ret) + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + //throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + // 多线程情况下还是exit有效果,抛异常不会停止程序 + if (keyUsed.intValue() >= GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed.incrementAndGet() + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 获取经纬度之间距离 + */ + static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { + return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) + } + + /** + * 获取地址经纬度 + */ + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy new file mode 100644 index 000000000..ff11d6991 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.temp.shenjian.util + +import org.apache.commons.httpclient.HttpClient +import org.apache.commons.httpclient.methods.PostMethod +import org.apache.commons.httpclient.methods.StringRequestEntity + +/** + * + * @author shenjian + * @since 2021/5/25 + */ +class HttpUtils { + + static boolean postForJson(String url, String jsonString) { + HttpClient httpClient = new HttpClient() + StringRequestEntity requestEntity = new StringRequestEntity( + jsonString, + "application/json", + "UTF-8"); + + PostMethod postMethod = new PostMethod(url) + postMethod.setRequestEntity(requestEntity); + int statusCode = httpClient.executeMethod(postMethod); + if (statusCode == 200) { + return true + } + return false + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy new file mode 100644 index 000000000..9fa4739f8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -0,0 +1,208 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.datastore.FieldValidatorUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.elasticsearch.geometry.utils.Geohash + +import java.util.concurrent.ConcurrentHashMap + +/** + * 小区工具类 + * + * @author shenjian + * @since 2021/5/17 + */ +class SeaweedUtils { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static def map = new ConcurrentHashMap() + + static getRegionByCode(def district_code) { + def region; + if (map.get("district_code") != null) { + region = map.get("district_code") + } else { + cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> + region = obj.fangjia_district + map.put("district_code", region) + } + } + return region + } + + /** + * 标准化输出绿化率 + */ + static formatVirescencePercent(def virescence_percent) { + // 绿化率 + if (virescence_percent && FieldValidatorUtils.checkValue(virescence_percent)) { + double val = Double.valueOf(virescence_percent) + if (val < 1) { + return val * 100 + } + if (val == 1) { + return val * 10 + } + if (val >= 70) { + return 65 + } + return val + } + return "" + } + + /** + * 标准化容积率输出 + */ + static formatCubagePercent(def cubage_percent) { + if (cubage_percent && FieldValidatorUtils.checkValue(cubage_percent)) { + double val = Double.valueOf(cubage_percent) + if (val >= 7 && val < 8) { + return 6.2 + } + if (val >= 8 && val < 9) { + return 6.3 + } + if (val >= 9 && val < 10) { + return 6.5 + } + if (val >= 10) { + return 6.7 + } + return val + } + return "" + } + + /** + * 标准化车位输出 + */ + static formatCarSiteCount(def car_site_count) { + if (car_site_count && FieldValidatorUtils.checkValue(car_site_count)) { + long val = Long.valueOf(car_site_count) + if (val > 2000 || val < 100) { + return "" + } + return val; + } + return "" + } + + /** + * 标准化建筑面积输出 + */ + static formatBuildArea(def build_area) { + if (build_area && FieldValidatorUtils.checkValue(build_area)) { + double val = build_area + if (val > 1000) { + return val + } + } + return "" + } + + /** + * 标准化总占地面积输出 + */ + static formatAllArea(def all_area) { + if (all_area) { + Double formattedVal = 0.0; + if (all_area.toString().indexOf("公顷") != -1) { + def val = all_area.toString() + formattedVal = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000) + } else if(FieldValidatorUtils.checkValue(all_area)) { + formattedVal = Double.valueOf(all_area) + } + if (formattedVal > 100000 || formattedVal == 0.0) { + return "" + } + return formattedVal + } + } + + /** + * 标准化户数输出 + */ + static formatHouseholdCount(def household_count) { + if (household_count && FieldValidatorUtils.checkValue(household_count) ) { + if (Long.valueOf(household_count) < 50) { + return "" // 户数 + } + return household_count + } + } + + /** + * 计算车位比 + */ + static calcCarAndHouseholdRadio(def car_site_count, def household_count) { + if (SeaweedUtils.formatCarSiteCount(car_site_count) && SeaweedUtils.formatHouseholdCount(household_count)) { + long formattedCarSiteCount = SeaweedUtils.formatCarSiteCount(car_site_count) + long formattedHouseholdCount = SeaweedUtils.formatHouseholdCount(household_count) + double val = DataHandleUtils.getDoubleRound(1, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(2, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(3, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 1.0) { + return "1:1" + } + return "1:" + val + } + return "" + } + + /** + * 标准化别名输出 + */ + static formatAlias(def alias) { + if (alias) { + // 小区可能存在多个别名 + def temp = [] + int len = 0 + for (def a : alias) { + len += a.toString().length() + if (len > 20) { + return temp.join(",") + } + temp.add(a) + } + return temp.join(",") + } + return "" + } + + /** + * 格式化经纬度 + */ + static def formatLatLng(def latOrLng) { + // 107.109878,29.192381 + return com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, latOrLng / 10000000000d) + } + + /** + * 小区经纬度geohash编码 + * @param args + */ + static def encodeByGeoHash(double lon, double lat, int level=7) { + return Geohash.stringEncode(lon, lat, level) + } + + /** + * 小区经纬度geohash解码 + * @param args + */ + static def decodeByGeoHash(String geohash) { + return Geohash.decodeLongitude(geohash) + "," + Geohash.decodeLatitude(geohash) + } + + static void main(String[] args) { + def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"] + println(formatAlias(alias)) + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java new file mode 100644 index 000000000..4a731fe11 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java @@ -0,0 +1,71 @@ +package com.fangjia.script.statistics.poi; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/5/25 + */ +@lombok.NoArgsConstructor +@lombok.Data +public class AmapPoi { + + @com.alibaba.fastjson.annotation.JSONField(name = "id") + private String id; + @com.alibaba.fastjson.annotation.JSONField(name = "parent") + private List parent; + @com.alibaba.fastjson.annotation.JSONField(name = "childtype") + private List childtype; + @com.alibaba.fastjson.annotation.JSONField(name = "name") + private String name; + @com.alibaba.fastjson.annotation.JSONField(name = "type") + private String type; + @com.alibaba.fastjson.annotation.JSONField(name = "typecode") + private String typecode; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_type") + private List bizType; + @com.alibaba.fastjson.annotation.JSONField(name = "address") + private String address; + @com.alibaba.fastjson.annotation.JSONField(name = "location") + private String location; + @com.alibaba.fastjson.annotation.JSONField(name = "tel") + private String tel; + @com.alibaba.fastjson.annotation.JSONField(name = "pname") + private String pname; + @com.alibaba.fastjson.annotation.JSONField(name = "cityname") + private String cityname; + @com.alibaba.fastjson.annotation.JSONField(name = "adname") + private String adname; + @com.alibaba.fastjson.annotation.JSONField(name = "importance") + private List importance; + @com.alibaba.fastjson.annotation.JSONField(name = "shopid") + private List shopid; + @com.alibaba.fastjson.annotation.JSONField(name = "shopinfo") + private String shopinfo; + @com.alibaba.fastjson.annotation.JSONField(name = "poiweight") + private List poiweight; + @com.alibaba.fastjson.annotation.JSONField(name = "distance") + private String distance; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_ext") + private BizExtDTO bizExt; + @com.alibaba.fastjson.annotation.JSONField(name = "photos") + private List photos; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BizExtDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "rating") + private List rating; + @com.alibaba.fastjson.annotation.JSONField(name = "cost") + private String cost; + } + + @lombok.NoArgsConstructor + @lombok.Data + public static class PhotosDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "title") + private List title; + @com.alibaba.fastjson.annotation.JSONField(name = "url") + private String url; + } +} From 7042809ecd9bf1fa3af87232fa71c16f124fd914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 18 Jun 2021 18:08:24 +0800 Subject: [PATCH 21/21] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=A4=9A=E7=BA=BF?= =?UTF-8?q?=E7=A8=8B=E5=A4=9Akey=E5=81=9C=E6=AD=A2bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 7 +- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 185 ++++++++++++++++ .../temp/shenjian/util/GaoDeUtils.groovy | 186 ++++++++++++++++ .../temp/shenjian/util/HttpUtils.groovy | 29 +++ .../temp/shenjian/util/SeaweedUtils.groovy | 208 ++++++++++++++++++ .../script/statistics/poi/AmapPoi.java | 71 ++++++ 6 files changed, 685 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index ee26c1830..75ca2b1ff 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -482,7 +482,12 @@ j2v8_win32_x86_64 4.6.0 - + + org.projectlombok + lombok + 1.18.20 + provided + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy new file mode 100644 index 000000000..00ee277ea --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -0,0 +1,185 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取 + * + * @author shenjian + * @since 2021/5/24 + */ +@BatchOwner(Watcher.ShenJian) +class SeaweedAroundPoi extends BaseBatch { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } + + /** + * 有限处理招标城市 + */ + static dealSpecialSeaweedAround(def types) { + def cities = ["重庆", "达州", "曲靖", "昆明", "大理", "迪庆州", "福安", "平潭", "三明", "石狮", "柳州", "苏州"] + for (def city : cities) { + DBObject q = new BasicDBObject(); + q.put("city", city); + if (city == "福安") { + city = "宁德" + q.put("city", city) + q.put("region", "福安") + } else if (city == "平潭") { + city = "福州" + q.put("city", city) + q.put("region", "平潭") + } else if (city == "石狮") { + city = "泉州" + q.put("city", city) + q.put("region", "石狮") + } + dealPoi(q, types) + } + } + + static dealPoi(DBObject query, def types) { + query.put("status", 1) + conn136.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + }) + } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List list = [] + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { + // 已存在小区,无需同步 + } else { + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + list << newObj + if (list.size() == 5000) { + conn136.insert(list) + list.clear() + } + } + } + conn136.insert(list) + list.clear() + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } + + public static final Map KEYWORDS_MAP = new HashMap(){{ + put("地铁", "150500"); + put("公交", "1507"); + put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); + put("写字楼", "120201"); + put("学校","141200|141201|141202|141203|141204"); + put("银行","1601|1603"); + put("大型超市","0604"); + put("商场","0601"); + put("餐饮","0501|0502|0503|0504"); + put("旅游景点","110000"); + put("图书馆","140500"); + }}; + + @Override + void process(Object args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy new file mode 100644 index 000000000..74505e6b5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -0,0 +1,186 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSONArray +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.DataUtil +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.HttpRequestUtil +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +import java.util.concurrent.atomic.AtomicInteger + +/** + * 高德工具类 + * + * @author shenjian + * @since 2021/5/15 + */ +class GaoDeUtils { + + // 新获取key + def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "35925efd7333f1df52145e51b8e4e56a"] + def static AtomicInteger keyUsed = new AtomicInteger(0) + + def static codeNameMap = new HashMap(){{ + put("150500", "地铁"); + put("1507", "公交"); + put("090100", "医院") + put("090101", "医院") + put("090200", "医院") + put("090202", "医院") + put("090202", "医院") + put("090204", "医院") + put("090205", "医院") + put("090206", "医院") + put("090206", "医院") + put("090208", "医院") + put("090209", "医院") + put("090210", "医院") + put("090211", "医院") + put("141200", "学校") + put("141201", "学校") + put("141202", "学校") + put("141203", "学校") + put("141204", "学校") + put("1601", "银行") + put("1603", "银行") + put("0601", "商场") + }}; + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|1603|0601"; @see com.fangjia.script.wenyan.auto.KeywordSearchUtils.KEYWORDS_MAP.get(kw) + * @return + */ + static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed] + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + result.addAll(DataUtil.toMongoDbObj(json.pois)) + } else if (json && json.infocode && json.infocode == 10003) { + throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + } else if (json && json.infocode && json.infocode == 10004) { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types + * @return + */ + static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed.intValue()] + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + result.addAll(ret) + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 > GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + break; + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 获取经纬度之间距离 + */ + static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { + return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) + } + + /** + * 获取地址经纬度 + */ + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy new file mode 100644 index 000000000..ff11d6991 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.temp.shenjian.util + +import org.apache.commons.httpclient.HttpClient +import org.apache.commons.httpclient.methods.PostMethod +import org.apache.commons.httpclient.methods.StringRequestEntity + +/** + * + * @author shenjian + * @since 2021/5/25 + */ +class HttpUtils { + + static boolean postForJson(String url, String jsonString) { + HttpClient httpClient = new HttpClient() + StringRequestEntity requestEntity = new StringRequestEntity( + jsonString, + "application/json", + "UTF-8"); + + PostMethod postMethod = new PostMethod(url) + postMethod.setRequestEntity(requestEntity); + int statusCode = httpClient.executeMethod(postMethod); + if (statusCode == 200) { + return true + } + return false + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy new file mode 100644 index 000000000..9fa4739f8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -0,0 +1,208 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.datastore.FieldValidatorUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.elasticsearch.geometry.utils.Geohash + +import java.util.concurrent.ConcurrentHashMap + +/** + * 小区工具类 + * + * @author shenjian + * @since 2021/5/17 + */ +class SeaweedUtils { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static def map = new ConcurrentHashMap() + + static getRegionByCode(def district_code) { + def region; + if (map.get("district_code") != null) { + region = map.get("district_code") + } else { + cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> + region = obj.fangjia_district + map.put("district_code", region) + } + } + return region + } + + /** + * 标准化输出绿化率 + */ + static formatVirescencePercent(def virescence_percent) { + // 绿化率 + if (virescence_percent && FieldValidatorUtils.checkValue(virescence_percent)) { + double val = Double.valueOf(virescence_percent) + if (val < 1) { + return val * 100 + } + if (val == 1) { + return val * 10 + } + if (val >= 70) { + return 65 + } + return val + } + return "" + } + + /** + * 标准化容积率输出 + */ + static formatCubagePercent(def cubage_percent) { + if (cubage_percent && FieldValidatorUtils.checkValue(cubage_percent)) { + double val = Double.valueOf(cubage_percent) + if (val >= 7 && val < 8) { + return 6.2 + } + if (val >= 8 && val < 9) { + return 6.3 + } + if (val >= 9 && val < 10) { + return 6.5 + } + if (val >= 10) { + return 6.7 + } + return val + } + return "" + } + + /** + * 标准化车位输出 + */ + static formatCarSiteCount(def car_site_count) { + if (car_site_count && FieldValidatorUtils.checkValue(car_site_count)) { + long val = Long.valueOf(car_site_count) + if (val > 2000 || val < 100) { + return "" + } + return val; + } + return "" + } + + /** + * 标准化建筑面积输出 + */ + static formatBuildArea(def build_area) { + if (build_area && FieldValidatorUtils.checkValue(build_area)) { + double val = build_area + if (val > 1000) { + return val + } + } + return "" + } + + /** + * 标准化总占地面积输出 + */ + static formatAllArea(def all_area) { + if (all_area) { + Double formattedVal = 0.0; + if (all_area.toString().indexOf("公顷") != -1) { + def val = all_area.toString() + formattedVal = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000) + } else if(FieldValidatorUtils.checkValue(all_area)) { + formattedVal = Double.valueOf(all_area) + } + if (formattedVal > 100000 || formattedVal == 0.0) { + return "" + } + return formattedVal + } + } + + /** + * 标准化户数输出 + */ + static formatHouseholdCount(def household_count) { + if (household_count && FieldValidatorUtils.checkValue(household_count) ) { + if (Long.valueOf(household_count) < 50) { + return "" // 户数 + } + return household_count + } + } + + /** + * 计算车位比 + */ + static calcCarAndHouseholdRadio(def car_site_count, def household_count) { + if (SeaweedUtils.formatCarSiteCount(car_site_count) && SeaweedUtils.formatHouseholdCount(household_count)) { + long formattedCarSiteCount = SeaweedUtils.formatCarSiteCount(car_site_count) + long formattedHouseholdCount = SeaweedUtils.formatHouseholdCount(household_count) + double val = DataHandleUtils.getDoubleRound(1, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(2, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(3, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 1.0) { + return "1:1" + } + return "1:" + val + } + return "" + } + + /** + * 标准化别名输出 + */ + static formatAlias(def alias) { + if (alias) { + // 小区可能存在多个别名 + def temp = [] + int len = 0 + for (def a : alias) { + len += a.toString().length() + if (len > 20) { + return temp.join(",") + } + temp.add(a) + } + return temp.join(",") + } + return "" + } + + /** + * 格式化经纬度 + */ + static def formatLatLng(def latOrLng) { + // 107.109878,29.192381 + return com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, latOrLng / 10000000000d) + } + + /** + * 小区经纬度geohash编码 + * @param args + */ + static def encodeByGeoHash(double lon, double lat, int level=7) { + return Geohash.stringEncode(lon, lat, level) + } + + /** + * 小区经纬度geohash解码 + * @param args + */ + static def decodeByGeoHash(String geohash) { + return Geohash.decodeLongitude(geohash) + "," + Geohash.decodeLatitude(geohash) + } + + static void main(String[] args) { + def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"] + println(formatAlias(alias)) + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java new file mode 100644 index 000000000..4a731fe11 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java @@ -0,0 +1,71 @@ +package com.fangjia.script.statistics.poi; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/5/25 + */ +@lombok.NoArgsConstructor +@lombok.Data +public class AmapPoi { + + @com.alibaba.fastjson.annotation.JSONField(name = "id") + private String id; + @com.alibaba.fastjson.annotation.JSONField(name = "parent") + private List parent; + @com.alibaba.fastjson.annotation.JSONField(name = "childtype") + private List childtype; + @com.alibaba.fastjson.annotation.JSONField(name = "name") + private String name; + @com.alibaba.fastjson.annotation.JSONField(name = "type") + private String type; + @com.alibaba.fastjson.annotation.JSONField(name = "typecode") + private String typecode; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_type") + private List bizType; + @com.alibaba.fastjson.annotation.JSONField(name = "address") + private String address; + @com.alibaba.fastjson.annotation.JSONField(name = "location") + private String location; + @com.alibaba.fastjson.annotation.JSONField(name = "tel") + private String tel; + @com.alibaba.fastjson.annotation.JSONField(name = "pname") + private String pname; + @com.alibaba.fastjson.annotation.JSONField(name = "cityname") + private String cityname; + @com.alibaba.fastjson.annotation.JSONField(name = "adname") + private String adname; + @com.alibaba.fastjson.annotation.JSONField(name = "importance") + private List importance; + @com.alibaba.fastjson.annotation.JSONField(name = "shopid") + private List shopid; + @com.alibaba.fastjson.annotation.JSONField(name = "shopinfo") + private String shopinfo; + @com.alibaba.fastjson.annotation.JSONField(name = "poiweight") + private List poiweight; + @com.alibaba.fastjson.annotation.JSONField(name = "distance") + private String distance; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_ext") + private BizExtDTO bizExt; + @com.alibaba.fastjson.annotation.JSONField(name = "photos") + private List photos; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BizExtDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "rating") + private List rating; + @com.alibaba.fastjson.annotation.JSONField(name = "cost") + private String cost; + } + + @lombok.NoArgsConstructor + @lombok.Data + public static class PhotosDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "title") + private List title; + @com.alibaba.fastjson.annotation.JSONField(name = "url") + private String url; + } +}