diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index 95e40fb9b..fdf2ce5f9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -165,6 +165,9 @@ class DistrictInfoExportBatch extends BaseBatch{ rets << "非钢混" } } + if(rets.size()==0){ + rets << "钢混" + } return rets.join(",") } return "钢混"; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy index c08426b9c..6a36f086d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy @@ -18,7 +18,7 @@ import org.apache.commons.codec.digest.DigestUtils */ class ThirdPriceExportBatch extends BaseBatch{ - def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")); + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -156,12 +156,12 @@ class ThirdPriceExportBatch extends BaseBatch{ if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){ retMap.cityhouse_esf_price = d.cityhouse_esf_price } - if(!retMap.fangtan_esf_price && d.fangtan_esf_price){ - retMap.fangtan_esf_price = d.fangtan_esf_price + if(!retMap.fangtan_esf_price && d.jiwu_esf_price){//使用吉屋替换房探 待抓取 + retMap.fangtan_esf_price = d.jiwu_esf_price } - if(!retMap.fangdoudou_esf_price && d.fangdoudou_esf_price){ - retMap.fangdoudou_esf_price = d.fangdoudou_esf_price + if(!retMap.fangdoudou_esf_price && d.get("365house_esf_price")){//使用淘房365替换房多多 + retMap.fangdoudou_esf_price = d.get("365house_esf_price") } if(!retMap.maitian_esf_price && d.maitian_esf_price){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 32b251e77..25c631832 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -47,7 +47,7 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-03-01" + static def statDate = "2021-04-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2021年5月\\二手房\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index e0f5df506..033464eb5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -9,7 +9,11 @@ class FangjiaClientUtils { public static def getMapData(def city, def region, def name, def address, def types, def mapCode){ JSONObject jsonObj = null try { - def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode + def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode + //def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode + if(address){ + url = url + "&address="+address + } // def url = "http://localhost:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode def content = new HttpRequest().newGet(url, null, "utf-8") jsonObj = JSONObject.fromObject(content) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalDBUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalDBUtil.groovy index 4f6bc253e..c0cc1f468 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalDBUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalDBUtil.groovy @@ -25,6 +25,7 @@ class CrawalDBUtil { static DBCollection data_process_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "data_process") static DBCollection average_statistics_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "average_statistics") static DBCollection map_district_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "map_district_process") + static DBCollection districtDetailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail") public static getSeedsCol() { return seedsColl; @@ -67,6 +68,10 @@ class CrawalDBUtil { return map_district_col; } + public static getDistrictDetailColl(){ + return districtDetailColl; + } + static def updateSeedsCount(def id, def count) { DBObject uquery = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 3a8ee77b2..c8ab1b6d8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -61,7 +61,7 @@ class DataStandardBatch extends BaseBatch{ def static final SHORT_DATE = Utils.getLastMonth() def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6] - def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"] + def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) static { thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 67e67e520..5f96db29c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -30,7 +30,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ "房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf", "链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new", "乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf", - "房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf" + "房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf" ] public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index f35379b65..a23842026 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -309,6 +309,31 @@ public class OpenApiClient { return null } + + public static JSONObject addressStandard(String city, String address) { + if (StringUtils.isBlank(city) || StringUtils.isBlank(address)) { + throw new IllegalArgumentException(); + } + + try { + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8") + JSONObject result = JSONObject.fromObject(content) + if (result.getInt("code") == 200) { + return result.getJSONObject("result"); + } else if (result.getInt("code") == 500) { + println("${city} ${address} ${result}") + } + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + public static JSONObject evaluateByGeo(String city, def lat, def lng, String buildDate, String estateType, String geoType) { try { Map params = new HashMap(); @@ -1214,4 +1239,6 @@ public class OpenApiClient { return null; } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeMapApiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeMapApiData.groovy index 4d472474c..c9e44b3e0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeMapApiData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeMapApiData.groovy @@ -223,6 +223,32 @@ class GaoDeMapApiData { doSearchByGaoDe(type, lng, lat, radius, page) } } + + def static doSearchByGaoDe(String type, Double lng, Double lat, int radius, int page, int pageSize) { + Map params = new HashMap() + params.put("key", GAO_DE_AK) + params.put("types", type) + params.put("location", Double.toString(lng)+ ","+ Double.toString(lat)) + params.put("radius",radius) + params.put("offset",pageSize) + params.put("page",page) + String content + try { + content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8") + } catch (Exception e) { + content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8") + } + JSONObject jsonObj = JSONObject.fromObject(content) + + if(jsonObj.get("infocode").equals("10000")){ + JSONArray pois = jsonObj.get("pois") + return pois + } else { + println jsonObj.get("infocode")+"exit........" + doSearchByGaoDe(type, lng, lat, radius, page, pageSize) + } + return null + } private synchronized static fzData(JSONArray pois) { pois.each { poi ->