From 4fb29fc530bd8361f4bfb8bf67ee8dc4ba33cc05 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 15 Aug 2023 17:07:59 +0800 Subject: [PATCH 01/13] update --- .../com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy | 5 +++-- .../com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy | 6 +++--- .../fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 3b25072f8..459e3e505 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -140,6 +140,7 @@ class DistrictSellStatBatch extends BaseBatch { try { totalCount.addAndGet(1) def key = dist.region +"##" + dist.name + def weekendPriceObj = weekendPriceMap.get(key) if(weekendPriceObj){ def curPriceObj = curMonthlyPriceMap.get(key) @@ -155,7 +156,7 @@ class DistrictSellStatBatch extends BaseBatch { if(curPriceObj){ curPrice = curPriceObj.avg_price }else{ - curPrice = weekendPriceObj.avg_price + curPrice = weekendPriceObj.avg_price } if(weekendPriceObj.value_type!=1){ @@ -197,7 +198,7 @@ class DistrictSellStatBatch extends BaseBatch { data.put("commid", dist._id.toString()) data.put("mom", mom) data.put("yoy", yoy) - data.put("avg_price", curPrice) + data.put("avg_price", weekendPriceObj.avg_price) DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false) println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy index f7dcc4293..11d76c586 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy @@ -76,7 +76,7 @@ class ThirdPriceExportBatch extends BaseBatch{ row << dist.name row << o.s_date - def zgf_price = getLastThirdPrice(o.zhugezhaofang_esf_price, curPrice) + def zgf_price = getLastThirdPrice(o.xingfuli_esf_price, curPrice) row << zgf_price if(zgf_price){ add = true; @@ -217,8 +217,8 @@ class ThirdPriceExportBatch extends BaseBatch{ if(!retMap.s_date){ retMap.s_date = curDate } - if(!retMap.zhugezhaofang_esf_price && d.zhugezhaofang_esf_price){ - retMap.zhugezhaofang_esf_price = d.zhugezhaofang_esf_price + if(!retMap.xingfuli_esf_price && d.xingfuli_esf_price){ + retMap.xingfuli_esf_price = d.xingfuli_esf_price } if(!retMap.anjuke_esf_price && d.get("anjuke_esf_price")){ retMap.anjuke_esf_price = d.get("anjuke_esf_price") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 1d0ca1224..18fdb2d2f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -64,7 +64,7 @@ class DataStandardBatch extends BaseBatch{ def static final SHORT_DATE = Utils.getLastMonth() def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7] - def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] + def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里"] public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) static { thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); @@ -92,7 +92,7 @@ class DataStandardBatch extends BaseBatch{ cityMap.each{city,alias-> println "开始加载【${city}】" -// if(city == "石家庄"){ +// if(city == "福州"){ def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) districtMap.putAll(detailMap) // } From af35c80ff01c5b71ad9090c7c35ad7ea42a209a4 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 15 Aug 2023 17:09:47 +0800 Subject: [PATCH 02/13] update --- .../fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy | 4 ++-- .../script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 175b2ebd1..7c9ed802c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -27,7 +27,7 @@ class SynchDistrictPrice extends BaseBatch{ // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田", // "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士" // def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"] - def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产"] + def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产","幸福里APP":"幸福里"] static main(args) { runBatch(new SynchDistrictPrice(), args); @@ -41,7 +41,7 @@ class SynchDistrictPrice extends BaseBatch{ def run1(){ println "starting......" - def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") + def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), 0), "yyyy-MM-dd") def citys = CommonUtils.findAllCity() esSources.each{esSource, mgSource-> for(def city : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 4fe5dd4a7..ad821196e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -32,7 +32,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ "房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf", "链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new", "乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf", - "房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf" + "房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf" ] public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2) From 64c1f4e1437ec94bb63f9e32831bb3e1b421fae3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 17 Aug 2023 14:48:36 +0800 Subject: [PATCH 03/13] update --- .../ccbevaluation/BigdataDealBatch.groovy | 2 +- .../script/commData/CommonDatautils.groovy | 120 ++++++++++ .../fuzhou/DistrictPriceExportBatch.groovy | 37 ++- .../fuzhou/DistrictSellStatBatch.groovy | 3 + .../script/fuzhou/FzDataResultStat.groovy | 62 ++--- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../fangjia/script/gj/CityStatExport.groovy | 35 ++- .../fangjia/script/gj/ccb/FileUtils.groovy | 49 ++++ .../gj/district/FangjiaClientUtils.groovy | 28 +++ .../gj/district/OfficeFieldCleanBatch.groovy | 37 ++- .../fangjia/script/gj/es/ExportHouse.groovy | 72 ++++++ .../gj/paimai/BeiKeCrawlByChrome.groovy | 74 ++++-- .../gj/paimai/LianjiaCrawlByChrome.groovy | 89 +++++--- .../gj/xiamen/GaoDePoiDataProcess.groovy | 211 ++++++++++++++++++ .../gj/xiamen/PoiAndSeaweedExportCSV.groovy | 35 +++ .../fangjia/script/wenyan/temp/ZhuoXin.groovy | 44 ++++ .../yinjihaun/amap/GaoDeApiUtils.groovy | 2 +- .../fangjia/script/zsbank/ZheShangBank.groovy | 83 +++++++ 18 files changed, 888 insertions(+), 97 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/ExportHouse.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiAndSeaweedExportCSV.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ZhuoXin.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/ZheShangBank.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 1f2867d17..03bc9ebbe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -217,7 +217,7 @@ class BigdataDealBatch extends BaseBatch{ def price = ccbPriceMap.get(obj._id.toString()) if(price){ def mod = java.lang.Math.abs((trade.price-price)*100/price) - if(mod<=10){ + if(mod<=20){ ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() ret << trade.city diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 5102257a3..2b6c315b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -3941,5 +3941,125 @@ class CommonDatautils { return files; } + static def findLogs(userId, requestUri, startDate, endDate, code){ + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri)) + if(startDate){ + def startTime = DateUtils.parseFromTightYYYYMMDD(startDate) + def endTime = DateUtils.parseFromTightYYYYMMDD(endDate) + def date_range_cdate = QueryBuilders.rangeQuery("c_date").from(startTime.getTime()).to(endTime.getTime()).includeLower(true).includeUpper(false); + queryBuilder.must(date_range_cdate); + } + if(code){ + queryBuilder.must(QueryBuilders.termQuery("code", code)) + } + def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); + return rets.size() + } + + static def findLogsBySql(userId, requestUri, startDate, endDate, code){ + String sql = "select count(*) as total from open_uri_log where user_id=${userId} and request_uri='${requestUri}' " + if(startDate){ + sql = sql + "and c_date >= '${startDate}' and c_date < '${endDate}'" + } + if(code){ + sql = sql + " and code=${code}" + } + def total = null + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.total + }) + return total; + } + + static def findAmapDetailBySql(city){ + String sql = "select * from amap_poi_detail where cityname='${city}' limit 10" + def rets = [] + ElasticUtils.executeSqlQuery(sql,{data-> +// if(data.name.contains('华山小区G栋')){ + def point = data.location +// def tag = null +// if(data.get("business.keytag")){ +// tag = data.get("business.keytag") +// } + def business_area = null + if(data.get("business.business_area")){ + business_area = data.get("business.business_area") + } + def location = [:] + location.lon = point.x + location.lat = point.y + rets << ["province":data.pname,"city":data.cityname, "region":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area] +// } + + }) + return rets; + } + + + //用sql有些字段取不出来 + static def findDetailLogsBySql(userId, requestUri){ + String sql = "select * from open_uri_log where user_id=${userId} and request_uri='${requestUri}' and code!=200" + + sql = sql + " order by c_date desc" + def total = null + def rets = [] + ElasticUtils.executeSqlQuery(sql,{data-> + def ret = [] + println data.toString() + ret << data.get("parameter_map.city") +// ret << data.get("parameter_map.district") +// ret << data.get("parameter_map.keyword") +// ret << data.get("parameter_map.name") + ret << data.get("parameter_map.address") +// ret << data.get("parameter_map.size") + ret << data.get("msg") + ret << data.get("code") + ret << data.get("c_date").toString()[0..10] + rets << ret + }) + return rets; + } + + static def findNewAmapDetail(city) { + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fjw_city", city)) + def rets = ElasticUtils.executeSimpleQuery("192.168.0.12", "amap_poi", queryBuilder); + def results = [] + if(rets) { + for (int k = 0; k < rets.size(); k++) { + def data = rets[k]; + println data + } + } + return results; + } + + + static def findLogs2(userId, requestUri){ + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri)) + + queryBuilder.mustNot(QueryBuilders.termQuery("code", 200)) + def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); + def results = [] + if(rets) { + for (int k = 0; k < rets.size(); k++) { + def data = rets[k]; + println data + def ret = [] + println data.toString() + ret << data.parameter_map.city + ret << data.parameter_map.district + ret << data.parameter_map.name + ret << data.parameter_map.address + ret << data.msg + ret << data.code + ret << data.c_date.toString()[0..10] + results << ret + } + } + return results + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 1e5037735..4ec710a3c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -47,19 +47,29 @@ class DistrictPriceExportBatch extends BaseBatch{ def ret = createNewDatas("福州") saveToDB(ret) + def results = selectLastResultDatas(ret); + def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv" def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv" dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512" } - def sb = appendResult(headers, ret) + def sb = appendResult(headers, results) DataHandleUtils.outPutFile(fileName, sb, charset); FzFtpUtils.ftpUpload(fileName) //生成加密数据文件并上传 FzFtpUtils.uploadSha512Data2(fileName, dataCodeFileName, charset); } + def selectLastResultDatas(def rets){ + def rows = [] + for(def r : rets){ + rows << [r[0],r[1], r[2], r[3], r[4],r[5], r[6]] + } + return rows; + } + def appendResult(def headers, def ret){ @@ -147,6 +157,29 @@ class DistrictPriceExportBatch extends BaseBatch{ m << 0 } m << curDate.toInteger(); + if(curPrice && dbo){ + if(dbo.value_type == 1){ + m << "指导价" + }else if(dbo.value_type == 2){ + m << "自身样本" + }else if(dbo.value_type == 3){ + m << "第三方" + }else if(dbo.value_type == 4){ + m << "交易数据" + }else if(dbo.value_type == 5){ + m << "周边楼盘" + }else if(dbo.value_type == 6){ + m << "推导" + }else if(dbo.value_type == 9){ + m << "神经网络" + }else if(dbo.value_type == 10){ + m << "无方式" + }else{ + m << "" + } + }else{ + m << "无" + } result << m } return result @@ -300,6 +333,7 @@ class DistrictPriceExportBatch extends BaseBatch{ //按comm_id修改,小区名重新赋值,防止区域和小区名修改 udata.put("region", r[3]) udata.put("name", r[4]) + udata.put("value_type", r[7]) FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata)) }else{ def o = new BasicDBObject() @@ -312,6 +346,7 @@ class DistrictPriceExportBatch extends BaseBatch{ o.put("avg_price", r[5].toInteger()) o.put("month", r[6].toInteger()) o.put("c_date", new Date()) + o.put("value_type", r[7]) FZ_DISTRICT_STAT_COLL.save(o) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 459e3e505..6f339f416 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -199,6 +199,8 @@ class DistrictSellStatBatch extends BaseBatch { data.put("mom", mom) data.put("yoy", yoy) data.put("avg_price", weekendPriceObj.avg_price) + data.put("value_type", weekendPriceObj.value_type) + data.put("credit", weekendPriceObj.credit) DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false) println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name } @@ -210,6 +212,7 @@ class DistrictSellStatBatch extends BaseBatch { data.put("estate_type", dist.estate_type2) data.put("date", curMonth) data.put("avg_price", curPrice) + data.put("value_type", weekendPriceObj.value_type) data.put("credit", weekendPriceObj.credit) data.put("m_date", new Date()) data.put("commid", dist._id.toString()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 917ddfc5e..b2d9b9338 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -51,59 +51,61 @@ class FzDataResultStat { }) } def priceMarkupRun(){ - def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15"); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); - List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv")) +// List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv")) + def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"]) + def idMaps = [:] + for(def d : dists){ + idMaps.put(d._id.toString(), d.status==0?"有效":"无效") + } + def list1 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"))) def distkeys = [] def map1 = [:] - for(def s : list1){ - def row = s.split("@\\|@") - def key = row[2]+"##"+row[3]+"##"+row[4] + for(def obj : list1){ + def key = obj.city + "##" + obj.region + "##" + obj.name if(!distkeys.contains(key)){ distkeys << key }else{ println "重复:"+key } - map1.put(key, row) + map1.put(key, obj) } - List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv")) +// List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv")) + def list2 = findFzDistrictStat(lastMonth) def map2 = [:] - for(def s : list2) { - def row = s.split("@\\|@") - def key = row[2] + "##" + row[3] + "##" + row[4] - map2.put(key, row) + for(def obj : list2) { + def key = obj.city + "##" + obj.region + "##" + obj.name + map2.put(key, obj) } def rows = [] rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"] - map1.each {k,list-> - def row = [] + def index = 0 + map1.each {k,obj1-> def keys = k.split("##") - def lastObj = map2.get(k) - def lastPrice = null - if(lastObj){ - lastPrice = lastObj[5] - } + def lastPrice = map2.get(k) + def markup = null try{ - if(list && lastPrice && lastPrice.toInteger()>0){ - markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger() + if(obj1 && lastPrice && lastPrice.avg_price.toInteger()>0){ + markup = (obj1.avg_price.toInteger() - lastPrice.avg_price.toInteger())*100.0/lastPrice.avg_price.toInteger() } }catch(Exception e){ e.printStackTrace() } - def dist = CommonDatautils.findDistrictById(list[1]) - def flag = "有效" - - if(dist && dist.status!=0){ - flag = "失效" + def flag = "无效" + if(obj1){ + flag = idMaps.get(obj1.comm_id) } - - rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag] + def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag] + rows << row + index++ + println row } ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据") @@ -255,4 +257,10 @@ class FzDataResultStat { } return files; } + + def findFzDistrictStat(lastMonth){ + def q = new BasicDBObject() + q.put("month", lastMonth) + return FZ_DISTRICT_STAT_COLL.find(q).toList() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 2220e87ab..8e3147530 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202306.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202307.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy index 5cae7c343..a7b5d4df1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy @@ -14,6 +14,8 @@ import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBObject import com.mongodb.QueryOperators +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders class CityStatExport { @@ -26,10 +28,21 @@ class CityStatExport { static main(args) { def task = new CityStatExport(); -// task.run() + task.run() // task.run1() - task.regionStat() +// task.regionStat() // task.stat() +// task.houseStat() + } + + def houseStat(){ + def fileName = "D:\\数据\\贝壳\\贝壳城市小区二手房统计V3-已重新补充.xlsx" + ExcelUtils.multiThreadingModify(fileName, { d -> +// def houseCount = findHouseCount(d['F'], 20230201) +// d['H'] = houseCount + d['J'] = findHouse(d['F']) + }) + } def stat(){ @@ -47,7 +60,7 @@ class CityStatExport { // def citys = CommonDatautils.findAllCity(); - def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"] + def citys = [ "广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"] def ret = [] ret << ["城市","小区总量"] // ret << ["城市","楼栋总量", "房号总量"] @@ -78,7 +91,7 @@ class CityStatExport { ret << m println m } - ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"数据") + ExcelUtils.write("D:\\数据\\广东省城市统计(楼盘量).xlsx",ret,"数据") } @@ -124,7 +137,7 @@ class CityStatExport { def findHouseCount(def city, def startDate){ def total = 0 try{ - String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} " + String sql = "select count(*) as c from seawater where cat='sell' and city='${city}' and p_date >= ${startDate} " ElasticUtils.executeSqlQuery(sql,{ data-> total = data.c @@ -136,6 +149,18 @@ class CityStatExport { return total } + + def findHouse(city) { + int startDate = 20230201 + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell")) + .must(date_range) + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + return houses.size() + } + def findLoudongCount(String city) { return LOUDONG.count(new BasicDBObject('city', city)) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 78270dc6c..1a8fb81be 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -90,4 +90,53 @@ class FileUtils { } return ret; } + + + static def batchCreateFileBySplitChar(ret, fileName, headers,charset, isAppend, splitChar){ + File file = new File(fileName); +// if (file.exists()) { +// file.delete(); +// } + + if (!file.getParentFile().exists()) { + boolean result = file.getParentFile().mkdirs(); + if (!result) { + System.out.println("创建目录失败"); + } + } + + + OutputStream out = new FileOutputStream(fileName, isAppend); + def sb = new StringBuffer(); + + if(headers){ + sb.append(headers.join(splitChar)) + sb.append("\n"); + } + + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + + def index = 0 + for(def row : ret){ + if(row){ + index++ + sb.append(StringUtils.join(row.toArray(), splitChar)) + sb.append("\n"); + + if(index%10000==0){ + println index+"开始写入CSV "+new Date().toLocaleString() + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + println index+"结束写入CSV "+new Date().toLocaleString() + } + } + } + + if(sb.length()>0){ + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + } + out.close(); + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index a3236b53a..210fa41c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -172,4 +172,32 @@ class FangjiaClientUtils { } } + + public static def addrEvaluation(city,region,address, size, floor, maxFloor, room, hall){ + def jsonObj = null + def content = null + try { + address = address.replace("{","") + def url = "http://localhost:8080/fangjia-open/property/addressEvaluate?city="+URLEncoder.encode(city,"UTF-8")+"&district="+URLEncoder.encode(region,"UTF-8")+"&address="+URLEncoder.encode(address,"UTF-8")+"&size=${size}&sampleCode=1" + if(floor){ + url = url + "&floor="+floor.toString().toDouble().toInteger() + } + if(maxFloor){ + url = url + "&maxFloor="+maxFloor.toString().toDouble().toInteger() + } + if(room){ + url = url + "&room="+room.toString().toDouble().toInteger() + } + if(hall){ + url = url + "&hall="+hall.toString().toDouble().toInteger() + } + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } catch (Exception e) { + println address +" "+content + e.printStackTrace(); + } + return jsonObj; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy index 0af10b4fa..f17e815dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.district +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -69,14 +70,15 @@ class OfficeFieldCleanBatch extends BaseBatch{ // strandField(offices) // } + + // addBlock() - // addFieldValue() - // updateErrorField() - export() - test() +// export() +// test() // addOffice() + fileAddBlock() } def export(){ @@ -706,7 +708,34 @@ class OfficeFieldCleanBatch extends BaseBatch{ }) } + def fileAddBlock(){ + def rets = findAllBlockRange() + ExcelUtils.multiThreadingModify("D:\\数据\\其它\\上海商铺房源.xlsx",{d-> + def location = d['T'] + if(location && !d['F']){ + def arrs = location.split(",") + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3) + if(baidu){ + def lng2 = baidu.x + def lat2 = baidu.y + def lastObj = null + for (def r : rets) { + def flag = InsidePolygon.isInArea(lng2, lat2, r.points) + if (flag && r.name) { + lastObj = r; + break; + } + } + if (lastObj) { + d['F'] = lastObj.name + } + } + + } + }) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/ExportHouse.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/ExportHouse.groovy new file mode 100644 index 000000000..8dd989f3c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/ExportHouse.groovy @@ -0,0 +1,72 @@ +package com.fangjia.script.gj.es + +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +def datas = findHouseCount("北京", 20230201, 20230430) +ExcelUtils.write("D:\\数据\\天弘基金.xlsx", datas, "样例") + + + +def findHouseCount(def city, def startDate, def endDate){ + def rets = [] + try{ +// String sql = "select * from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and root='ke.com' order by p_date desc" + def startMdate = DateUtils.parseFromTightByPattern(startDate.toString(),"yyyyMMdd") + def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd") + + rets << ["城市","区域","板块","名称","面积","单价","总价","室","厅","卫","装修","朝向","发布时间","物业类型","楼层状况","楼高","建筑结构","建筑年代","标题","URL"] + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + // QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false); + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell")).must(QueryBuilders.termQuery("root", "ke.com")) + .must(date_range).must(area_range)//.must(date_range_mdate) + + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(6); + + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + + if(houses) { + + for (int k = 0; k < houses.size(); k++) { + println k + def data = houses[k]; + def r = [] + r << data.city + r << data.region + r << data.block + r << data.district_name + r << data.area + r << data.avg_price + r << data.total + r << data.room + r << data.hall + r << data.toilet + r << data.fitment + r << data.direction + r << data.p_date + r << data.estate_type + r << data.floor_info + r << data.height + r << data.build_structure + r << data.build_date + r << data.name + r << data.url + rets << r + if(rets.size()>500){ + break; + } + } + } + }catch(Exception e){ + e.printStackTrace() + } + + return rets +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 88b01f6c1..2003db863 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome { def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") - + WebDriver driver = null; //广州,佛山 重庆 static main(args) { @@ -50,7 +50,7 @@ public class BeiKeCrawlByChrome { System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); - WebDriver driver = new ChromeDriver() + driver = new ChromeDriver() def citys = getAllCityUrls(driver) driver = getLogin(driver, "https://sh.ke.com/") @@ -93,19 +93,28 @@ public class BeiKeCrawlByChrome { } def checkDocument(driver, url){ + driver.get(url) def s = driver.getPageSource() def flag = true while(flag){ - if(s && (s.contains("无法访问此网站") || s.contains("登录"))){ + try{ + if(s && (s.contains("无法访问此网站") || s.contains("登录"))){ + driver = getLogin(driver, url) + driver.get(url) + Thread.sleep(5000) + s = driver.getPageSource() + }else{ + flag = false + } + }catch(Exception e){ + driver.close() + driver = new ChromeDriver() + Thread.sleep(15000) driver = getLogin(driver, url) - driver.get(url) - Thread.sleep(5000) - s = driver.getPageSource() - }else{ - flag = false + println "异常Url:"+url } } - return Jsoup.parse(s); + return driver; } def crawlData(driver, citys){ @@ -130,7 +139,8 @@ public class BeiKeCrawlByChrome { def newUrl = url+"/chengjiao/" driver.get(newUrl) Thread.sleep(10000) - def doc = checkDocument(driver, newUrl); + driver = checkDocument(driver, newUrl); + def doc = Jsoup.parse(driver.getPageSource()) def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) if(regions){ for(def r : regions){ @@ -142,8 +152,9 @@ public class BeiKeCrawlByChrome { regionText = "昆山" } - def detailUrls = getPageDeail(driver, newCity, href, regionText) - println url +" 详情链接:"+detailUrls.size() + def map = getPageDeail(driver, newCity, href, regionText) + driver = map.driver + println url +" 详情链接:"+map.detailUrls.size() } } FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\贝壳已抓城市.txt", [], "utf-8", true); @@ -191,9 +202,9 @@ public class BeiKeCrawlByChrome { def detailUrls = [] while(!stop){ def url = cityUrl + "pg${page}/" - driver.get(url) // def doc = Jsoup.parse(driver.getPageSource()); - def doc = checkDocument(driver, url); + driver = checkDocument(driver, url); + def doc = Jsoup.parse(driver.getPageSource()) def elements = JQueryUtils.select(".listContent li", doc) if(elements){ for(def e : elements){ @@ -203,10 +214,11 @@ public class BeiKeCrawlByChrome { def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl + println "开始查库 "+new Date().toString() def dbo = findTrade(durl , "贝壳找房") - + println "结束查库"+new Date().toString() if (!dbo) { - parse(city, e, region, driver, null) + driver = parse(city, e, region, driver, null) } // else if(dbo && !dbo.lift){ // parse(city, e, region, driver, dbo._id) @@ -225,7 +237,10 @@ public class BeiKeCrawlByChrome { stop = true; } } - return detailUrls + def m = [:] + m.driver = driver + m.detailUrls = detailUrls + return m } def parse(city,e, region, driver, id){ @@ -233,8 +248,10 @@ public class BeiKeCrawlByChrome { def detailUrl = e.select(".title a").attr("href") driver.get(detailUrl) - def detailDoc = checkDocument(driver, detailUrl); + driver = checkDocument(driver, detailUrl); + def detailDoc = Jsoup.parse(driver.getPageSource()) + println "打开获取页面 "+new Date().toString() def title = e.select(".title a").text() if (title) { def detail = new BasicDBObject() @@ -335,18 +352,21 @@ public class BeiKeCrawlByChrome { detail.create_date = new Date() detail.m_date = new Date() + println "解析页面完成 "+new Date().toString() println detail if(id){ DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail)) }else{ DEAL_PRICE_COLL.save(detail) } + println "保存修改完成 "+new Date().toString() } } }catch(Exception ex){ ex.printStackTrace() println e.select(".title a").attr("href") } + return driver } @@ -469,16 +489,16 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = "https:"+e.attr("href") -// if (citys.contains(e.text())){ + if (citys.contains(e.text())){ list << [e.text(), href] -// } + } } } return list } def getLogin(driver, url){ -// WebDriver driver = new ChromeDriver() + def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4) driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F"); WebDriverWait wait = new WebDriverWait(driver,20) @@ -494,17 +514,25 @@ public class BeiKeCrawlByChrome { WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 + elemUsername.sendKeys("17148854005"); + elemPassword.sendKeys("12qwaszx"); + // elemUsername.sendKeys("15021384995"); // elemPassword.sendKeys("12qwaszx"); - elemUsername.sendKeys("16283166371"); - elemPassword.sendKeys("Goojia123456"); +// elemUsername.sendKeys("16283166371"); +// elemPassword.sendKeys("Goojia123456"); btn.submit(); // 点击行为; Thread.sleep(10000) }catch(Exception e){ + driver.close() + driver = new ChromeDriver() Thread.sleep(15000) + driver = getLogin(driver, url) + println "异常Url:"+url } + return driver } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 52e1665f4..e43dddbb0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -34,6 +34,7 @@ public class LianjiaCrawlByChrome { def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + WebDriver driver = null; static main(args) { def task = new LianjiaCrawlByChrome(); @@ -122,7 +123,7 @@ public class LianjiaCrawlByChrome { System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); - WebDriver driver = new ChromeDriver() + driver = new ChromeDriver() def cityMap = getAllCityUrls(driver) // cityMap = cityMap.reverse() driver = getLogin(driver, "https://sh.lianjia.com/") @@ -206,17 +207,11 @@ public class LianjiaCrawlByChrome { } def cityUrl = ret[1] def newUrl = cityUrl+"chengjiao/" - try{ - driver.get(newUrl) - }catch(Exception e){ - println newUrl +" 链接有问题!" - continue - } - +// driver.get(newUrl) //def detailUrls = getPageUrls(driver, cityUrl) - Thread.sleep(10000) - def doc = checkDocument(driver, newUrl); - +// Thread.sleep(10000) + driver = checkDocument(driver, newUrl); + def doc = Jsoup.parse(driver.getPageSource()) def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) if(regions){ for(def r : regions){ @@ -226,8 +221,9 @@ public class LianjiaCrawlByChrome { regionText = "昆山" } println "开始抓取【${city}】【${regionText}】【${href}】" - def detailUrls = getPageDeail(driver, newCity, href, regionText) - println cityUrl +" 详情链接:"+detailUrls.size() + def map = getPageDeail(driver, newCity, href, regionText) + driver = map.driver + println newUrl +" 详情链接:"+map.detailUrls.size() } } // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) @@ -267,30 +263,41 @@ public class LianjiaCrawlByChrome { } def checkDocument(driver, url){ + driver.get(url) def s = driver.getPageSource() def flag = true while(flag){ - if(s && (s.contains("无法访问此网站") || s.contains("登录"))){ - driver = getLogin(driver, url) - driver.get(url) - Thread.sleep(5000) - s = driver.getPageSource() - }else{ - flag = false + try{ + if(s && (s.contains("无法访问此网站") || s.contains("登录"))){ + driver = getLogin(driver, url) + driver.get(url) + Thread.sleep(5000) + s = driver.getPageSource() + }else{ + flag = false + } + }catch(Exception e){ + driver.close() + driver = new ChromeDriver() + Thread.sleep(15000) + driver = getLogin(driver, url) + println "异常Url:"+url } } - return Jsoup.parse(s); + return driver; } + def getPageDeail(driver, city, cityUrl, regionText){ def stop = false def page = 1 def detailUrls = [] while(!stop){ def url = cityUrl + "pg${page}/" - driver.get(url) +// driver.get(url) //def detailUrls = getPageUrls(driver, cityUrl) - def doc = checkDocument(driver, url); + driver = checkDocument(driver, url); + def doc = Jsoup.parse(driver.getPageSource()) def elements = JQueryUtils.select(".listContent li", doc) if(elements){ for(def e : elements){ @@ -303,7 +310,7 @@ public class LianjiaCrawlByChrome { detailUrls << durl def dbo = findTrade(durl , "链家在线") if (!dbo) { - parse(city, e, regionText, driver, null) + driver = parse(city, e, regionText, driver, null) } // else if(dbo && !dbo.lift){ // parse(city, e, regionText, driver, dbo._id) @@ -326,7 +333,10 @@ public class LianjiaCrawlByChrome { stop = true; } } - return detailUrls + def m = [:] + m.driver = driver + m.detailUrls = detailUrls + return m } def parse(city,e, regionText, driver, id){ @@ -334,8 +344,11 @@ public class LianjiaCrawlByChrome { def price = e.select(".totalPrice .number").text() if(price){ def detailUrl = e.select(".title a").attr("href") - driver.get(detailUrl) - def detailDoc = checkDocument(driver, detailUrl); + +// driver.get(detailUrl) + driver = checkDocument(driver, detailUrl); + def detailDoc = Jsoup.parse(driver.getPageSource()) + def title = e.select(".title a").text() if (title) { def detail = new BasicDBObject() @@ -450,6 +463,7 @@ public class LianjiaCrawlByChrome { ex.printStackTrace() println e.select(".title a").attr("href") } + return driver; } @@ -571,7 +585,7 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia")){ + if (!href.contains("fang.lianjia") && citys.contains(e.text())){ cityList << [e.text(), href] } } @@ -593,16 +607,23 @@ public class LianjiaCrawlByChrome { WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 -// elemUsername.sendKeys("15021384995"); -// elemPassword.sendKeys("12qwaszx"); + elemUsername.sendKeys("17148854005"); + elemPassword.sendKeys("12qwaszx"); - elemUsername.sendKeys("16283166371"); - elemPassword.sendKeys("Goojia123456"); +// elemUsername.sendKeys("17740818975"); +// elemPassword.sendKeys("13qeadzc"); + +// elemUsername.sendKeys("16283166371"); +// elemPassword.sendKeys("Goojia123456"); btn.submit(); // 点击行为; - Thread.sleep(15000) + Thread.sleep(20000) }catch(Exception e){ - Thread.sleep(10000) + driver.close() + driver = new ChromeDriver() + Thread.sleep(15000) + driver = getLogin(driver, url) + println "异常Url:"+url } return driver } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy new file mode 100644 index 000000000..e313fe216 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -0,0 +1,211 @@ +package com.fangjia.script.gj.xiamen + +import com.fangjia.core.RegionData +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import org.bson.types.ObjectId +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.support.WriteRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 厦门银行 + * 福建省,重庆市poi数据处理 + */ +@BatchOwner(Watcher.GaoJuan) +class GaoDePoiDataProcess extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + static main(args) { + runBatch(new GaoDePoiDataProcess(), args) + } + + @Override + void process(Object args) { + //三明市,宁德市,泉州市,厦门市,福州市,漳州市,南平市,龙岩市,莆田市,重庆市 + def citys = ["三明市","漳州市","厦门市","福州市"] + for(def city : citys){ + def results = CommonDatautils.findAmapDetailBySql(city) + println results.size() + def rows = running(results) + save(rows) + println city + " 完成"+new Date().toString() + } + } + + def running(results){ + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(results.size()) + + for(def i=0;i=2){ + map.secondary_type = arrs[1] + } + if(arrs.size()>=3){ + map.small_type = arrs[2] + } + } + + //高德转百度 + def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon) + if(baiduMap){ + def location = [:] + location.lon = baiduMap.lng + location.lat = baiduMap.lat + map.bd_location = location + } + + //高德转WGS84 + def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon) + if(gpsLoactionMap){ + def gpslocation = [:] + gpslocation.lon = gpsLoactionMap.lng + gpslocation.lat = gpsLoactionMap.lat + map.gps_location = gpslocation + } + + def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat) + if(gaodeMap && gaodeMap.township){ + map.township = gaodeMap.township + } + + if(!fjw_region){ + if(gaodeMap && gaodeMap.district){ + fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district) + } + } + if(!fjw_region){ + println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap + } + map.fjw_region = fjw_region + return map + } + + def save(rows){ + BulkRequest request = new BulkRequest(); + int batchSize = 1000; + int processedCount = 0; + def client = ElasticUtils.getClient("192.168.0.12") + for (int i = 0; i < rows.size(); i++) { + def map = rows[i] + request.add(new IndexRequest("amap_poi", "_doc", map.id).source(map, XContentType.JSON)); + if(i%1000==0){ + print "." + } + + // 批量提交数据 + if (request.numberOfActions() >= batchSize) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + print "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + + request = new BulkRequest(); + } + } + + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + } + + println "结束时间 "+new Date() + } + + + def getTownshipInfo(def glocation){ + def township = null + def district = null + def jsonObject = null + def stop = false + def count = 0 + while(!stop){ + if(glocation){ + jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + district = addressComponent.district + } + } + } + if(district || count>10){ + stop = true + }else{ + count ++ + Thread.sleep(3000) + print "." + } + } + + def map = [:] + map.district = district + map.township = township + return map; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiAndSeaweedExportCSV.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiAndSeaweedExportCSV.groovy new file mode 100644 index 000000000..d691850c2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiAndSeaweedExportCSV.groovy @@ -0,0 +1,35 @@ +package com.fangjia.script.gj.xiamen + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils + +@BatchOwner(Watcher.GaoJuan) +class PoiAndSeaweedExportCSV extends BaseBatch{ + + static main(args) { + runBatch(new PoiAndSeaweedExportCSV(), args); + } + + @Override + public void process(Object args) { + def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"] + + poiExport(citys) + + } + + def poiExport(citys){ + def fileName = "/home/goojia/xiamen/BIGDATA_POI.csv" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\厦门\\BIGDATA_POI.csv" + } + + for(def c : citys){ + def rets = CommonDatautils.findNewAmapDetail(c) + FileUtils.batchCreateFileBySplitChar(rets, fileName, null, "utf-8", true, "@|@"); + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ZhuoXin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ZhuoXin.groovy new file mode 100644 index 000000000..8817fefd3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ZhuoXin.groovy @@ -0,0 +1,44 @@ +package com.fangjia.script.wenyan.temp + +import com.fangjia.core.Constants +import com.fangjia.core.util.DateUtil +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.client.MongoCollection +import net.sf.json.JSONObject +import org.bson.Document + +import static com.mongodb.client.model.Filters.* + +MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") +def startDate = DateUtil.parseDate("20230701", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230801", "yyyyMMdd") +Calendar calendar = Calendar.getInstance() +calendar.setTime(startDate) +def count = 0 +def hashSet = [] +def data = [["编号", "公司名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "返回参数"]] +while (startDate.before(endDate)) { + calendar.add(Calendar.DATE, 1) + def nextDate = calendar.getTime() + count++ + ///property/addressEvaluate /address/financialRating /v2/property/detail + OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/v2/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> + def result = JSONObject.fromObject(doc.res_map.result) +// if (!hashSet.contains(result)) { + def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] + data << item +// hashSet << result +// } else { +// println("已存在") +// } + } + if (count % 7 == 0) { + hashSet.clear() + } + + startDate = nextDate +} +ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信.xlsx", data, "楼盘评情") + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index 4d301536b..ab5c90233 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -12,7 +12,7 @@ import org.jsoup.Jsoup import java.util.zip.GZIPInputStream class GaoDeApiUtils { - def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" + def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" /** * 经纬度转地址
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/ZheShangBank.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/ZheShangBank.groovy new file mode 100644 index 000000000..b3af27092 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/ZheShangBank.groovy @@ -0,0 +1,83 @@ +package com.fangjia.script.zsbank + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils + +class ZheShangBank { + static main(args) { + def task = new ZheShangBank() +// task.run() +// task.detail() + task.sample() + } + + def sample(){ + def runedCitys = FileUtils.readTxt("D:\\数据\\浙商\\result.txt") + def rets = [] + rets << ["时间","Msg", "城市", "区域", "名称","月份"] + def map = [:] + for(def line : runedCitys){ + def ret = [] + def date = line.substring(line.indexOf("log.")+4, line.indexOf(":")) + def msg = line.substring(line.lastIndexOf("error :")+7, line.size()) + def arrs = msg.split(" ") + ret << date + for(def i=0;i + println m+" "+r + } +// ExcelUtils.write("D:\\数据\\浙商\\浙商失败案例.xlsx", rets, "风险缓释") + } + def run(){ + def start = "2021-03-01" + def rets = [] + rets << ["月份","总调用量","成功量","失败量"] + while (true){ + def end = DateUtils.addMonth(DateUtils.parseFromTightYYYYMMDD(start), 1) +// def total = CommonDatautils.findLogs("1007569", "/address/risk", start, end, null) +// def successCount = CommonDatautils.findLogs("1007569", "/address/risk", start, end, 200) + + def total = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, null) + def successCount = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, 200) + + def failCount = total - successCount + println start[0..6] +" "+ total +" "+successCount +" "+ failCount + rets << [start[0..6], total, successCount, failCount] + start = end; + if(end == "2023-09-01"){ + break; + } + } + ExcelUtils.write("D:\\数据\\浙商\\浙商调用统计.xlsx", rets, "估值地址") + } + + + def detail(){ + def datas = CommonDatautils.findLogs2("1007569", "/property/addressEvaluate") + ExcelUtils.write("D:\\数据\\浙商\\浙商失败案例.xlsx", datas, "估值地址") + } +} From faafc4375c7ce83556cbd858541322352c2d2159 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 17 Aug 2023 14:54:10 +0800 Subject: [PATCH 04/13] update --- .../groovy/com/fangjia/script/commData/CommonDatautils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 2b6c315b3..8ee79c56c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -3973,7 +3973,7 @@ class CommonDatautils { } static def findAmapDetailBySql(city){ - String sql = "select * from amap_poi_detail where cityname='${city}' limit 10" + String sql = "select * from amap_poi_detail where cityname='${city}'" def rets = [] ElasticUtils.executeSqlQuery(sql,{data-> // if(data.name.contains('华山小区G栋')){ From 5fc4d9fd4e0e33f806a41a68b7b2c7ead87fad02 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 17 Aug 2023 14:59:49 +0800 Subject: [PATCH 05/13] =?UTF-8?q?=E7=BA=BF=E7=A8=8B20-=E3=80=8B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index e313fe216..4a7adf2b1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -29,7 +29,7 @@ import java.util.concurrent.atomic.AtomicLong @BatchOwner(Watcher.GaoJuan) class GaoDePoiDataProcess extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5) static main(args) { runBatch(new GaoDePoiDataProcess(), args) @@ -43,8 +43,7 @@ class GaoDePoiDataProcess extends BaseBatch{ def results = CommonDatautils.findAmapDetailBySql(city) println results.size() def rows = running(results) - save(rows) - println city + " 完成"+new Date().toString() + save(city, rows) } } @@ -132,7 +131,7 @@ class GaoDePoiDataProcess extends BaseBatch{ return map } - def save(rows){ + def save(city,rows){ BulkRequest request = new BulkRequest(); int batchSize = 1000; int processedCount = 0; @@ -172,7 +171,7 @@ class GaoDePoiDataProcess extends BaseBatch{ } } - println "结束时间 "+new Date() + println "[${city}] 结束时间 "+new Date() } From 19e4d20ed44d6a91a256e5b2efedb01602df8e4f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 17 Aug 2023 18:10:07 +0800 Subject: [PATCH 06/13] update --- .../script/commData/CommonDatautils.groovy | 67 ++++++++ .../script/gj/xiamen/SeaweedDataSynch.groovy | 146 ++++++++++++++++++ 2 files changed, 213 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 8ee79c56c..b81154fb8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1,8 +1,10 @@ package com.fangjia.script.commData import com.fangjia.core.RegionData +import com.fangjia.httpclient.UserAgents import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.JacksonUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import groovy.json.JsonSlurper @@ -32,6 +34,7 @@ import org.elasticsearch.index.query.QueryBuilders import java.util.regex.Matcher import java.util.regex.Pattern +import java.util.zip.GZIPInputStream class CommonDatautils { @@ -252,6 +255,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); +// q.put("name","华润佘山九里二期") DBObject keys = new BasicDBObject(); if(fields){ @@ -4061,5 +4065,68 @@ class CommonDatautils { return results } + static def httpSaveData(def newUrl,def params){ + def header = [:] + header.put("Authorization","Basic ZWxhc3RpYzpnb29qaWExMjM0NTY=") + doPost(newUrl,header, params, true, "utf-8") + } + + + static def doPost(def targetUrl, def headers, def params, def isJson, def charset){ + StringBuilder buffer = new StringBuilder(); + try{ + URL url = new URL(targetUrl); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setUseCaches(false); + conn.setConnectTimeout(5000); + conn.setReadTimeout(5000); + conn.setRequestProperty("User-Agent", UserAgents.get()); + String param = null; + BufferedWriter mWriter = null; + InputStreamReader inStream = null; + + + if (headers != null && headers.keySet().size() > 0) { + for (String head : headers.keySet()) { + conn.setRequestProperty(head, headers.get(head)); + } + } else { + conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8" ); + } + + if (isJson) { + conn.setRequestProperty("Content-Type", "application/json; charset=" + charset); + param = JacksonUtil.beanToJson(params) + } + + conn.connect(); + + //println param.toString() + //输出流 + if (StringUtils.isNotEmpty(param)) { + mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset)); + mWriter.write(param); + mWriter.flush(); + mWriter.close(); + } + if (conn.getHeaderField("Content-Encoding") != null + && conn.getHeaderField("Content-Encoding").equals("gzip")) { + inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset); + } else { + inStream = new InputStreamReader(conn.getInputStream(), charset); + } + char[] bf = new char[1024]; + int len; + while ((len = inStream.read(bf)) != -1) { + buffer.append(bf, 0, len); + } + } catch (Exception e) { + e.printStackTrace() + } + return buffer.toString(); + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy new file mode 100644 index 000000000..8efdcd037 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -0,0 +1,146 @@ +package com.fangjia.script.gj.xiamen + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtil +import com.fangjia.script.util.DateUtils + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class SeaweedDataSynch { + + static def targetUrl = "http://192.168.0.12:9200/seaweed/" + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + static main(args) { + def task = new SeaweedDataSynch() + task.run() + } + + def run(){ + def citys = CommonDatautils.findAllCity() + def index = 0 + for(def c : citys){ + index++ + updating(c, "district") + updating(c, "office") + updating(c, "shop") + println index +" "+ c + " 完成" + } + } + + def updating(city, cat){ + def dists = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","address","block","alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area"]) + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i Date: Fri, 18 Aug 2023 13:46:42 +0800 Subject: [PATCH 07/13] update --- .../fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 4a7adf2b1..081873448 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -29,7 +29,7 @@ import java.util.concurrent.atomic.AtomicLong @BatchOwner(Watcher.GaoJuan) class GaoDePoiDataProcess extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) static main(args) { runBatch(new GaoDePoiDataProcess(), args) @@ -37,8 +37,9 @@ class GaoDePoiDataProcess extends BaseBatch{ @Override void process(Object args) { - //三明市,宁德市,泉州市,厦门市,福州市,漳州市,南平市,龙岩市,莆田市,重庆市 - def citys = ["三明市","漳州市","厦门市","福州市"] + //"三明市","漳州市","厦门市","福州市" + //三明市,"宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市" + def citys = ["福州市","重庆市"] for(def city : citys){ def results = CommonDatautils.findAmapDetailBySql(city) println results.size() @@ -128,6 +129,7 @@ class GaoDePoiDataProcess extends BaseBatch{ println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap } map.fjw_region = fjw_region + map.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ") return map } From a02526369a7811653b212099f2ec07240d3c1e1b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 26 Aug 2023 13:34:26 +0800 Subject: [PATCH 08/13] update --- .../java/com/fangjia/core/RegionData.java | 89 ++++++++- .../script/ccbevaluation/AddSzDealData.groovy | 2 +- .../BigdataCommunityBatch.groovy | 108 +++++------ .../DealAndThirdPriceDataProcess.groovy | 4 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../script/ccbevaluation/TradeStatLog.groovy | 10 +- .../script/commData/CommonDatautils.groovy | 4 +- .../fangjia/script/gj/YouDaDataParser.groovy | 4 +- .../gj/{ => city}/CityStatExport.groovy | 24 ++- .../gj/location/DistrictLocationBatch.groovy | 4 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 4 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 4 +- .../script/gj/township/StreetProcess.groovy | 179 ++++++++++++++++++ .../gj/xiamen/GaoDePoiDataProcess.groovy | 172 ++++++++++++++++- .../script/gj/xiamen/SeaweedDataSynch.groovy | 46 ++++- .../script/looplineinfo/InsidePolygon.groovy | 21 ++ 16 files changed, 580 insertions(+), 97 deletions(-) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/{ => city}/CityStatExport.groovy (91%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/township/StreetProcess.groovy diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/RegionData.java b/projects/fangjia-core/src/main/java/com/fangjia/core/RegionData.java index d8b75d0c5..4add48eb1 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/RegionData.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/RegionData.java @@ -25,6 +25,7 @@ public class RegionData implements PriorityOrdered { private final static Logger LOGGER = LoggerFactory.getLogger(RegionData.class); private final static long REGION_RELOAD_INTERVAL = 1000; private static DBCollection REGION_CONN = null; + private static DBCollection TOWN_CONN = null; private static Set ALL_CITY = new HashSet(); private static Map> cityAndAliasMap = new HashMap>(); @@ -32,12 +33,15 @@ public class RegionData implements PriorityOrdered { * 城市对应级别Map */ private static Map cityGradeMap = new HashMap(); + + protected final static Map>>>> townDataMap = new HashMap>>>>(); private static void init() { try { REGION_CONN = MongoUtils.getCollectionByName("fangjia", "region_block", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017); - DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017); + TOWN_CONN = MongoUtils.getCollectionByName("fangjia", "township", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017); + DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017); DBCursor cursor = SERVER_AREA.find(new BasicDBObject(), new BasicDBObject("display", 1).append("city_grade", 1).append("alias", 1)); BasicDBObject dbo; while(cursor.hasNext()){ @@ -58,6 +62,7 @@ public class RegionData implements PriorityOrdered { start(REGION_RELOAD_INTERVAL); reload(); +// reloadTown(); } private static void start(long time) { @@ -114,6 +119,66 @@ public class RegionData implements PriorityOrdered { LOGGER.info("RegionData reload() size [" + regionDataMap.size() + "]"); } + public static void reloadTown(){ + DBObject query = new BasicDBObject(); + query.put("year", 2022); + + DBObject fields = new BasicDBObject(); + fields.put("fangjia_city", 1); + fields.put("fangjia_area", 1); + fields.put("street_name", 1); + fields.put("alias", 1); + + + DBCursor cursor = TOWN_CONN.find(query, fields); + while (cursor.hasNext()) { + DBObject dbo = cursor.next(); + String city = (String) dbo.get("fangjia_city"); + String region = (String) dbo.get("fangjia_area"); + String street = (String) dbo.get("street_name"); + List ps = (List) dbo.get("alias"); + if(!ALL_CITY.contains(city)){ + continue; + } + Map>>> cityRegionMap = townDataMap.get(city); + if(cityRegionMap == null){ + cityRegionMap = new HashMap>>>(); + townDataMap.put(city, cityRegionMap); + } + + Map>> regoinMap = cityRegionMap.get(region); + if(regoinMap == null){ + regoinMap = new HashMap>>(); + cityRegionMap.put(region, regoinMap); + } + + Map> townMap = regoinMap.get("street"); + if(townMap == null){ + townMap = new HashMap>(); + regoinMap.put(street, townMap); + } + if (CollectionUtils.isNotEmpty(ps) && ps.size() > 1) { + Collections.sort(ps, new Comparator() { + public int compare(String arg0, String arg1) { + if (StringUtils.isBlank(arg0)) { + return 1; + } + if (StringUtils.isBlank(arg1)) { + return -1; + } + return arg1.length() - arg0.length(); + } + }); + } + if(null != ps && !ps.contains(street)){ + ps.add(street); + } + townMap.put(street, ps); + + } + + LOGGER.info("TownData reload() size [" + townDataMap.size() + "]"); + } static class ScheduleRunTask extends TimerTask { private int nextScheduleHour; @@ -174,6 +239,28 @@ public class RegionData implements PriorityOrdered { return ret; } + + public static String formatStreet(String city, String region, String street) { + if (StringUtils.isBlank(region) || StringUtils.isBlank(city) || StringUtils.isBlank(street)) { + return null; + } + + String ret = null; + Map>>> cityRegionMap = townDataMap.get(city); + if(MapUtils.isNotEmpty(cityRegionMap)){ + Map>> streetMap = cityRegionMap.get(region); + if (MapUtils.isNotEmpty(streetMap)) { + for (Map> crm : streetMap.values()) { + if (null != ret) { + break; + } + ret = format(crm, street); + } + } + } + return ret; + } + public static String parseRegionFormAddress(String city, String address){ if(StringUtils.isBlank(address)){ return null; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 7ddc3a095..1453028a4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳6月份二手房成交数据汇总.xls", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳7月份二手房成交数据汇总.xls", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 49d4de707..7e64a374b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -40,61 +40,61 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.getRunningCitys(args) - def rows = [] - - def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", - //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", - //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"] - - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def index = 0 - def today = new Date().getDate() - - println "starting 。。。。。。。。。" - for(def city : citys){ - - index ++ - - - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) - - - def ccbMaps = null - if(today<=10){ - ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 - }else{ - ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 - } - - def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) - rets = addDeleteDist(dists, ccbMaps, rets) - rows.addAll(rets) - println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size() - -// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); - } - - def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat" - } - - def allDatas = [] - allDatas << headers - allDatas.addAll(rows) - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); - println "txt 生成->导Oracle" - - if(today > 10){ - delPgData() - saveToPgSql(rows) - } +// def citys = CcbCommonUtils.getRunningCitys(args) +// def rows = [] +// +// def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", +// //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", +// //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"] +// +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// def index = 0 +// def today = new Date().getDate() +// +// println "starting 。。。。。。。。。" +// for(def city : citys){ +// +// index ++ +// +// +// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) +// def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) +// +// +// def ccbMaps = null +// if(today<=10){ +// ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 +// }else{ +// ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 +// } +// +// def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) +// rets = addDeleteDist(dists, ccbMaps, rets) +// rows.addAll(rets) +// println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size() +// +//// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); +// } +// +// def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat" +// } +// +// def allDatas = [] +// allDatas << headers +// allDatas.addAll(rows) +// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); +// println "txt 生成->导Oracle" +// +// if(today > 10){ +// delPgData() +// saveToPgSql(rows) +// } // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") -// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat") -// delPgData() -// saveToPgSql2(datas) + def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat") + delPgData() + saveToPgSql2(datas) } def addDeleteDist(dists, ccbMaps, rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 87e5fa8d9..c2b5ce129 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -392,12 +392,12 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ def citys = CcbCommonUtils.citys_all - def month = 202306 + def month = 202307 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() for(def city : citys) { - def rets = findPeGteThirdParty2(city, 202307) + def rets = findPeGteThirdParty2(city, 202308) def total = 0 println city +" "+rets.size() if(rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 99dc7bf92..67ea43dda 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-06-26", "2023-07-26") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-07-26", "2023-08-25") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 71420fef7..4f338e082 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -93,13 +93,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["城市","23年4月总量","23年5月总量","23年6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","23年5月总量","23年6月总量","23年7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2023-04-01" - def date = "2023-05-01" - def endDate = "2023-06-01" - def newDate = "2023-07-01" + def startDate = "2023-05-01" + def date = "2023-06-01" + def endDate = "2023-07-01" + def newDate = "2023-08-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index b81154fb8..c59cdcd87 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -3977,7 +3977,7 @@ class CommonDatautils { } static def findAmapDetailBySql(city){ - String sql = "select * from amap_poi_detail where cityname='${city}'" + String sql = "select * from amap_poi_detail where cityname='${city}' and status=0" def rets = [] ElasticUtils.executeSqlQuery(sql,{data-> // if(data.name.contains('华山小区G栋')){ @@ -3993,7 +3993,7 @@ class CommonDatautils { def location = [:] location.lon = point.x location.lat = point.y - rets << ["province":data.pname,"city":data.cityname, "region":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area] + rets << ["province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area] // } }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index ac0ef538c..f06dd1590 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2023-06-01" + static def statDate = "2023-07-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2023年7月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2023年8月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy similarity index 91% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy index a7b5d4df1..4781397c2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy @@ -1,16 +1,14 @@ -package com.fangjia.script.gj +package com.fangjia.script.gj.city import com.fangjia.core.RegionData import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils -import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject -import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBObject import com.mongodb.QueryOperators @@ -28,11 +26,25 @@ class CityStatExport { static main(args) { def task = new CityStatExport(); - task.run() +// task.run() // task.run1() // task.regionStat() // task.stat() // task.houseStat() + task.addFieldStat() + } + + def addFieldStat(){ + ExcelUtils.multiThreadingModify("D:\\数据\\楼盘量.xlsx", { d -> + def cityInfoDto = CommonDatautils.findCityInfo(d['B']) + if(cityInfoDto){ + def detailMap = cityInfoDto.district_detail + d['D'] = detailMap.address.toDouble()*100 + d['E'] = 100 + d['F'] = cityInfoDto.price_coverage.toDouble()*100 + } + }) + } def houseStat(){ @@ -60,7 +72,7 @@ class CityStatExport { // def citys = CommonDatautils.findAllCity(); - def citys = [ "广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"] + def citys = [ "贵阳","六盘水","遵义","安顺","铜仁","毕节","黔西南","黔东南","黔南"] def ret = [] ret << ["城市","小区总量"] // ret << ["城市","楼栋总量", "房号总量"] @@ -91,7 +103,7 @@ class CityStatExport { ret << m println m } - ExcelUtils.write("D:\\数据\\广东省城市统计(楼盘量).xlsx",ret,"数据") + ExcelUtils.write("D:\\数据\\楼盘量.xlsx",ret,"数据3") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy index c92e387fe..3c74690da 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy @@ -25,8 +25,8 @@ class DistrictLocationBatch { static main(args) { def task = new DistrictLocationBatch() -// task.run() - task.deleteDist() + task.run() +// task.deleteDist() } def deleteDist(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 2003db863..8634731d6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -489,9 +489,9 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = "https:"+e.attr("href") - if (citys.contains(e.text())){ +// if (citys.contains(e.text())){ list << [e.text(), href] - } +// } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index e43dddbb0..bcfef9581 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -65,7 +65,7 @@ public class LianjiaCrawlByChrome { } def run1(){ - System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() @@ -581,7 +581,7 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityList = [] - def citys= ["福州"] + def citys= ["衢州"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/township/StreetProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/township/StreetProcess.groovy new file mode 100644 index 000000000..abb5bedc4 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/township/StreetProcess.groovy @@ -0,0 +1,179 @@ +package com.fangjia.script.gj.township + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.ProvinceData +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class StreetProcess { + + final DBCollection TOWNSHIP = MongoUtils.getCollectionByName("fangjia", "township", "192.168.0.43") + + public static final def MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + public static final DBCollection TOWNSHIP100 = MongoUtils.getCollectionByName2("fangjia", "township", MONGODB_URL) + + + final DBCollection POLYLINE235 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline") + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + + static main(args) { + def task = new StreetProcess(); +// println RegionData.formatRegion("鞍山","铁东区") + task.run() + } + def run(){ + + def region = RegionData.formatRegion("三明市", "三元区") + println region + +// standard() +// matchStreet() +// synchTownship() + } + + def synchTownship(){ + def q = new BasicDBObject() +// q.put("city_name","福州市") + q.put("shape",new BasicDBObject('$exists', true)) + def dbcursor = TOWNSHIP.find(q) + for(def d : dbcursor){ + def udata = new BasicDBObject() + udata.put("shape", d.shape) + TOWNSHIP100.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata)) + } + } + + def matchStreet(){ + def polylineMap = findStreetPolyline() + def q = new BasicDBObject() +// q.put("fangjia_city", "福州") +// q.put("shape",new BasicDBObject('$exists', false)) + def dbcursor = TOWNSHIP.find(q) + def index = 0 + for(def d : dbcursor){ + index++ + def streetMap = polylineMap.get(d.fangjia_city+"##"+d.fangjia_area) + if(streetMap){ + def alias = d.alias + if(!alias){ + alias = [] + } + if(!alias.contains(d.street_name)){ + alias << d.street_name + } + for(def streetName : alias){ + def polyline = streetMap.get(streetName) + if(polyline){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + TOWNSHIP.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata)) + println index + " "+ d.get("_id") + break; + } + } + } + } + } + + def findStreetPolyline(){ + def q = new BasicDBObject() +// q.put("fjw_city","福州") + def dbcursor = POLYLINE235.find(q) + + def map = [:] + for(def d : dbcursor){ + def key = d.fjw_city+"##"+d.fjw_region + def streetMap = map.get(key) + if(!streetMap){ + streetMap = [:] + map.put(key, streetMap) + } + streetMap.put(d.town, d.polyline) + } + return map; + } + + + + + def standard(){ + def q = new BasicDBObject() + q.put("fjw_city",new BasicDBObject('$exists', false)) + def dbcursor = POLYLINE235.find(q) + def index = 0 + for(def d : dbcursor){ + index ++ + def city = RegionData.formatCity(d.city) + def region = RegionData.formatRegion(d.city, d.region) +// def street = null +// if(city && region){ +// street = RegionData.formatStreet(city, region, d.town) +// } + def udata = new BasicDBObject() + udata.put("fjw_city", city) + udata.put("fjw_region", region) +// if(street){ +// udata.put("fjw_street", street) +// } + POLYLINE235.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + println index + } + } + + def standard1(){ + def list = TOWNSHIP.find(new BasicDBObject()).toList() + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(list.size()) + for(def i=0;i + if(k!="_id"){ + udata.put(k,v) + } + } + SEAWEED_CONN.update(new BasicDBObject('_id', map._id), new BasicDBObject('$set', udata), true, false) + } + def createData(dist){ def map = [:] - map.id = dist._id.toString() + map._id = dist._id map.city = dist.city map.region = dist.region map.name = dist.name @@ -79,8 +105,8 @@ class SeaweedDataSynch { map.township = dist.township if(dist.lng){ def loc = [:] - loc.lon = dist.lng/10000000000.0 - loc.lat = dist.lat/10000000000.0 + loc.lon = DataHandleUtils.getDoubleRound(8, dist.lng/10000000000.0) + loc.lat = DataHandleUtils.getDoubleRound(8, dist.lat/10000000000.0) map.location = loc } @@ -88,8 +114,8 @@ class SeaweedDataSynch { def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon) if(baiduMap){ def location = [:] - location.lon = baiduMap.lng - location.lat = baiduMap.lat + location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng) + location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat) map.bd_location = location } @@ -97,8 +123,8 @@ class SeaweedDataSynch { def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon) if(gpsLoactionMap){ def gpslocation = [:] - gpslocation.lon = gpsLoactionMap.lng - gpslocation.lat = gpsLoactionMap.lat + gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng) + gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat) map.gps_location = gpslocation } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy index 03f3e2b73..ac85c450c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy @@ -272,4 +272,25 @@ class InsidePolygon { } return flag; } + + public static def isInArea2(lng, lat, pointMap){ + def flag = false; + Set keys = pointMap.keySet(); + def name = null + for(String key :keys){ + def prints = pointMap.get(key) + for(def pointArr : prints){ + flag = InsidePolygon.isPtInPoly(lng, lat, pointArr) + if(flag){ + def arrs = key.split("##") + name = arrs[1]; + break; + } + } + if(flag){ + break; + } + } + return name; + } } From c11bf5ce0e811171765051cb84b81b3598158922 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 26 Aug 2023 13:45:53 +0800 Subject: [PATCH 09/13] update --- .../com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 9bf1926ea..a9511c985 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -67,7 +67,8 @@ class GaoDePoiDataProcess extends BaseBatch{ def townMap = findTownRange(c) def companyDatas = findCompanyDatas(c) println c+ " 加载公司数据:"+companyDatas.size() - running(companyDatas, "company", townMap) + def rows = running(companyDatas, "company", townMap) + save(c, rows) println "完成" } } From f11f846024e5af843fcbdf89da007c5845cef26f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 27 Aug 2023 11:42:51 +0800 Subject: [PATCH 10/13] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=92=A4=E5=9B=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pgsql/ExportPgsqlEvaluationResult.groovy | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy index 5f5bcb8ac..e38b64b94 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy @@ -34,19 +34,19 @@ class ExportPgsqlEvaluationResult { def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); //计算结果相关 -// exportAvmEvaluationGJNB(curMonth) -// exportAVMEvaluationItemGJNB(curMonth) -// -// exportCityPrice(curMonth) -// exportRegionPrice(curMonth) -// -// exportCommunityBasePrice(curMonth) + exportAvmEvaluationGJNB(curMonth) + exportAVMEvaluationItemGJNB(curMonth) + + exportCityPrice(curMonth) + exportRegionPrice(curMonth) + + exportCommunityBasePrice(curMonth) exportCommunityPdescribeGj(curMonth) -// exportCommCaseDetailGj(curMonth) + exportCommCaseDetailGj(curMonth) exportInOk() - +// SSHHelperTest jschCommand = new SSHHelperTest(); System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); From 97b933451d67f3ef90926a435fc6a4e4606d8771 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 28 Aug 2023 11:00:41 +0800 Subject: [PATCH 11/13] =?UTF-8?q?=E5=BB=BA=E8=A1=8C=E4=B8=B4=E6=97=B6?= =?UTF-8?q?=E5=AF=BC=E5=87=BA=E4=B8=80=E4=B8=AA=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pgsql/ExportPgsqlEvaluationResult.groovy | 21 ++- .../script/fuzhou/AmapAddressUpdate.groovy | 10 +- .../gj/xiamen/GaoDePoiDataProcess.groovy | 5 +- .../gj/xiamen/NetworkDistanceBatch.groovy | 124 ++++++++++++++++++ .../script/gj/xiamen/SeaweedDataSynch.groovy | 10 +- 5 files changed, 149 insertions(+), 21 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/NetworkDistanceBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy index 6686d596f..5f5bcb8ac 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy @@ -34,20 +34,19 @@ class ExportPgsqlEvaluationResult { def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); //计算结果相关 - exportAvmEvaluationGJNB(curMonth) - exportAVMEvaluationItemGJNB(curMonth) - - exportCityPrice(curMonth) - exportRegionPrice(curMonth) - - exportCommunityBasePrice(curMonth) -// exportCommunityBasePrice1(curMonth) +// exportAvmEvaluationGJNB(curMonth) +// exportAVMEvaluationItemGJNB(curMonth) +// +// exportCityPrice(curMonth) +// exportRegionPrice(curMonth) +// +// exportCommunityBasePrice(curMonth) exportCommunityPdescribeGj(curMonth) - exportCommCaseDetailGj(curMonth) -// +// exportCommCaseDetailGj(curMonth) + exportInOk() -// + SSHHelperTest jschCommand = new SSHHelperTest(); System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy index cecc0536d..0be2c85d2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy @@ -38,10 +38,10 @@ class AmapAddressUpdate { static main(args) { def task = new AmapAddressUpdate() - def path = "d:\\数据\\小区\\福州小区地址与高德地图V2.xlsx" - def addr = task.formatAddress("福州", "北三环路201号") - println addr -// task.run(path) + def path = "d:\\数据\\小区\\福建省小区地址与高德地图V2.xlsx" +// def addr = task.formatAddress("福州", "北三环路201号") +// println addr + task.run(path) // task.similarMatch(path) // task.addressRun(path) // task.addressSource(path) @@ -52,7 +52,7 @@ class AmapAddressUpdate { // task.updateLocation(path)//更新经纬度 } def run(path){ - def citys = ["福州"]//CommonDatautils.findAllCity() + def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田"]//CommonDatautils.findAllCity() def allrets = [] allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度"] for (def city : citys) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index a9511c985..3ec5214f7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -33,7 +33,7 @@ import java.util.concurrent.atomic.AtomicLong @BatchOwner(Watcher.GaoJuan) class GaoDePoiDataProcess extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) public static def version = new Date().getTime() @@ -62,7 +62,8 @@ class GaoDePoiDataProcess extends BaseBatch{ def synchCompany(){ - def citys = ["三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"] + //"三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市", + def citys = ["重庆市"] for(def c : citys){ def townMap = findTownRange(c) def companyDatas = findCompanyDatas(c) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/NetworkDistanceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/NetworkDistanceBatch.groovy new file mode 100644 index 000000000..bd8531aae --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/NetworkDistanceBatch.groovy @@ -0,0 +1,124 @@ +package com.fangjia.script.gj.xiamen + +import com.fangjia.core.util.GeoUtils +import com.fangjia.datastore.driver.DSCursor +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class NetworkDistanceBatch extends BaseBatch{ + + + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + + final DBCollection NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network", "192.168.0.43") + + static main(args) { + runBatch(new NetworkDistanceBatch(), args) + } + + @Override + void process(Object args) { + + def networks = ExcelUtils.read("D:\\数据\\厦门\\网点.xlsx", 0, 1) + def citys = ["厦门","三明","宁德","泉州","福州","漳州","南平","龙岩","莆田","重庆"] + for(def c : citys){ + run(c, "district", networks) + } + } + + def run(city, cat, networks){ + def datas = findDistrictData(city, cat, ["city","region", "name", "category", "location"]) + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;i Date: Tue, 29 Aug 2023 15:37:55 +0800 Subject: [PATCH 12/13] =?UTF-8?q?=E8=AF=84=E4=BC=B0=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E6=A5=BC=E5=B1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/core/util/DataUtil.java | 7 +++++ .../open/controllers/PropertyController.java | 26 +++++++++++++++++++ .../service/api/AddressMatchProxyService.java | 4 +++ .../api/AddressMatchProxyServiceImpl.java | 24 ++++++++++++++++- 4 files changed, 60 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java index 81420c56c..644c9ec8a 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java @@ -1463,6 +1463,13 @@ public class DataUtil { return false; } + public static boolean isNumber(String s) { + if (s != null && !"".equals(s.trim())) + return s.matches("^[0-9]*$"); + else + return false; + } + public static double roundDouble(double d, int scale) { try { double ret = new BigDecimal(String.valueOf(d)).setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue(); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index e44529895..5f53b5229 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -37,6 +37,8 @@ import com.fangjia.dal.mongo.open.model.EvaluateRatio; import com.fangjia.dal.seawater.dao.SeawaterDao; import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.open.dto.*; +import com.fangjia.open.dto.addressGovernance.StructureDto; +import com.fangjia.open.util.CommonDtoUtils; import com.fangjia.service.district.*; import com.fangjia.service.mongo.fangjia.*; import com.fangjia.service.open.*; @@ -1085,6 +1087,15 @@ public class PropertyController { return mav; } + if(model.getFloor() == null){ + JSONObject structureOb = addressMatchProxyService.addressStructure(district.getCity(), model.getAddress()); + if(structureOb != null){ + model.setBuildingNumber(structureOb.getString("buildingNumber")); + String roomNumber = structureOb.getString("roomNumber"); + model.setFloor(getFloor(roomNumber)); + } + } + if(model.getMaxFloor() == null && model.getFloor() != null &&StringUtils.isNotBlank(model.getBuildingNumber())){ Loudong loudong = loudongService.findForEvaluate(district.getId(), model.getBuildingNumber(), model.getUnitNumber()); if(loudong != null && loudong.getFloors() >= model.getFloor()){ @@ -1108,6 +1119,8 @@ public class PropertyController { } else { model.setCalDate(DateUtil.getFirstDayOfWeek(new Date())); } + + Evaluation evaluation = evaluationService.evaluate(district, buildEvaluation(model, request), model.getCalDate()); Double cityPriceCredit = null; @@ -4057,6 +4070,19 @@ public class PropertyController { return "fangjia.com"; } + private Integer getFloor(String roomNumber){ + if(StringUtils.isNotBlank(roomNumber) && DataUtil.isNumber(roomNumber)){ + if(roomNumber.length() == 3){ + return Integer.valueOf(roomNumber.substring(0,1)); + }else if(roomNumber.length() == 4){ + return Integer.valueOf(roomNumber.substring(0,2)); + }else{ + return null; + } + } + return null; + } + private boolean checkExists(FacetQueryResult houseBoList, NewHousePojo house) { for (NewHousePojo bo : houseBoList) { boolean sameName = bo.getName().equals(house.getName()); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AddressMatchProxyService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AddressMatchProxyService.java index 1fff04070..01e5b41dc 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AddressMatchProxyService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AddressMatchProxyService.java @@ -84,5 +84,9 @@ public interface AddressMatchProxyService { * @return */ MatchDistrictResult checkMatchDistrictCorrect(CheckMatchDistrictCorrectParam param); + + + + JSONObject addressStructure(String city, String address); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AddressMatchProxyServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AddressMatchProxyServiceImpl.java index 578be76e3..d0b7ca0ea 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AddressMatchProxyServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AddressMatchProxyServiceImpl.java @@ -271,7 +271,29 @@ public class AddressMatchProxyServiceImpl extends AbstractHttpApiService impleme } return matchResult; } - + + @Override + public JSONObject addressStructure(String city, String address) { + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure/v2", params, "UTF-8"); + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + if (json.containsKey("result")) { + return (JSONObject) json.get("result"); + } + } + }catch(Exception e){ + LOGGER.error("地址标准化失败", e); + } + return null; + } + /** * 获取高德中分数最高的信息 * @param city From ed2be5b0c6f055e76e042d591778aafb506d4c4e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 30 Aug 2023 17:19:24 +0800 Subject: [PATCH 13/13] =?UTF-8?q?=E5=88=A4=E6=96=AD=E6=98=AF=E5=90=A6?= =?UTF-8?q?=E5=AD=98=E5=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/open/controllers/PropertyController.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 5f53b5229..5943c0a20 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -1090,9 +1090,15 @@ public class PropertyController { if(model.getFloor() == null){ JSONObject structureOb = addressMatchProxyService.addressStructure(district.getCity(), model.getAddress()); if(structureOb != null){ - model.setBuildingNumber(structureOb.getString("buildingNumber")); - String roomNumber = structureOb.getString("roomNumber"); - model.setFloor(getFloor(roomNumber)); + if(structureOb.containsKey("buildingNumber")){ + model.setBuildingNumber(structureOb.getString("buildingNumber")); + } + + if(structureOb.containsKey("roomNumber")){ + String roomNumber = structureOb.getString("roomNumber"); + model.setFloor(getFloor(roomNumber)); + } + } }