diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index ee00efea2..cd0782adc 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -100,11 +100,7 @@ - - com.fangjia.cloud.api.client - FangJiaApiSwaggerClient - 1.0.0-SNAPSHOT - + org.apache.commons commons-lang3 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy index 6f114e516..d4d8007e7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy @@ -61,7 +61,8 @@ class CityInfoAnaBatch extends BaseBatch { all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', car_site :'carSite',estate_charge :'estateCharge',floorplan_pics :'floorplanPics',out_pics :'outPics',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter', start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',licence :'licence',build_structure :'buildStructure', - lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure' + lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure', + alias:'alias',addr_alias:'addrAlias' ] public static final def OFFICE_FIELDS_MAP = [ name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate', diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LandDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LandDetailCrawl.groovy index 1f7d635e9..6a44c0463 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LandDetailCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LandDetailCrawl.groovy @@ -31,7 +31,7 @@ class LandDetailCrawl { def task = new LandDetailCrawl(); // task.run() task.updateData(); - task.undateOldData(); +// task.undateOldData(); } def run(){ @@ -65,7 +65,7 @@ class LandDetailCrawl { } def updateData(){ - def datas = ExcelUtils.read("D:\\数据\\友达\\土地--12月.xlsx",0, 1) + def datas = ExcelUtils.read("D:\\数据\\土地\\土地成交3月(2).xlsx",0, 1) if(datas && datas.size()>0){ for(def data : datas){ def id = data['A'] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 257dfff8b..76ef3729b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -44,7 +44,7 @@ class OpenApiUtils { static def matchDistrict2(def city, def address, def preciseLevelflag){ - address = address.replace("(","(").replace(")",")").replace(" ","") + address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","") def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1" if(preciseLevelflag){ url = url +"&preciseLevel=strict" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy new file mode 100644 index 000000000..17c422651 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy @@ -0,0 +1,224 @@ +package com.fangjia.script.gj + +import com.fangjia.core.Constants +import com.fangjia.core.util.HttpRequest +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.client.MongoCollection +import net.sf.json.JSONObject +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.math.RandomUtils +import org.apache.commons.lang3.StringUtils +import org.bson.Document + +import java.util.regex.Pattern + +/** + * 抓取区域边框 + */ +class RegionPointCrawl { + + static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code") + + static main(args) { + def task = new RegionPointCrawl(); + //河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、 +// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"] + def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"] + for(def p : provinces){ + task.crawlCityAndDistrictCode(p); + task.run(p); + } + + } + + def crawlCityAndDistrictCode(def province){ + crawlCity(province) + } + + def run(city){ + def cityCodes = findCityCode(city); + if(cityCodes){ + for(def cityMap : cityCodes){ + def codeMap = findAllDistrictCode(cityMap.city_code); + def index = 0 + for(def m : codeMap){ + index ++ + def ret = polygon(m.ad_code, "0", "all"); + if(!ret){ + println m.name +" " + m.ad_code + "无结果" + } + println index +" "+cityMap.name+" "+m.name+" "+ret.size() + if(ret.size()>0){ + saveData(cityMap.name, m.name, m.ad_code , ret); + } + } + } + } + } + + def crawlCity(def province){ + def ret = polygon(province, "2", "base"); + if(ret.size()>0){ + for(def map : ret){ + if(map.level == "province"){ + def cityList = map.districts + if(cityList && cityList.size()>0){ + for(def cityMap : cityList){ + if(cityMap.level == "city"){ + saveGaoDeCityRegionCode(province, cityMap.name, cityMap.citycode, cityMap.adcode, cityMap.center, cityMap.level); + saveGaoDeRegionCode(province, cityMap.districts); + } + } + } + } + } + } + } + + def saveGaoDeRegionCode(province, districts){ + if(districts){ + for(def dist : districts){ + if(dist.level=="district"){ + saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level); + } + } + } + } + + def findCityCode(city){ + Pattern pattern = Pattern.compile(city, Pattern.CASE_INSENSITIVE); + def q = null; + if(city.contains("省") || city.endsWith("自治区")){ + q = new BasicDBObject("province", pattern); + }else{ + q = new BasicDBObject("name", pattern); + } + q.put("level", "city") + def dbcursor = GAODECITYCODECOLL.find(q) + def rets = [] + for(def d : dbcursor){ + def city_code = d.get("city_code") + if(!rets.contains(city_code)){ + rets << d + } + } + return rets; + } + + private BasicDBObject getLikeStr(String findStr) { + Pattern pattern = Pattern.compile(findStr + ".*\$", Pattern.MULTILINE); + return new BasicDBObject("$regex", pattern); + } + + def findAllDistrictCode(def citycode){ + def q = new BasicDBObject() + q.put("city_code", citycode) + q.put("level", "district") + def dbcursor = GAODECITYCODECOLL.find(q) + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets; + } + + def saveGaoDeCityRegionCode(province, name, citycode, adcode, center, level){ + def q = new BasicDBObject() + q.put("city_code", citycode) + q.put("province", province) + q.put("level", level) + q.put("name", name) + if(adcode){ + q.put("ad_code", adcode) + } + + def o = GAODECITYCODECOLL.findOne(q) + if(!o){ + q.put("c_date", new Date()) + q.put("center", center) + GAODECITYCODECOLL.save(q); + }else{ + println name +" "+adcode+" 已存在" + } + } + + def saveData(city, region, code, ret){ + def q = new BasicDBObject() + q.put("code", code) + def dbo = POLYLINECOLL.findOne(q) + if(!dbo){ + def o = new BasicDBObject() + o.put("city", city) + o.put("region", region) + o.put("code", code) + o.put("center", ret.center.get(0)) + o.put("level", ret.level.get(0)) + o.put("polyline", ret.polyline.get(0)); + POLYLINECOLL.save(o); + } + } + + + public static def polygon(def keyword, def subdistrict, def extensions) { + String url = "https://restapi.amap.com/v3/config/district"; + + def params = [:] + params.put("keywords", keyword) + params.put("subdistrict",subdistrict) + params.put("extensions",extensions) + + String html = null; + def jo = null; + int errorCount = 0; + String status = "error" + def resultMap = [:] + try { + HttpRequest http = new HttpRequest(); + while (errorCount<5){//"2af0b8533415a612b98d59bb93d664b9" 公司key + def key = getRandomKey2() // + params.put("key", key) + html = http.get(url, params,"utf-8") + if (StringUtils.isNotEmpty(html)) { + jo = JSONObject.fromObject(html) + if(jo.infocode!="10000"){ + errorCount ++ + Thread.sleep(30*1000) +// println " 不可用Key: "+key + }else{ + errorCount = 100 + status = "normal" +// println " 可用Key: "+key + " "+html + } + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + return jo.districts; + } + + static String getRandomKey2() { + def KeyList = [ + "2af0b8533415a612b98d59bb93d664b9", + "0246c57bbcf24aa246c207b3c99dfe41", + "042328b3f0fa5daf426802d379521d87", + "055d4f1b5b64a2bce18f4fe1f15c2639", + "d5484f5170502405d88bbaffe1b496f0", + "e74ccc4258079d2200199695110cceb1", + "f58629fc799fbc8537ca85046c6d44a5", + "291073f17ee0b963ccb1927ed92bf265", + "2a1fb75990f2b39f6221e50892dd3988" + ] + + def count = RandomUtils.nextInt(KeyList.size()) + return KeyList.get(count); + } +} +H \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index e7ef96749..a3a6e7989 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -47,7 +47,7 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2020-11-01" + static def statDate = "2021-02-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2020年11月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2021年3月\\二手房\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy index 84bf1edc9..7dae269d1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -60,9 +60,14 @@ class HouseExport { static main(args) { def task = new HouseExport(); - task.run() +// task.run() +// task.marge() + } + + + def run(){ // "重庆", def citys = ["北京"] @@ -777,4 +782,23 @@ def GenNonDuplicateID(randomLength){ + def marge(){ + File file = new File("D:\\数据\\建行\\新建文本文档.txt"); + def ids = new HashSet(); + def index = 0 + + file.eachLine{it-> + index++ + if(index>1){ + def arrs = it.split("\\|@\\|") + def id = arrs[1] + if(!ids.contains(id)){ + ids << id + } + } + } + println index + println ids.size() + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy new file mode 100644 index 000000000..6131ce01e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy @@ -0,0 +1,40 @@ +package com.fangjia.script.gj.data + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils + +class FieldStat { + + static main(args) { + def task = new FieldStat(); + task.run() + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区融合结果概况2.xlsx", { row -> + def city = row['A'] + if(city){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]); + def total = districts.size() + + def aliasIndex = 0 + def aliasTotalCount = 0 + for(def d : districts){ + def alias = d.alias + if(alias){ + aliasIndex++ + aliasTotalCount = aliasTotalCount + alias.size() + } + } + row['J'] = aliasTotalCount + row['K'] = aliasIndex + def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total) + row['L'] = p+"%" + row['M'] = total + println city +" "+p + } + + }); + println "finish" + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy index 8c5009841..05808e33a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy @@ -23,7 +23,9 @@ class AnjukeCrawlToDB { // task.run() - task.stat() +// task.stat() + + task.addAliasStat() } def stat(){ @@ -77,6 +79,7 @@ class AnjukeCrawlToDB { } } + println estate_chargeCount def estate_charge_pre = DataHandleUtils.getDoubleRound(2, estate_chargeCount*100/total) def build_area_pre = DataHandleUtils.getDoubleRound(2, build_areaCount*100/total) @@ -104,6 +107,36 @@ class AnjukeCrawlToDB { } + def addAliasStat(){ + + ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\城市覆盖率统计(2021-03-14) (1).xls",{row-> + def city = row['B'] + def districts = CommonDatautils.findAllDistricts(city, "district", ["alias", "addr_alias"]); + + def total = districts.size() + + def aliasIndex = 0 + def addrAliasIndex = 0 + for(def d : districts){ + def alias = d.alias + if(alias){ + aliasIndex++ + } + + def addrAlias = d.addr_alias + if(addrAlias){ + addrAliasIndex ++ + } + } + def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total) + row['BA'] = p + + def p1 = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, addrAliasIndex*100/total) + row['BB'] = p1 + + },0) + } + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/JiangShuBlankDataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/JiangShuBlankDataStat.groovy new file mode 100644 index 000000000..cc5c8b7b3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/JiangShuBlankDataStat.groovy @@ -0,0 +1,292 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.QueryOperators +import org.apache.commons.io.FileUtils +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.regex.Matcher +import java.util.regex.Pattern + +class JiangShuBlankDataStat { + + def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new JiangShuBlankDataStat() +// task.run() +// task.matchCredit(); +// task.repeat(); +// task.matchHouse(); + +// task.villa() + task.addressParse() + } + + def repeat(){ + def datas = ExcelUtils.read("D:\\数据\\江苏农行\\江苏农行1484条.xlsx", 0,1) + def keys = [] + for(def data : datas){ + def key = data['W']+"##"+data['X']+"##"+data['Y'] + if(!keys.contains(key)){ + keys << key + } + } + println keys.size(); + } + + def run(){ + def esDatas = findLogs(); + def keys = [] + def datas = [] + for(def d : esDatas){ + def key = d.fjw_city +"##"+d.fjw_region+" "+d.fjw_name + if(!keys.contains(key)){ + keys << key + datas << [d.parameter_city, d.parameter_region, d.parameter_address, d.parameter_size, d.fjw_city, d.fjw_region, d.fjw_name] + }else{ + println key + } + } + + ExcelUtils.write("D:\\数据\\江苏农行\\全数据.xlsx", datas, "数据") + +// ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\江苏农行930条.xlsx",{row-> +//// def address = row['G'] +//// if(!row['X']){ +//// def area = row['H'] +//// def retMap = findRow(address, area, esDatas); +//// if(retMap){ +//// row['V'] = retMap.fjw_city +//// row['W'] = retMap.fjw_region +//// row['X'] = retMap.fjw_name +//// row['Y'] = retMap.fjw_avgPrice +//// row['Z'] = retMap.fjw_totalPrice +//// +//// row['AG'] = retMap.parameter_city +//// row['AH'] = retMap.parameter_region +//// } +// def inputRegion = row['X'] +// def matchRegion = row['M'] +// if( inputRegion&& matchRegion && inputRegion.contains(matchRegion)){ +// row['AA'] = "包含" +// } +//// def matchName = row['Y'] +//// if(matchName && address && address.contains(matchName)){ +//// row['AL'] = "包含名称" +//// } +//// } +// },0) + } + + def matchCredit(){ + ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\全数据.xlsx",{row-> + def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G'], "district") + if(obj){ + row['H'] = obj.estate_type2 + } +// def dbo = CommonDatautils.queryDistrictPrice(row['E'], row['F'], row['G'], DateUtils.parseFromTightByPattern("2021-03-14","yyyy-MM-dd")) +// if(dbo){ +// row['H'] = dbo.avg_price +// row['I'] = dbo.credit +// row['J'] = dbo.value_type +// } +// +// def lianjiaObj = findThirdPrice(row['E'], row['F'], row['G'], "lianjia_esf_price"); +// if(lianjiaObj){ +// row['k'] = lianjiaObj.lianjia_esf_price +// row['L'] = lianjiaObj.s_date +// } +// def beikeObj = findThirdPrice(row['E'], row['F'], row['G'], "beike_esf_price"); +// if(beikeObj){ +// row['M'] = beikeObj.beike_esf_price +// row['N'] = beikeObj.s_date +// } + },0) + } + + def findThirdPrice(def city, def region, def name, def fieldName){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true)) + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1)) + def obj = null; + for(def d : dbcursor){ + obj = d; + break; + } + return obj; + } + + def findRow(address, area, esDatas){ + def retMap = [:] + for(def d : esDatas){ + + if(d.parameter_address ==address && area==d.parameter_size.toDouble()){ + retMap = d; + break; + } + } + return retMap; + } + + def findLogs(){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007753')).must(QueryBuilders.termQuery("request_uri", "/property/addressEvaluate")) + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); + def rets = [] + def infos = [] + if(houses){ + StringBuilder sbu=new StringBuilder(); + for (int k=0;k + def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G'], "district") + def mod = row['D'].toDouble()*0.2 + def startArea = row['D'].toDouble()- mod + def endArea = row['D'].toDouble() +mod + def prices = findHouses(row['E'], row['F'], "sell", startArea, endArea, row['G'], obj.alias); + if(!prices){ + if(row['D'].toDouble()<=100){ + prices = findHouses(row['E'], row['F'], "sell", 30, 100, row['G'], obj.alias); + }else if(row['D'].toDouble()<=200){ + prices = findHouses(row['E'], row['F'], "sell", 100, 200, row['G'], obj.alias); + }else{ + prices = findHouses(row['E'], row['F'], "sell", 200, 800, row['G'], obj.alias); + } + } + def avg_price = null + if(prices){ + avg_price = getAvgPrice(prices); + } + if(avg_price){ + row['Q'] = avg_price.toInteger() + } + + },0); + } + + def getAvgPrice(def prices){ + def avg_price = 0 + if(prices){ + def sum = prices.sum(); + avg_price = sum/prices.size() + } + return avg_price + } + + def findHouses(def city, def region, def cat, def startArea, def endArea, def name, def alias) { + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(20200301).to(20210318).includeLower(true).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(startArea).to(endArea).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if (!alias) { + alias = [] + } + if (!alias.contains(name)) { + alias.add(name) + } + + if (alias && alias.size() > 0) { + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for (def s : alias) { + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + inBuilder.should(QueryBuilders.matchQuery("root", "lianjia.com")); + inBuilder.should(QueryBuilders.matchQuery("root", "ke.com")); + queryBuilder.must(inBuilder) + + def prices = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + if (houses) { + for (int k = 0; k < houses.size(); k++) { + def map = houses[k]; + DBObject seawater = new BasicDBObject(map) + prices << seawater.avg_price + } + } + + println city +" "+region+" "+name+" "+ prices.size() + return prices; + } + + def villa(){ + ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以上.xlsx", { row -> + def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G']+"别墅", "district") + if(obj){ + def dbo = CommonDatautils.queryDistrictPrice(obj.city, obj.region, obj.name, DateUtils.parseFromTightByPattern("2021-03-14","yyyy-MM-dd")) + if(dbo){ + row['R'] = obj.name + row['S'] = dbo.avg_price + } + } + + },0); + } + + def addressParse(){ + ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以上.xlsx", { row -> + def resp = OpenApiUtils.matchDistrict2(row['E'], row['C'], true) + if(resp){ + println resp + row['T'] = resp.roomNumber + row['U'] = getFloor(resp.roomNumber); + } + + },0); + } + + def getFloor(def s){ + def floor = null + if(s){ + Pattern pa=Pattern.compile("^(\\d{1,2})0\\d{1}") + Matcher ma=pa.matcher(s) + if(ma.find()){ + floor = ma.group(1) + } else { + floor = null + } + } + return floor; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZBlankDataExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZBlankDataExport.groovy index 9d46a2d08..20d3b1dff 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZBlankDataExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZBlankDataExport.groovy @@ -630,128 +630,133 @@ class CZBlankDataExport { sqlBuffer.append(" and m_date >='"+startDate+"'").append(" and m_date <'"+endDate+"'"); // sqlBuffer.append(" and m_date >='"+pDate+"'").append(" and m_date <='"+pDate+"'"); sqlBuffer.append(" ORDER BY p_date desc limit 200") - ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + + ResultSet rs = null def index = 0; def houses = []; - while (rs.next()) { + try{ + rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) - def area = rs.getObject("area") + while (rs.next()) { - - - if(area && (area>1000||area<30)){ - continue; - } - - def floor = rs.getObject("floor") - if(floor && floor>100){ - continue; - } - - def height = rs.getObject("height") - if(height && (height>100||height==0)){ - continue; - } - - def total_price = rs.getObject("total_price") - def house_area = area; - def avg_price = total_price/house_area; - - def distPrice = distPriceMaps.get(dbo.city +"##"+dbo.region+"##"+dbo.name); - if(distPrice){ - def maxValue = distPrice + distPrice*0.15; - def minValue = distPrice - distPrice*0.15; - if(avg_price>maxValue.toInteger() || avg_price1000||area<30)){ continue; } - } - def district_name = rs.getObject("district_name") - if(district_name.endsWith("别墅") && height && height>=10){ - continue; - } + def floor = rs.getObject("floor") + if(floor && floor>100){ + continue; + } - def m = [:]; - //小区名称 - m.name = dbo.name; - if(dbo.alias){ - def newAlias = []; - for(def tname : dbo.alias){ - if(tname && tname.length()>=2){ - newAlias << tname; + def height = rs.getObject("height") + if(height && (height>100||height==0)){ + continue; + } + + def total_price = rs.getObject("total_price") + def house_area = area; + def avg_price = total_price/house_area; + + def distPrice = distPriceMaps.get(dbo.city +"##"+dbo.region+"##"+dbo.name); + if(distPrice){ + def maxValue = distPrice + distPrice*0.15; + def minValue = distPrice - distPrice*0.15; + if(avg_price>maxValue.toInteger() || avg_price=10){ + continue; + } + + def m = [:]; + //小区名称 + m.name = dbo.name; + if(dbo.alias){ + def newAlias = []; + for(def tname : dbo.alias){ + if(tname && tname.length()>=2){ + newAlias << tname; + } + } + if(newAlias){ + m.all_name = dbo.name + "("+newAlias.join(",")+")" + }else{ + m.all_name = dbo.name; + } }else{ m.all_name = dbo.name; } - }else{ - m.all_name = dbo.name; + m.city = rs.getObject("city"); + m.region = rs.getObject("region"); + m.district_id= dbo.num_id; + + m.house_id = createHouseId(rs.getObject("url"));//rs.getObject("_id").toString(); + //建筑物总楼层 + m.height = height>0?height:null; + //所在楼层 + m.floor = getHouseFloor(floor, m.height); + //建筑面积 + m.house_area = area; + //挂牌价 + m.total_price = total_price; + + def keys = m.region +"-"+ m.name+"-"+m.house_area+"-"+m.height+"-"+m.floor+"-"+m.total_price; + if(!houses.contains(keys)){ + houses << keys; + }else{ + println "此房源已存在....." + continue; + } + + if(index > 30){ + break; + } + m.room = rs.getObject("room"); + m.hall = rs.getObject("hall"); + m.estate_type = rs.getObject("estate_type"); + //建成年代 + m.build_date = getBuildDate(rs.getObject("build_date"), dbo); + //总使用年限 + if(districtIds.contains(m.district_id)){ + m.property_life = 70; + }else{ + m.property_life = getPropertyLife(dbo.property_life); + } + //朝向 + m.direction = getDirectionCode(rs.getObject("direction"), dbo.estate_type2); + //建筑结构 + m.build_structure_code = getBuildStructureCode(dbo.build_date, dbo.build_structure); + //建筑类别 + m.build_type_code = getBuildTypeCode(dbo.name, dbo.build_type, height); + if(m.build_type_code=='3' && m.height>6){ + m.height = null + } + + //内部装修与成新度 + m.fitment_code = getFitmentCode(rs.getObject("fitment")); + //是否有电梯 + m.elevator_code = getElevatorCode(height, m.build_type_code); + //房价网评估价 + //m.evaluation_price = getEvaluationPrice(m); + //println "[${index++}] "+new Date(); + //评估时间 + m.evaluation_date = DateUtils.formatDateByPattern(new Date(),"yyyyMMdd"); + //挂牌时间 + m.publish_date = rs.getObject("p_date").toString()//DateUtils.formatDateByPattern(obj.publish_date,"yyyyMMdd"); + + m.status=1; + + index++; + saveData(m); } - m.city = rs.getObject("city"); - m.region = rs.getObject("region"); - m.district_id= dbo.num_id; + }catch(Exception e){ - m.house_id = createHouseId(rs.getObject("url"));//rs.getObject("_id").toString(); - //建筑物总楼层 - m.height = height>0?height:null; - //所在楼层 - m.floor = getHouseFloor(floor, m.height); - //建筑面积 - m.house_area = area; - //挂牌价 - m.total_price = total_price; - - def keys = m.region +"-"+ m.name+"-"+m.house_area+"-"+m.height+"-"+m.floor+"-"+m.total_price; - if(!houses.contains(keys)){ - houses << keys; - }else{ - println "此房源已存在....." - continue; - } - - if(index > 30){ - break; - } - m.room = rs.getObject("room"); - m.hall = rs.getObject("hall"); - m.estate_type = rs.getObject("estate_type"); - //建成年代 - m.build_date = getBuildDate(rs.getObject("build_date"), dbo); - //总使用年限 - if(districtIds.contains(m.district_id)){ - m.property_life = 70; - }else{ - m.property_life = getPropertyLife(dbo.property_life); - } - //朝向 - m.direction = getDirectionCode(rs.getObject("direction"), dbo.estate_type2); - //建筑结构 - m.build_structure_code = getBuildStructureCode(dbo.build_date, dbo.build_structure); - //建筑类别 - m.build_type_code = getBuildTypeCode(dbo.name, dbo.build_type, height); - if(m.build_type_code=='3' && m.height>6){ - m.height = null - } - - //内部装修与成新度 - m.fitment_code = getFitmentCode(rs.getObject("fitment")); - //是否有电梯 - m.elevator_code = getElevatorCode(height, m.build_type_code); - //房价网评估价 - //m.evaluation_price = getEvaluationPrice(m); - //println "[${index++}] "+new Date(); - //评估时间 - m.evaluation_date = DateUtils.formatDateByPattern(new Date(),"yyyyMMdd"); - //挂牌时间 - m.publish_date = rs.getObject("p_date").toString()//DateUtils.formatDateByPattern(obj.publish_date,"yyyyMMdd"); - - m.status=1; - - index++; - saveData(m); } + return index; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 8eed9d7e9..cc38a3d50 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"] for(def source :sources){ - def count1 = findTotalCount(source, 202010); - def count2 = findTotalCount(source, 202011); + def count1 = findTotalCount(source, 202101); + def count2 = findTotalCount(source, 202102); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/BasicInfoDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/BasicInfoDataSynch.groovy index cbe84d9f7..69a998c48 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/BasicInfoDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/BasicInfoDataSynch.groovy @@ -4,11 +4,14 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject - +/** + * 娴欏晢鏈湴鍖栭儴缃插悓姝ヨ剼鏈� + */ @BatchOwner(Watcher.GaoJuan) class BasicInfoDataSynch extends BaseBatch{ // final static DBCollection SERVER_AREA_CONN = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo) @@ -29,10 +32,10 @@ class BasicInfoDataSynch extends BaseBatch{ @Override public void process(Object args) { -// synchCitys(); -// synchRegion() -// synchTown() -// synchDict() + synchCitys(); + synchRegion() + synchTown() + synchDict() synchDistrict("district") synchDistrict("office") } @@ -47,46 +50,49 @@ class BasicInfoDataSynch extends BaseBatch{ zsCitysMap.put(zsCitys._id.toString(), zsCitys) zsDatas << zsCitys } - //数据存在则修改,不存在则保存 + //鏁版嵁瀛樺湪鍒欎慨鏀癸紝涓嶅瓨鍦ㄥ垯淇濆瓨 if(fangjiaCitys){ for(def fjCity : fangjiaCitys){ - if(fjCity.display!="全国"){ + if(fjCity.display!="鍏ㄥ浗"){ fjCitysMap.put(fjCity._id.toString(), fjCity) } def zsCity = zsCitysMap.get(fjCity._id.toString()) - if(zsCity){//修改 + if(zsCity){//淇敼 if(zsCity.province != fjCity.province || zsCity.alias!=fjCity.alias){ def udata = new BasicDBObject() udata.put("province",fjCity.province) udata.put("alias",fjCity.alias) udata.put("status",0) + udata.put("update_date",new Date().getTime()) ZS_CITYS_CONN.update(new BasicDBObject("_id",zsCity._id), new BasicDBObject('$set',udata)) } - }else{//保存 - def addUata = new BasicDBObject() + }else{//淇濆瓨 + def addUata = new BasicDBObject() addUata.put("_id",fjCity._id) addUata.put("display",fjCity.display) addUata.put("province",fjCity.province) addUata.put("alias",fjCity.alias) addUata.put("status",0) + addUata.put("update_date",new Date().getTime()) ZS_CITYS_CONN.save(addUata); } } } - //老数据不在新数据中,被删除时,状态修改为1 + //鑰佹暟鎹笉鍦ㄦ柊鏁版嵁涓紝琚垹闄ゆ椂锛岀姸鎬佷慨鏀逛负1 if(zsDatas){ for(def zsCity : zsDatas){ def fjCity = fjCitysMap.get(zsCity._id.toString()) if(!fjCity){ def udata = new BasicDBObject() udata.put("status",1) + udata.put("update_date",new Date().getTime()) ZS_CITYS_CONN.update(new BasicDBObject("_id",zsCity._id), new BasicDBObject('$set',udata)) } } } - println "[${fangjiaCitys.size()}]个城市同步完成" + println "[${fangjiaCitys.size()}]涓煄甯傚悓姝ュ畬鎴�" } def synchRegion(){ @@ -110,6 +116,7 @@ class BasicInfoDataSynch extends BaseBatch{ udata.put("pattern",fjRegion.pattern) udata.put("org_name",fjRegion.org_name) udata.put("status",0) + udata.put("update_date",new Date().getTime()) if(zsRegionObj){ ZS_REGION_CONN.update(new BasicDBObject("_id",zsRegionObj._id), new BasicDBObject('$set',udata)) }else{ @@ -125,11 +132,12 @@ class BasicInfoDataSynch extends BaseBatch{ if(!fjRegion){ def udata = new BasicDBObject() udata.put("status",1) + udata.put("update_date",new Date().getTime()) ZS_REGION_CONN.update(new BasicDBObject("_id",zsRegion._id), new BasicDBObject('$set',udata)) } } } - println "[${fjRegions.size()}]个行政区域同步完成" + println "[${fjRegions.size()}]涓鏀垮尯鍩熷悓姝ュ畬鎴�" } def synchTown(){ @@ -152,6 +160,7 @@ class BasicInfoDataSynch extends BaseBatch{ udata.put("short_name",fjTown.short_name) udata.put("name",fjTown.name) udata.put("status",0) + udata.put("update_date",new Date().getTime()) if(zsTownObj){ if(fjTown.city!=zsTownObj.city||fjTown.region!=zsTownObj.region||fjTown.short_name!=zsTownObj.short_name||fjTown.name!=zsTownObj.name||zsTownObj.status!=0){ ZS_TOWN_CONN.update(new BasicDBObject("_id",zsTownObj._id), new BasicDBObject('$set',udata)) @@ -169,11 +178,12 @@ class BasicInfoDataSynch extends BaseBatch{ if(!fjTown){ def udata = new BasicDBObject() udata.put("status",1) + udata.put("update_date",new Date().getTime()) ZS_TOWN_CONN.update(new BasicDBObject("_id",zsTown._id), new BasicDBObject('$set',udata)) } } } - println "[${fjTowns.size()}]个乡镇同步完成" + println "[${fjTowns.size()}]涓埂闀囧悓姝ュ畬鎴�" } def synchDict(){ def fjDicts = CommonDatautils.getAllDictionary() @@ -194,6 +204,7 @@ class BasicInfoDataSynch extends BaseBatch{ udata.put("cat",fjDict.cat) udata.put("name",fjDict.name) udata.put("status",0) + udata.put("update_date",new Date().getTime()) if(zsDictObj){ if(fjDict.cat!=zsDictObj.cat||fjDict.name!=zsDictObj.name||zsDictObj.status!=0){ ZS_DICT_CONN.update(new BasicDBObject("_id",zsDictObj._id), new BasicDBObject('$set',udata)) @@ -220,11 +231,12 @@ class BasicInfoDataSynch extends BaseBatch{ if(!fjDict){ def udata = new BasicDBObject() udata.put("status",1) + udata.put("update_date",new Date().getTime()) ZS_DICT_CONN.update(new BasicDBObject("_id",zsDict._id), new BasicDBObject('$set',udata)) } } } - println "[${fjDicts.size()}]个字典数据同步完成" + println "[${fjDicts.size()}]涓瓧鍏告暟鎹悓姝ュ畬鎴�" } def synchDistrict(cat){ def fangjiaCitys = CommonDatautils.findAllCity() @@ -259,18 +271,25 @@ class BasicInfoDataSynch extends BaseBatch{ udata.put("township",fjDistrict.township) udata.put("alias",fjDistrict.alias) udata.put("cat",cat) + udata.put("update_date",new Date().getTime()) if(fjDistrict.addr_alias){ udata.put("addr_alias",fjDistrict.addr_alias) } udata.put("status",0) - def map = new BasicDBObject() - map.put("lat",fjDistrict.lat/10000000000.0d) - map.put("lon", fjDistrict.lng/10000000000.0d) + def map = null; + if(fjDistrict && fjDistrict.lat){ + map = new BasicDBObject() + map.put("lat",fjDistrict.lat/10000000000.0d) + map.put("lon", fjDistrict.lng/10000000000.0d) + } + if(zsDistrictObj){ if(fjDistrict.city!=zsDistrictObj.city||fjDistrict.region!=zsDistrictObj.region||fjDistrict.block!=zsDistrictObj.block||fjDistrict.name!=zsDistrictObj.name||fjDistrict.address!=zsDistrictObj.address||fjDistrict.estate_type2!=zsDistrictObj.estate_type||fjDistrict.township!=zsDistrictObj.township||fjDistrict.alias!=zsDistrictObj.alias||fjDistrict.addr_alias!=zsDistrictObj.addr_alias||map!=zsDistrictObj.location||zsDistrictObj.status!=0){ - udata.put("location", map) - ZS_SEAWEED_CONN.update(new BasicDBObject("_id",zsDistrictObj._id), new BasicDBObject('$set',udata)) + if(map){ + udata.put("location", map) + ZS_SEAWEED_CONN.update(new BasicDBObject("_id",zsDistrictObj._id), new BasicDBObject('$set',udata)) + } } }else{ udata.put("location", map) @@ -295,10 +314,11 @@ class BasicInfoDataSynch extends BaseBatch{ if(!fjDistrict){ def udata = new BasicDBObject() udata.put("status",1) + udata.put("update_date",new Date().getTime()) ZS_SEAWEED_CONN.update(new BasicDBObject("_id",fjDistrict._id), new BasicDBObject('$set',udata)) } } } - println "[${fjCity}-${cat}][${districts.size()}]个小区数据同步完成" + println "[${fjCity}-${cat}][${districts.size()}]涓皬鍖烘暟鎹悓姝ュ畬鎴�" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/DistrictLevelBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/DistrictLevelBatch.groovy index e6576a569..6c1344eb9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/DistrictLevelBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/DistrictLevelBatch.groovy @@ -9,6 +9,9 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +/** + * 娴欏晢鏈湴鍖栭儴缃插悓姝ヨ剼鏈� + */ @BatchOwner(Watcher.GaoJuan) class DistrictLevelBatch extends BaseBatch{ @@ -28,7 +31,7 @@ class DistrictLevelBatch extends BaseBatch{ def datas = addPriceData(dists, priceMap); datas = calcuteLevel(datas); saveOrUpdateDB(city, datas); - println city +" 楼盘评级同步完成" + println city +" 妤肩洏璇勭骇鍚屾瀹屾垚" } } @@ -109,6 +112,7 @@ class DistrictLevelBatch extends BaseBatch{ udata.put("level", d.level) udata.put("avg_price", d.avg_price) udata.put("status",0) + udata.put("update_date",new Date().getTime()) if(zsDistLevel){ if(zsDistLevel.city!=d.city||zsDistLevel.region!=d.region||zsDistLevel.name!=d.name||zsDistLevel.level!=d.level||zsDistLevel.avg_price!=d.avg_price||zsDistLevel.status!=0){ ZS_LEVEL_CONN.update(new BasicDBObject("_id",zsDistLevel._id), new BasicDBObject('$set',udata)) @@ -127,10 +131,11 @@ class DistrictLevelBatch extends BaseBatch{ } if(zsDistLevels.size()>0){ for(def zs : zsDistLevels){ - def newData = newDataMap.get(zs._id.toString()); + def newData = newDataMap.get(zs._id.toString()); if(!newData){ def udata = new BasicDBObject() udata.put("status",1) + udata.put("update_date",new Date().getTime()) ZS_LEVEL_CONN.update(new BasicDBObject("_id",zs._id), new BasicDBObject('$set',udata)) } }