diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy index 6f114e516..d4d8007e7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy @@ -61,7 +61,8 @@ class CityInfoAnaBatch extends BaseBatch { all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', car_site :'carSite',estate_charge :'estateCharge',floorplan_pics :'floorplanPics',out_pics :'outPics',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter', start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',licence :'licence',build_structure :'buildStructure', - lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure' + lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure', + alias:'alias',addr_alias:'addrAlias' ] public static final def OFFICE_FIELDS_MAP = [ name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate', diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 257dfff8b..76ef3729b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -44,7 +44,7 @@ class OpenApiUtils { static def matchDistrict2(def city, def address, def preciseLevelflag){ - address = address.replace("(","(").replace(")",")").replace(" ","") + address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","") def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1" if(preciseLevelflag){ url = url +"&preciseLevel=strict" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy index 631de988e..17c422651 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy @@ -20,13 +20,13 @@ import java.util.regex.Pattern /** * 抓取区域边框 */ -class RegionPoint { +class RegionPointCrawl { static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code") static main(args) { - def task = new RegionPoint(); + def task = new RegionPointCrawl(); //河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、 // def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"] def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index e7ef96749..a3a6e7989 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -47,7 +47,7 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2020-11-01" + static def statDate = "2021-02-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2020年11月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2021年3月\\二手房\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy index 9d191a6d4..5ed93cde1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -61,9 +61,13 @@ class HouseExport { def task = new HouseExport(); // task.run() - task.marge() +// task.marge() + } + + + def run(){ // "重庆", def citys = ["北京"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy index 8c5009841..05808e33a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy @@ -23,7 +23,9 @@ class AnjukeCrawlToDB { // task.run() - task.stat() +// task.stat() + + task.addAliasStat() } def stat(){ @@ -77,6 +79,7 @@ class AnjukeCrawlToDB { } } + println estate_chargeCount def estate_charge_pre = DataHandleUtils.getDoubleRound(2, estate_chargeCount*100/total) def build_area_pre = DataHandleUtils.getDoubleRound(2, build_areaCount*100/total) @@ -104,6 +107,36 @@ class AnjukeCrawlToDB { } + def addAliasStat(){ + + ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\城市覆盖率统计(2021-03-14) (1).xls",{row-> + def city = row['B'] + def districts = CommonDatautils.findAllDistricts(city, "district", ["alias", "addr_alias"]); + + def total = districts.size() + + def aliasIndex = 0 + def addrAliasIndex = 0 + for(def d : districts){ + def alias = d.alias + if(alias){ + aliasIndex++ + } + + def addrAlias = d.addr_alias + if(addrAlias){ + addrAliasIndex ++ + } + } + def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total) + row['BA'] = p + + def p1 = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, addrAliasIndex*100/total) + row['BB'] = p1 + + },0) + } + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/JiangShuBlankDataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/JiangShuBlankDataStat.groovy new file mode 100644 index 000000000..cc5c8b7b3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/JiangShuBlankDataStat.groovy @@ -0,0 +1,292 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.QueryOperators +import org.apache.commons.io.FileUtils +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.regex.Matcher +import java.util.regex.Pattern + +class JiangShuBlankDataStat { + + def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new JiangShuBlankDataStat() +// task.run() +// task.matchCredit(); +// task.repeat(); +// task.matchHouse(); + +// task.villa() + task.addressParse() + } + + def repeat(){ + def datas = ExcelUtils.read("D:\\数据\\江苏农行\\江苏农行1484条.xlsx", 0,1) + def keys = [] + for(def data : datas){ + def key = data['W']+"##"+data['X']+"##"+data['Y'] + if(!keys.contains(key)){ + keys << key + } + } + println keys.size(); + } + + def run(){ + def esDatas = findLogs(); + def keys = [] + def datas = [] + for(def d : esDatas){ + def key = d.fjw_city +"##"+d.fjw_region+" "+d.fjw_name + if(!keys.contains(key)){ + keys << key + datas << [d.parameter_city, d.parameter_region, d.parameter_address, d.parameter_size, d.fjw_city, d.fjw_region, d.fjw_name] + }else{ + println key + } + } + + ExcelUtils.write("D:\\数据\\江苏农行\\全数据.xlsx", datas, "数据") + +// ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\江苏农行930条.xlsx",{row-> +//// def address = row['G'] +//// if(!row['X']){ +//// def area = row['H'] +//// def retMap = findRow(address, area, esDatas); +//// if(retMap){ +//// row['V'] = retMap.fjw_city +//// row['W'] = retMap.fjw_region +//// row['X'] = retMap.fjw_name +//// row['Y'] = retMap.fjw_avgPrice +//// row['Z'] = retMap.fjw_totalPrice +//// +//// row['AG'] = retMap.parameter_city +//// row['AH'] = retMap.parameter_region +//// } +// def inputRegion = row['X'] +// def matchRegion = row['M'] +// if( inputRegion&& matchRegion && inputRegion.contains(matchRegion)){ +// row['AA'] = "包含" +// } +//// def matchName = row['Y'] +//// if(matchName && address && address.contains(matchName)){ +//// row['AL'] = "包含名称" +//// } +//// } +// },0) + } + + def matchCredit(){ + ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\全数据.xlsx",{row-> + def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G'], "district") + if(obj){ + row['H'] = obj.estate_type2 + } +// def dbo = CommonDatautils.queryDistrictPrice(row['E'], row['F'], row['G'], DateUtils.parseFromTightByPattern("2021-03-14","yyyy-MM-dd")) +// if(dbo){ +// row['H'] = dbo.avg_price +// row['I'] = dbo.credit +// row['J'] = dbo.value_type +// } +// +// def lianjiaObj = findThirdPrice(row['E'], row['F'], row['G'], "lianjia_esf_price"); +// if(lianjiaObj){ +// row['k'] = lianjiaObj.lianjia_esf_price +// row['L'] = lianjiaObj.s_date +// } +// def beikeObj = findThirdPrice(row['E'], row['F'], row['G'], "beike_esf_price"); +// if(beikeObj){ +// row['M'] = beikeObj.beike_esf_price +// row['N'] = beikeObj.s_date +// } + },0) + } + + def findThirdPrice(def city, def region, def name, def fieldName){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true)) + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1)) + def obj = null; + for(def d : dbcursor){ + obj = d; + break; + } + return obj; + } + + def findRow(address, area, esDatas){ + def retMap = [:] + for(def d : esDatas){ + + if(d.parameter_address ==address && area==d.parameter_size.toDouble()){ + retMap = d; + break; + } + } + return retMap; + } + + def findLogs(){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007753')).must(QueryBuilders.termQuery("request_uri", "/property/addressEvaluate")) + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); + def rets = [] + def infos = [] + if(houses){ + StringBuilder sbu=new StringBuilder(); + for (int k=0;k + def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G'], "district") + def mod = row['D'].toDouble()*0.2 + def startArea = row['D'].toDouble()- mod + def endArea = row['D'].toDouble() +mod + def prices = findHouses(row['E'], row['F'], "sell", startArea, endArea, row['G'], obj.alias); + if(!prices){ + if(row['D'].toDouble()<=100){ + prices = findHouses(row['E'], row['F'], "sell", 30, 100, row['G'], obj.alias); + }else if(row['D'].toDouble()<=200){ + prices = findHouses(row['E'], row['F'], "sell", 100, 200, row['G'], obj.alias); + }else{ + prices = findHouses(row['E'], row['F'], "sell", 200, 800, row['G'], obj.alias); + } + } + def avg_price = null + if(prices){ + avg_price = getAvgPrice(prices); + } + if(avg_price){ + row['Q'] = avg_price.toInteger() + } + + },0); + } + + def getAvgPrice(def prices){ + def avg_price = 0 + if(prices){ + def sum = prices.sum(); + avg_price = sum/prices.size() + } + return avg_price + } + + def findHouses(def city, def region, def cat, def startArea, def endArea, def name, def alias) { + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(20200301).to(20210318).includeLower(true).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(startArea).to(endArea).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if (!alias) { + alias = [] + } + if (!alias.contains(name)) { + alias.add(name) + } + + if (alias && alias.size() > 0) { + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for (def s : alias) { + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + inBuilder.should(QueryBuilders.matchQuery("root", "lianjia.com")); + inBuilder.should(QueryBuilders.matchQuery("root", "ke.com")); + queryBuilder.must(inBuilder) + + def prices = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + if (houses) { + for (int k = 0; k < houses.size(); k++) { + def map = houses[k]; + DBObject seawater = new BasicDBObject(map) + prices << seawater.avg_price + } + } + + println city +" "+region+" "+name+" "+ prices.size() + return prices; + } + + def villa(){ + ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以上.xlsx", { row -> + def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G']+"别墅", "district") + if(obj){ + def dbo = CommonDatautils.queryDistrictPrice(obj.city, obj.region, obj.name, DateUtils.parseFromTightByPattern("2021-03-14","yyyy-MM-dd")) + if(dbo){ + row['R'] = obj.name + row['S'] = dbo.avg_price + } + } + + },0); + } + + def addressParse(){ + ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以上.xlsx", { row -> + def resp = OpenApiUtils.matchDistrict2(row['E'], row['C'], true) + if(resp){ + println resp + row['T'] = resp.roomNumber + row['U'] = getFloor(resp.roomNumber); + } + + },0); + } + + def getFloor(def s){ + def floor = null + if(s){ + Pattern pa=Pattern.compile("^(\\d{1,2})0\\d{1}") + Matcher ma=pa.matcher(s) + if(ma.find()){ + floor = ma.group(1) + } else { + floor = null + } + } + return floor; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 8eed9d7e9..cc38a3d50 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"] for(def source :sources){ - def count1 = findTotalCount(source, 202010); - def count2 = findTotalCount(source, 202011); + def count1 = findTotalCount(source, 202101); + def count2 = findTotalCount(source, 202102); println source +" "+count1 +" "+count2 } }