This commit is contained in:
2021-03-19 18:04:17 +08:00
parent 768fb924db
commit c1993be364
8 changed files with 340 additions and 10 deletions
@@ -61,7 +61,8 @@ class CityInfoAnaBatch extends BaseBatch {
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
car_site :'carSite',estate_charge :'estateCharge',floorplan_pics :'floorplanPics',out_pics :'outPics',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',licence :'licence',build_structure :'buildStructure',
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure'
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
alias:'alias',addr_alias:'addrAlias'
]
public static final def OFFICE_FIELDS_MAP = [
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',
@@ -44,7 +44,7 @@ class OpenApiUtils {
static def matchDistrict2(def city, def address, def preciseLevelflag){
address = address.replace("(","(").replace(")",")").replace(" ","")
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","")
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
if(preciseLevelflag){
url = url +"&preciseLevel=strict"
@@ -20,13 +20,13 @@ import java.util.regex.Pattern
/**
* 抓取区域边框
*/
class RegionPoint {
class RegionPointCrawl {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
static main(args) {
def task = new RegionPoint();
def task = new RegionPointCrawl();
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
@@ -47,7 +47,7 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2020-11-01"
static def statDate = "2021-02-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2020年11月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年3月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -61,9 +61,13 @@ class HouseExport {
def task = new HouseExport();
// task.run()
task.marge()
// task.marge()
}
def run(){
// "重庆",
def citys = ["北京"]
@@ -23,7 +23,9 @@ class AnjukeCrawlToDB {
// task.run()
task.stat()
// task.stat()
task.addAliasStat()
}
def stat(){
@@ -77,6 +79,7 @@ class AnjukeCrawlToDB {
}
}
println estate_chargeCount
def estate_charge_pre = DataHandleUtils.getDoubleRound(2, estate_chargeCount*100/total)
def build_area_pre = DataHandleUtils.getDoubleRound(2, build_areaCount*100/total)
@@ -104,6 +107,36 @@ class AnjukeCrawlToDB {
}
def addAliasStat(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\城市覆盖率统计(2021-03-14) (1).xls",{row->
def city = row['B']
def districts = CommonDatautils.findAllDistricts(city, "district", ["alias", "addr_alias"]);
def total = districts.size()
def aliasIndex = 0
def addrAliasIndex = 0
for(def d : districts){
def alias = d.alias
if(alias){
aliasIndex++
}
def addrAlias = d.addr_alias
if(addrAlias){
addrAliasIndex ++
}
}
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
row['BA'] = p
def p1 = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, addrAliasIndex*100/total)
row['BB'] = p1
},0)
}
@@ -0,0 +1,292 @@
package com.fangjia.script.gj.test
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import org.apache.commons.io.FileUtils
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.regex.Matcher
import java.util.regex.Pattern
class JiangShuBlankDataStat {
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new JiangShuBlankDataStat()
// task.run()
// task.matchCredit();
// task.repeat();
// task.matchHouse();
// task.villa()
task.addressParse()
}
def repeat(){
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\江苏农行1484条.xlsx", 0,1)
def keys = []
for(def data : datas){
def key = data['W']+"##"+data['X']+"##"+data['Y']
if(!keys.contains(key)){
keys << key
}
}
println keys.size();
}
def run(){
def esDatas = findLogs();
def keys = []
def datas = []
for(def d : esDatas){
def key = d.fjw_city +"##"+d.fjw_region+" "+d.fjw_name
if(!keys.contains(key)){
keys << key
datas << [d.parameter_city, d.parameter_region, d.parameter_address, d.parameter_size, d.fjw_city, d.fjw_region, d.fjw_name]
}else{
println key
}
}
ExcelUtils.write("D:\\数据\\江苏农行\\全数据.xlsx", datas, "数据")
// ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\江苏农行930条.xlsx",{row->
//// def address = row['G']
//// if(!row['X']){
//// def area = row['H']
//// def retMap = findRow(address, area, esDatas);
//// if(retMap){
//// row['V'] = retMap.fjw_city
//// row['W'] = retMap.fjw_region
//// row['X'] = retMap.fjw_name
//// row['Y'] = retMap.fjw_avgPrice
//// row['Z'] = retMap.fjw_totalPrice
////
//// row['AG'] = retMap.parameter_city
//// row['AH'] = retMap.parameter_region
//// }
// def inputRegion = row['X']
// def matchRegion = row['M']
// if( inputRegion&& matchRegion && inputRegion.contains(matchRegion)){
// row['AA'] = "包含"
// }
//// def matchName = row['Y']
//// if(matchName && address && address.contains(matchName)){
//// row['AL'] = "包含名称"
//// }
//// }
// },0)
}
def matchCredit(){
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\全数据.xlsx",{row->
def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G'], "district")
if(obj){
row['H'] = obj.estate_type2
}
// def dbo = CommonDatautils.queryDistrictPrice(row['E'], row['F'], row['G'], DateUtils.parseFromTightByPattern("2021-03-14","yyyy-MM-dd"))
// if(dbo){
// row['H'] = dbo.avg_price
// row['I'] = dbo.credit
// row['J'] = dbo.value_type
// }
//
// def lianjiaObj = findThirdPrice(row['E'], row['F'], row['G'], "lianjia_esf_price");
// if(lianjiaObj){
// row['k'] = lianjiaObj.lianjia_esf_price
// row['L'] = lianjiaObj.s_date
// }
// def beikeObj = findThirdPrice(row['E'], row['F'], row['G'], "beike_esf_price");
// if(beikeObj){
// row['M'] = beikeObj.beike_esf_price
// row['N'] = beikeObj.s_date
// }
},0)
}
def findThirdPrice(def city, def region, def name, def fieldName){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1))
def obj = null;
for(def d : dbcursor){
obj = d;
break;
}
return obj;
}
def findRow(address, area, esDatas){
def retMap = [:]
for(def d : esDatas){
if(d.parameter_address ==address && area==d.parameter_size.toDouble()){
retMap = d;
break;
}
}
return retMap;
}
def findLogs(){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007753')).must(QueryBuilders.termQuery("request_uri", "/property/addressEvaluate"))
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
def rets = []
def infos = []
if(houses){
StringBuilder sbu=new StringBuilder();
for (int k=0;k<houses.size();k++) {
println k
def map = houses[k];
println map
if(map.parameter_map && map.res_map.result && map.code==200){
def pMap = map.parameter_map;
def rMap = map.res_map.result
rets << ["parameter_city":pMap.city, "parameter_region":pMap.district, "parameter_address":pMap.address, "parameter_size":pMap.size,"fjw_city":rMap.city, "fjw_region":rMap.district, "fjw_name":rMap.name,"fjw_avgPrice":rMap.avgPrice, "fjw_totalPrice":rMap.totalPrice]
sbu.append(pMap.address+"\n");
}
}
//FileUtils.writeStringToFile(new File("D:\\数据\\江苏农行\\log.txt"), sbu.toString(), "GBK")
}
return rets;
}
def matchHouse(){
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\全数据.xlsx",{row->
def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G'], "district")
def mod = row['D'].toDouble()*0.2
def startArea = row['D'].toDouble()- mod
def endArea = row['D'].toDouble() +mod
def prices = findHouses(row['E'], row['F'], "sell", startArea, endArea, row['G'], obj.alias);
if(!prices){
if(row['D'].toDouble()<=100){
prices = findHouses(row['E'], row['F'], "sell", 30, 100, row['G'], obj.alias);
}else if(row['D'].toDouble()<=200){
prices = findHouses(row['E'], row['F'], "sell", 100, 200, row['G'], obj.alias);
}else{
prices = findHouses(row['E'], row['F'], "sell", 200, 800, row['G'], obj.alias);
}
}
def avg_price = null
if(prices){
avg_price = getAvgPrice(prices);
}
if(avg_price){
row['Q'] = avg_price.toInteger()
}
},0);
}
def getAvgPrice(def prices){
def avg_price = 0
if(prices){
def sum = prices.sum();
avg_price = sum/prices.size()
}
return avg_price
}
def findHouses(def city, def region, def cat, def startArea, def endArea, def name, def alias) {
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(20200301).to(20210318).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(startArea).to(endArea).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if (!alias) {
alias = []
}
if (!alias.contains(name)) {
alias.add(name)
}
if (alias && alias.size() > 0) {
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for (def s : alias) {
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
inBuilder.should(QueryBuilders.matchQuery("root", "lianjia.com"));
inBuilder.should(QueryBuilders.matchQuery("root", "ke.com"));
queryBuilder.must(inBuilder)
def prices = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
if (houses) {
for (int k = 0; k < houses.size(); k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
prices << seawater.avg_price
}
}
println city +" "+region+" "+name+" "+ prices.size()
return prices;
}
def villa(){
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以上.xlsx", { row ->
def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G']+"别墅", "district")
if(obj){
def dbo = CommonDatautils.queryDistrictPrice(obj.city, obj.region, obj.name, DateUtils.parseFromTightByPattern("2021-03-14","yyyy-MM-dd"))
if(dbo){
row['R'] = obj.name
row['S'] = dbo.avg_price
}
}
},0);
}
def addressParse(){
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以上.xlsx", { row ->
def resp = OpenApiUtils.matchDistrict2(row['E'], row['C'], true)
if(resp){
println resp
row['T'] = resp.roomNumber
row['U'] = getFloor(resp.roomNumber);
}
},0);
}
def getFloor(def s){
def floor = null
if(s){
Pattern pa=Pattern.compile("^(\\d{1,2})0\\d{1}")
Matcher ma=pa.matcher(s)
if(ma.find()){
floor = ma.group(1)
} else {
floor = null
}
}
return floor;
}
}
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
for(def source :sources){
def count1 = findTotalCount(source, 202010);
def count2 = findTotalCount(source, 202011);
def count1 = findTotalCount(source, 202101);
def count2 = findTotalCount(source, 202102);
println source +" "+count1 +" "+count2
}
}