update
This commit is contained in:
+2
-1
@@ -61,7 +61,8 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
|
||||
car_site :'carSite',estate_charge :'estateCharge',floorplan_pics :'floorplanPics',out_pics :'outPics',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
|
||||
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',licence :'licence',build_structure :'buildStructure',
|
||||
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure'
|
||||
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
|
||||
alias:'alias',addr_alias:'addrAlias'
|
||||
]
|
||||
public static final def OFFICE_FIELDS_MAP = [
|
||||
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',
|
||||
|
||||
@@ -44,7 +44,7 @@ class OpenApiUtils {
|
||||
|
||||
|
||||
static def matchDistrict2(def city, def address, def preciseLevelflag){
|
||||
address = address.replace("(","(").replace(")",")").replace(" ","")
|
||||
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","")
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||
if(preciseLevelflag){
|
||||
url = url +"&preciseLevel=strict"
|
||||
|
||||
@@ -20,13 +20,13 @@ import java.util.regex.Pattern
|
||||
/**
|
||||
* 抓取区域边框
|
||||
*/
|
||||
class RegionPoint {
|
||||
class RegionPointCrawl {
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
|
||||
|
||||
static main(args) {
|
||||
def task = new RegionPoint();
|
||||
def task = new RegionPointCrawl();
|
||||
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
|
||||
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
|
||||
def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||
|
||||
+2
-2
@@ -47,7 +47,7 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2020-11-01"
|
||||
static def statDate = "2021-02-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2020年11月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2021年3月\\二手房\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+5
-1
@@ -61,9 +61,13 @@ class HouseExport {
|
||||
def task = new HouseExport();
|
||||
|
||||
// task.run()
|
||||
task.marge()
|
||||
// task.marge()
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
// "重庆",
|
||||
def citys = ["北京"]
|
||||
|
||||
+34
-1
@@ -23,7 +23,9 @@ class AnjukeCrawlToDB {
|
||||
|
||||
// task.run()
|
||||
|
||||
task.stat()
|
||||
// task.stat()
|
||||
|
||||
task.addAliasStat()
|
||||
}
|
||||
|
||||
def stat(){
|
||||
@@ -77,6 +79,7 @@ class AnjukeCrawlToDB {
|
||||
}
|
||||
}
|
||||
|
||||
println estate_chargeCount
|
||||
|
||||
def estate_charge_pre = DataHandleUtils.getDoubleRound(2, estate_chargeCount*100/total)
|
||||
def build_area_pre = DataHandleUtils.getDoubleRound(2, build_areaCount*100/total)
|
||||
@@ -104,6 +107,36 @@ class AnjukeCrawlToDB {
|
||||
|
||||
}
|
||||
|
||||
def addAliasStat(){
|
||||
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\城市覆盖率统计(2021-03-14) (1).xls",{row->
|
||||
def city = row['B']
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["alias", "addr_alias"]);
|
||||
|
||||
def total = districts.size()
|
||||
|
||||
def aliasIndex = 0
|
||||
def addrAliasIndex = 0
|
||||
for(def d : districts){
|
||||
def alias = d.alias
|
||||
if(alias){
|
||||
aliasIndex++
|
||||
}
|
||||
|
||||
def addrAlias = d.addr_alias
|
||||
if(addrAlias){
|
||||
addrAliasIndex ++
|
||||
}
|
||||
}
|
||||
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
|
||||
row['BA'] = p
|
||||
|
||||
def p1 = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, addrAliasIndex*100/total)
|
||||
row['BB'] = p1
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+292
@@ -0,0 +1,292 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class JiangShuBlankDataStat {
|
||||
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new JiangShuBlankDataStat()
|
||||
// task.run()
|
||||
// task.matchCredit();
|
||||
// task.repeat();
|
||||
// task.matchHouse();
|
||||
|
||||
// task.villa()
|
||||
task.addressParse()
|
||||
}
|
||||
|
||||
def repeat(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\江苏农行1484条.xlsx", 0,1)
|
||||
def keys = []
|
||||
for(def data : datas){
|
||||
def key = data['W']+"##"+data['X']+"##"+data['Y']
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
}
|
||||
}
|
||||
println keys.size();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def esDatas = findLogs();
|
||||
def keys = []
|
||||
def datas = []
|
||||
for(def d : esDatas){
|
||||
def key = d.fjw_city +"##"+d.fjw_region+" "+d.fjw_name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
datas << [d.parameter_city, d.parameter_region, d.parameter_address, d.parameter_size, d.fjw_city, d.fjw_region, d.fjw_name]
|
||||
}else{
|
||||
println key
|
||||
}
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\江苏农行\\全数据.xlsx", datas, "数据")
|
||||
|
||||
// ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\江苏农行930条.xlsx",{row->
|
||||
//// def address = row['G']
|
||||
//// if(!row['X']){
|
||||
//// def area = row['H']
|
||||
//// def retMap = findRow(address, area, esDatas);
|
||||
//// if(retMap){
|
||||
//// row['V'] = retMap.fjw_city
|
||||
//// row['W'] = retMap.fjw_region
|
||||
//// row['X'] = retMap.fjw_name
|
||||
//// row['Y'] = retMap.fjw_avgPrice
|
||||
//// row['Z'] = retMap.fjw_totalPrice
|
||||
////
|
||||
//// row['AG'] = retMap.parameter_city
|
||||
//// row['AH'] = retMap.parameter_region
|
||||
//// }
|
||||
// def inputRegion = row['X']
|
||||
// def matchRegion = row['M']
|
||||
// if( inputRegion&& matchRegion && inputRegion.contains(matchRegion)){
|
||||
// row['AA'] = "包含"
|
||||
// }
|
||||
//// def matchName = row['Y']
|
||||
//// if(matchName && address && address.contains(matchName)){
|
||||
//// row['AL'] = "包含名称"
|
||||
//// }
|
||||
//// }
|
||||
// },0)
|
||||
}
|
||||
|
||||
def matchCredit(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\全数据.xlsx",{row->
|
||||
def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G'], "district")
|
||||
if(obj){
|
||||
row['H'] = obj.estate_type2
|
||||
}
|
||||
// def dbo = CommonDatautils.queryDistrictPrice(row['E'], row['F'], row['G'], DateUtils.parseFromTightByPattern("2021-03-14","yyyy-MM-dd"))
|
||||
// if(dbo){
|
||||
// row['H'] = dbo.avg_price
|
||||
// row['I'] = dbo.credit
|
||||
// row['J'] = dbo.value_type
|
||||
// }
|
||||
//
|
||||
// def lianjiaObj = findThirdPrice(row['E'], row['F'], row['G'], "lianjia_esf_price");
|
||||
// if(lianjiaObj){
|
||||
// row['k'] = lianjiaObj.lianjia_esf_price
|
||||
// row['L'] = lianjiaObj.s_date
|
||||
// }
|
||||
// def beikeObj = findThirdPrice(row['E'], row['F'], row['G'], "beike_esf_price");
|
||||
// if(beikeObj){
|
||||
// row['M'] = beikeObj.beike_esf_price
|
||||
// row['N'] = beikeObj.s_date
|
||||
// }
|
||||
},0)
|
||||
}
|
||||
|
||||
def findThirdPrice(def city, def region, def name, def fieldName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1))
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d;
|
||||
break;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
def findRow(address, area, esDatas){
|
||||
def retMap = [:]
|
||||
for(def d : esDatas){
|
||||
|
||||
if(d.parameter_address ==address && area==d.parameter_size.toDouble()){
|
||||
retMap = d;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
def findLogs(){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007753')).must(QueryBuilders.termQuery("request_uri", "/property/addressEvaluate"))
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
|
||||
def rets = []
|
||||
def infos = []
|
||||
if(houses){
|
||||
StringBuilder sbu=new StringBuilder();
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
println k
|
||||
def map = houses[k];
|
||||
println map
|
||||
if(map.parameter_map && map.res_map.result && map.code==200){
|
||||
def pMap = map.parameter_map;
|
||||
def rMap = map.res_map.result
|
||||
|
||||
|
||||
rets << ["parameter_city":pMap.city, "parameter_region":pMap.district, "parameter_address":pMap.address, "parameter_size":pMap.size,"fjw_city":rMap.city, "fjw_region":rMap.district, "fjw_name":rMap.name,"fjw_avgPrice":rMap.avgPrice, "fjw_totalPrice":rMap.totalPrice]
|
||||
|
||||
sbu.append(pMap.address+"\n");
|
||||
}
|
||||
}
|
||||
//FileUtils.writeStringToFile(new File("D:\\数据\\江苏农行\\log.txt"), sbu.toString(), "GBK")
|
||||
}
|
||||
|
||||
return rets;
|
||||
}
|
||||
|
||||
def matchHouse(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\全数据.xlsx",{row->
|
||||
def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G'], "district")
|
||||
def mod = row['D'].toDouble()*0.2
|
||||
def startArea = row['D'].toDouble()- mod
|
||||
def endArea = row['D'].toDouble() +mod
|
||||
def prices = findHouses(row['E'], row['F'], "sell", startArea, endArea, row['G'], obj.alias);
|
||||
if(!prices){
|
||||
if(row['D'].toDouble()<=100){
|
||||
prices = findHouses(row['E'], row['F'], "sell", 30, 100, row['G'], obj.alias);
|
||||
}else if(row['D'].toDouble()<=200){
|
||||
prices = findHouses(row['E'], row['F'], "sell", 100, 200, row['G'], obj.alias);
|
||||
}else{
|
||||
prices = findHouses(row['E'], row['F'], "sell", 200, 800, row['G'], obj.alias);
|
||||
}
|
||||
}
|
||||
def avg_price = null
|
||||
if(prices){
|
||||
avg_price = getAvgPrice(prices);
|
||||
}
|
||||
if(avg_price){
|
||||
row['Q'] = avg_price.toInteger()
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
def getAvgPrice(def prices){
|
||||
def avg_price = 0
|
||||
if(prices){
|
||||
def sum = prices.sum();
|
||||
avg_price = sum/prices.size()
|
||||
}
|
||||
return avg_price
|
||||
}
|
||||
|
||||
def findHouses(def city, def region, def cat, def startArea, def endArea, def name, def alias) {
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(20200301).to(20210318).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(startArea).to(endArea).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if (!alias) {
|
||||
alias = []
|
||||
}
|
||||
if (!alias.contains(name)) {
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
if (alias && alias.size() > 0) {
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for (def s : alias) {
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
inBuilder.should(QueryBuilders.matchQuery("root", "lianjia.com"));
|
||||
inBuilder.should(QueryBuilders.matchQuery("root", "ke.com"));
|
||||
queryBuilder.must(inBuilder)
|
||||
|
||||
def prices = []
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
if (houses) {
|
||||
for (int k = 0; k < houses.size(); k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
prices << seawater.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
println city +" "+region+" "+name+" "+ prices.size()
|
||||
return prices;
|
||||
}
|
||||
|
||||
def villa(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以上.xlsx", { row ->
|
||||
def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G']+"别墅", "district")
|
||||
if(obj){
|
||||
def dbo = CommonDatautils.queryDistrictPrice(obj.city, obj.region, obj.name, DateUtils.parseFromTightByPattern("2021-03-14","yyyy-MM-dd"))
|
||||
if(dbo){
|
||||
row['R'] = obj.name
|
||||
row['S'] = dbo.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
def addressParse(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以上.xlsx", { row ->
|
||||
def resp = OpenApiUtils.matchDistrict2(row['E'], row['C'], true)
|
||||
if(resp){
|
||||
println resp
|
||||
row['T'] = resp.roomNumber
|
||||
row['U'] = getFloor(resp.roomNumber);
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
def getFloor(def s){
|
||||
def floor = null
|
||||
if(s){
|
||||
Pattern pa=Pattern.compile("^(\\d{1,2})0\\d{1}")
|
||||
Matcher ma=pa.matcher(s)
|
||||
if(ma.find()){
|
||||
floor = ma.group(1)
|
||||
} else {
|
||||
floor = null
|
||||
}
|
||||
}
|
||||
return floor;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202010);
|
||||
def count2 = findTotalCount(source, 202011);
|
||||
def count1 = findTotalCount(source, 202101);
|
||||
def count2 = findTotalCount(source, 202102);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user