Merge remote-tracking branch 'origin/master'

This commit is contained in:
2021-03-25 10:39:03 +08:00
14 changed files with 784 additions and 144 deletions
+1 -5
View File
@@ -100,11 +100,7 @@
</pluginManagement>
</build>
<dependencies>
<dependency>
<groupId>com.fangjia.cloud.api.client</groupId>
<artifactId>FangJiaApiSwaggerClient</artifactId>
<version>1.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
@@ -61,7 +61,8 @@ class CityInfoAnaBatch extends BaseBatch {
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
car_site :'carSite',estate_charge :'estateCharge',floorplan_pics :'floorplanPics',out_pics :'outPics',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',licence :'licence',build_structure :'buildStructure',
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure'
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
alias:'alias',addr_alias:'addrAlias'
]
public static final def OFFICE_FIELDS_MAP = [
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',
@@ -31,7 +31,7 @@ class LandDetailCrawl {
def task = new LandDetailCrawl();
// task.run()
task.updateData();
task.undateOldData();
// task.undateOldData();
}
def run(){
@@ -65,7 +65,7 @@ class LandDetailCrawl {
}
def updateData(){
def datas = ExcelUtils.read("D:\\数据\\友达\\土地--12月.xlsx",0, 1)
def datas = ExcelUtils.read("D:\\数据\\土地\\土地成交3月(2).xlsx",0, 1)
if(datas && datas.size()>0){
for(def data : datas){
def id = data['A']
@@ -44,7 +44,7 @@ class OpenApiUtils {
static def matchDistrict2(def city, def address, def preciseLevelflag){
address = address.replace("(","(").replace(")",")").replace(" ","")
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","")
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
if(preciseLevelflag){
url = url +"&preciseLevel=strict"
@@ -0,0 +1,224 @@
package com.fangjia.script.gj
import com.fangjia.core.Constants
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.math.RandomUtils
import org.apache.commons.lang3.StringUtils
import org.bson.Document
import java.util.regex.Pattern
/**
* 抓取区域边框
*/
class RegionPointCrawl {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
static main(args) {
def task = new RegionPointCrawl();
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
for(def p : provinces){
task.crawlCityAndDistrictCode(p);
task.run(p);
}
}
def crawlCityAndDistrictCode(def province){
crawlCity(province)
}
def run(city){
def cityCodes = findCityCode(city);
if(cityCodes){
for(def cityMap : cityCodes){
def codeMap = findAllDistrictCode(cityMap.city_code);
def index = 0
for(def m : codeMap){
index ++
def ret = polygon(m.ad_code, "0", "all");
if(!ret){
println m.name +" " + m.ad_code + "无结果"
}
println index +" "+cityMap.name+" "+m.name+" "+ret.size()
if(ret.size()>0){
saveData(cityMap.name, m.name, m.ad_code , ret);
}
}
}
}
}
def crawlCity(def province){
def ret = polygon(province, "2", "base");
if(ret.size()>0){
for(def map : ret){
if(map.level == "province"){
def cityList = map.districts
if(cityList && cityList.size()>0){
for(def cityMap : cityList){
if(cityMap.level == "city"){
saveGaoDeCityRegionCode(province, cityMap.name, cityMap.citycode, cityMap.adcode, cityMap.center, cityMap.level);
saveGaoDeRegionCode(province, cityMap.districts);
}
}
}
}
}
}
}
def saveGaoDeRegionCode(province, districts){
if(districts){
for(def dist : districts){
if(dist.level=="district"){
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
}
}
}
}
def findCityCode(city){
Pattern pattern = Pattern.compile(city, Pattern.CASE_INSENSITIVE);
def q = null;
if(city.contains("省") || city.endsWith("自治区")){
q = new BasicDBObject("province", pattern);
}else{
q = new BasicDBObject("name", pattern);
}
q.put("level", "city")
def dbcursor = GAODECITYCODECOLL.find(q)
def rets = []
for(def d : dbcursor){
def city_code = d.get("city_code")
if(!rets.contains(city_code)){
rets << d
}
}
return rets;
}
private BasicDBObject getLikeStr(String findStr) {
Pattern pattern = Pattern.compile(findStr + ".*\$", Pattern.MULTILINE);
return new BasicDBObject("$regex", pattern);
}
def findAllDistrictCode(def citycode){
def q = new BasicDBObject()
q.put("city_code", citycode)
q.put("level", "district")
def dbcursor = GAODECITYCODECOLL.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
def saveGaoDeCityRegionCode(province, name, citycode, adcode, center, level){
def q = new BasicDBObject()
q.put("city_code", citycode)
q.put("province", province)
q.put("level", level)
q.put("name", name)
if(adcode){
q.put("ad_code", adcode)
}
def o = GAODECITYCODECOLL.findOne(q)
if(!o){
q.put("c_date", new Date())
q.put("center", center)
GAODECITYCODECOLL.save(q);
}else{
println name +" "+adcode+" 已存在"
}
}
def saveData(city, region, code, ret){
def q = new BasicDBObject()
q.put("code", code)
def dbo = POLYLINECOLL.findOne(q)
if(!dbo){
def o = new BasicDBObject()
o.put("city", city)
o.put("region", region)
o.put("code", code)
o.put("center", ret.center.get(0))
o.put("level", ret.level.get(0))
o.put("polyline", ret.polyline.get(0));
POLYLINECOLL.save(o);
}
}
public static def polygon(def keyword, def subdistrict, def extensions) {
String url = "https://restapi.amap.com/v3/config/district";
def params = [:]
params.put("keywords", keyword)
params.put("subdistrict",subdistrict)
params.put("extensions",extensions)
String html = null;
def jo = null;
int errorCount = 0;
String status = "error"
def resultMap = [:]
try {
HttpRequest http = new HttpRequest();
while (errorCount<5){//"2af0b8533415a612b98d59bb93d664b9" 公司key
def key = getRandomKey2() //
params.put("key", key)
html = http.get(url, params,"utf-8")
if (StringUtils.isNotEmpty(html)) {
jo = JSONObject.fromObject(html)
if(jo.infocode!="10000"){
errorCount ++
Thread.sleep(30*1000)
// println " 不可用Key: "+key
}else{
errorCount = 100
status = "normal"
// println " 可用Key: "+key + " "+html
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
return jo.districts;
}
static String getRandomKey2() {
def KeyList = [
"2af0b8533415a612b98d59bb93d664b9",
"0246c57bbcf24aa246c207b3c99dfe41",
"042328b3f0fa5daf426802d379521d87",
"055d4f1b5b64a2bce18f4fe1f15c2639",
"d5484f5170502405d88bbaffe1b496f0",
"e74ccc4258079d2200199695110cceb1",
"f58629fc799fbc8537ca85046c6d44a5",
"291073f17ee0b963ccb1927ed92bf265",
"2a1fb75990f2b39f6221e50892dd3988"
]
def count = RandomUtils.nextInt(KeyList.size())
return KeyList.get(count);
}
}
H
@@ -47,7 +47,7 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2020-11-01"
static def statDate = "2021-02-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2020年11月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年3月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -60,9 +60,14 @@ class HouseExport {
static main(args) {
def task = new HouseExport();
task.run()
// task.run()
// task.marge()
}
def run(){
// "重庆",
def citys = ["北京"]
@@ -777,4 +782,23 @@ def GenNonDuplicateID(randomLength){
def marge(){
File file = new File("D:\\数据\\建行\\新建文本文档.txt");
def ids = new HashSet();
def index = 0
file.eachLine{it->
index++
if(index>1){
def arrs = it.split("\\|@\\|")
def id = arrs[1]
if(!ids.contains(id)){
ids << id
}
}
}
println index
println ids.size()
}
}
@@ -0,0 +1,40 @@
package com.fangjia.script.gj.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
class FieldStat {
static main(args) {
def task = new FieldStat();
task.run()
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区融合结果概况2.xlsx", { row ->
def city = row['A']
if(city){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]);
def total = districts.size()
def aliasIndex = 0
def aliasTotalCount = 0
for(def d : districts){
def alias = d.alias
if(alias){
aliasIndex++
aliasTotalCount = aliasTotalCount + alias.size()
}
}
row['J'] = aliasTotalCount
row['K'] = aliasIndex
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
row['L'] = p+"%"
row['M'] = total
println city +" "+p
}
});
println "finish"
}
}
@@ -23,7 +23,9 @@ class AnjukeCrawlToDB {
// task.run()
task.stat()
// task.stat()
task.addAliasStat()
}
def stat(){
@@ -77,6 +79,7 @@ class AnjukeCrawlToDB {
}
}
println estate_chargeCount
def estate_charge_pre = DataHandleUtils.getDoubleRound(2, estate_chargeCount*100/total)
def build_area_pre = DataHandleUtils.getDoubleRound(2, build_areaCount*100/total)
@@ -104,6 +107,36 @@ class AnjukeCrawlToDB {
}
def addAliasStat(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\城市覆盖率统计(2021-03-14) (1).xls",{row->
def city = row['B']
def districts = CommonDatautils.findAllDistricts(city, "district", ["alias", "addr_alias"]);
def total = districts.size()
def aliasIndex = 0
def addrAliasIndex = 0
for(def d : districts){
def alias = d.alias
if(alias){
aliasIndex++
}
def addrAlias = d.addr_alias
if(addrAlias){
addrAliasIndex ++
}
}
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
row['BA'] = p
def p1 = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, addrAliasIndex*100/total)
row['BB'] = p1
},0)
}
@@ -0,0 +1,292 @@
package com.fangjia.script.gj.test
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import org.apache.commons.io.FileUtils
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.regex.Matcher
import java.util.regex.Pattern
class JiangShuBlankDataStat {
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new JiangShuBlankDataStat()
// task.run()
// task.matchCredit();
// task.repeat();
// task.matchHouse();
// task.villa()
task.addressParse()
}
def repeat(){
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\江苏农行1484条.xlsx", 0,1)
def keys = []
for(def data : datas){
def key = data['W']+"##"+data['X']+"##"+data['Y']
if(!keys.contains(key)){
keys << key
}
}
println keys.size();
}
def run(){
def esDatas = findLogs();
def keys = []
def datas = []
for(def d : esDatas){
def key = d.fjw_city +"##"+d.fjw_region+" "+d.fjw_name
if(!keys.contains(key)){
keys << key
datas << [d.parameter_city, d.parameter_region, d.parameter_address, d.parameter_size, d.fjw_city, d.fjw_region, d.fjw_name]
}else{
println key
}
}
ExcelUtils.write("D:\\数据\\江苏农行\\全数据.xlsx", datas, "数据")
// ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\江苏农行930条.xlsx",{row->
//// def address = row['G']
//// if(!row['X']){
//// def area = row['H']
//// def retMap = findRow(address, area, esDatas);
//// if(retMap){
//// row['V'] = retMap.fjw_city
//// row['W'] = retMap.fjw_region
//// row['X'] = retMap.fjw_name
//// row['Y'] = retMap.fjw_avgPrice
//// row['Z'] = retMap.fjw_totalPrice
////
//// row['AG'] = retMap.parameter_city
//// row['AH'] = retMap.parameter_region
//// }
// def inputRegion = row['X']
// def matchRegion = row['M']
// if( inputRegion&& matchRegion && inputRegion.contains(matchRegion)){
// row['AA'] = "包含"
// }
//// def matchName = row['Y']
//// if(matchName && address && address.contains(matchName)){
//// row['AL'] = "包含名称"
//// }
//// }
// },0)
}
def matchCredit(){
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\全数据.xlsx",{row->
def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G'], "district")
if(obj){
row['H'] = obj.estate_type2
}
// def dbo = CommonDatautils.queryDistrictPrice(row['E'], row['F'], row['G'], DateUtils.parseFromTightByPattern("2021-03-14","yyyy-MM-dd"))
// if(dbo){
// row['H'] = dbo.avg_price
// row['I'] = dbo.credit
// row['J'] = dbo.value_type
// }
//
// def lianjiaObj = findThirdPrice(row['E'], row['F'], row['G'], "lianjia_esf_price");
// if(lianjiaObj){
// row['k'] = lianjiaObj.lianjia_esf_price
// row['L'] = lianjiaObj.s_date
// }
// def beikeObj = findThirdPrice(row['E'], row['F'], row['G'], "beike_esf_price");
// if(beikeObj){
// row['M'] = beikeObj.beike_esf_price
// row['N'] = beikeObj.s_date
// }
},0)
}
def findThirdPrice(def city, def region, def name, def fieldName){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1))
def obj = null;
for(def d : dbcursor){
obj = d;
break;
}
return obj;
}
def findRow(address, area, esDatas){
def retMap = [:]
for(def d : esDatas){
if(d.parameter_address ==address && area==d.parameter_size.toDouble()){
retMap = d;
break;
}
}
return retMap;
}
def findLogs(){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007753')).must(QueryBuilders.termQuery("request_uri", "/property/addressEvaluate"))
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
def rets = []
def infos = []
if(houses){
StringBuilder sbu=new StringBuilder();
for (int k=0;k<houses.size();k++) {
println k
def map = houses[k];
println map
if(map.parameter_map && map.res_map.result && map.code==200){
def pMap = map.parameter_map;
def rMap = map.res_map.result
rets << ["parameter_city":pMap.city, "parameter_region":pMap.district, "parameter_address":pMap.address, "parameter_size":pMap.size,"fjw_city":rMap.city, "fjw_region":rMap.district, "fjw_name":rMap.name,"fjw_avgPrice":rMap.avgPrice, "fjw_totalPrice":rMap.totalPrice]
sbu.append(pMap.address+"\n");
}
}
//FileUtils.writeStringToFile(new File("D:\\数据\\江苏农行\\log.txt"), sbu.toString(), "GBK")
}
return rets;
}
def matchHouse(){
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\全数据.xlsx",{row->
def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G'], "district")
def mod = row['D'].toDouble()*0.2
def startArea = row['D'].toDouble()- mod
def endArea = row['D'].toDouble() +mod
def prices = findHouses(row['E'], row['F'], "sell", startArea, endArea, row['G'], obj.alias);
if(!prices){
if(row['D'].toDouble()<=100){
prices = findHouses(row['E'], row['F'], "sell", 30, 100, row['G'], obj.alias);
}else if(row['D'].toDouble()<=200){
prices = findHouses(row['E'], row['F'], "sell", 100, 200, row['G'], obj.alias);
}else{
prices = findHouses(row['E'], row['F'], "sell", 200, 800, row['G'], obj.alias);
}
}
def avg_price = null
if(prices){
avg_price = getAvgPrice(prices);
}
if(avg_price){
row['Q'] = avg_price.toInteger()
}
},0);
}
def getAvgPrice(def prices){
def avg_price = 0
if(prices){
def sum = prices.sum();
avg_price = sum/prices.size()
}
return avg_price
}
def findHouses(def city, def region, def cat, def startArea, def endArea, def name, def alias) {
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(20200301).to(20210318).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(startArea).to(endArea).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if (!alias) {
alias = []
}
if (!alias.contains(name)) {
alias.add(name)
}
if (alias && alias.size() > 0) {
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for (def s : alias) {
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
inBuilder.should(QueryBuilders.matchQuery("root", "lianjia.com"));
inBuilder.should(QueryBuilders.matchQuery("root", "ke.com"));
queryBuilder.must(inBuilder)
def prices = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
if (houses) {
for (int k = 0; k < houses.size(); k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
prices << seawater.avg_price
}
}
println city +" "+region+" "+name+" "+ prices.size()
return prices;
}
def villa(){
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以上.xlsx", { row ->
def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G']+"别墅", "district")
if(obj){
def dbo = CommonDatautils.queryDistrictPrice(obj.city, obj.region, obj.name, DateUtils.parseFromTightByPattern("2021-03-14","yyyy-MM-dd"))
if(dbo){
row['R'] = obj.name
row['S'] = dbo.avg_price
}
}
},0);
}
def addressParse(){
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以上.xlsx", { row ->
def resp = OpenApiUtils.matchDistrict2(row['E'], row['C'], true)
if(resp){
println resp
row['T'] = resp.roomNumber
row['U'] = getFloor(resp.roomNumber);
}
},0);
}
def getFloor(def s){
def floor = null
if(s){
Pattern pa=Pattern.compile("^(\\d{1,2})0\\d{1}")
Matcher ma=pa.matcher(s)
if(ma.find()){
floor = ma.group(1)
} else {
floor = null
}
}
return floor;
}
}
@@ -630,128 +630,133 @@ class CZBlankDataExport {
sqlBuffer.append(" and m_date >='"+startDate+"'").append(" and m_date <'"+endDate+"'");
// sqlBuffer.append(" and m_date >='"+pDate+"'").append(" and m_date <='"+pDate+"'");
sqlBuffer.append(" ORDER BY p_date desc limit 200")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
ResultSet rs = null
def index = 0;
def houses = [];
while (rs.next()) {
try{
rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def area = rs.getObject("area")
while (rs.next()) {
if(area && (area>1000||area<30)){
continue;
}
def floor = rs.getObject("floor")
if(floor && floor>100){
continue;
}
def height = rs.getObject("height")
if(height && (height>100||height==0)){
continue;
}
def total_price = rs.getObject("total_price")
def house_area = area;
def avg_price = total_price/house_area;
def distPrice = distPriceMaps.get(dbo.city +"##"+dbo.region+"##"+dbo.name);
if(distPrice){
def maxValue = distPrice + distPrice*0.15;
def minValue = distPrice - distPrice*0.15;
if(avg_price>maxValue.toInteger() || avg_price<minValue.toInteger()){
def area = rs.getObject("area")
if(area && (area>1000||area<30)){
continue;
}
}
def district_name = rs.getObject("district_name")
if(district_name.endsWith("别墅") && height && height>=10){
continue;
}
def floor = rs.getObject("floor")
if(floor && floor>100){
continue;
}
def m = [:];
//小区名称
m.name = dbo.name;
if(dbo.alias){
def newAlias = [];
for(def tname : dbo.alias){
if(tname && tname.length()>=2){
newAlias << tname;
def height = rs.getObject("height")
if(height && (height>100||height==0)){
continue;
}
def total_price = rs.getObject("total_price")
def house_area = area;
def avg_price = total_price/house_area;
def distPrice = distPriceMaps.get(dbo.city +"##"+dbo.region+"##"+dbo.name);
if(distPrice){
def maxValue = distPrice + distPrice*0.15;
def minValue = distPrice - distPrice*0.15;
if(avg_price>maxValue.toInteger() || avg_price<minValue.toInteger()){
continue;
}
}
if(newAlias){
m.all_name = dbo.name + "("+newAlias.join(",")+")"
def district_name = rs.getObject("district_name")
if(district_name.endsWith("别墅") && height && height>=10){
continue;
}
def m = [:];
//小区名称
m.name = dbo.name;
if(dbo.alias){
def newAlias = [];
for(def tname : dbo.alias){
if(tname && tname.length()>=2){
newAlias << tname;
}
}
if(newAlias){
m.all_name = dbo.name + "("+newAlias.join(",")+")"
}else{
m.all_name = dbo.name;
}
}else{
m.all_name = dbo.name;
}
}else{
m.all_name = dbo.name;
m.city = rs.getObject("city");
m.region = rs.getObject("region");
m.district_id= dbo.num_id;
m.house_id = createHouseId(rs.getObject("url"));//rs.getObject("_id").toString();
//建筑物总楼层
m.height = height>0?height:null;
//所在楼层
m.floor = getHouseFloor(floor, m.height);
//建筑面积
m.house_area = area;
//挂牌价
m.total_price = total_price;
def keys = m.region +"-"+ m.name+"-"+m.house_area+"-"+m.height+"-"+m.floor+"-"+m.total_price;
if(!houses.contains(keys)){
houses << keys;
}else{
println "此房源已存在....."
continue;
}
if(index > 30){
break;
}
m.room = rs.getObject("room");
m.hall = rs.getObject("hall");
m.estate_type = rs.getObject("estate_type");
//建成年代
m.build_date = getBuildDate(rs.getObject("build_date"), dbo);
//总使用年限
if(districtIds.contains(m.district_id)){
m.property_life = 70;
}else{
m.property_life = getPropertyLife(dbo.property_life);
}
//朝向
m.direction = getDirectionCode(rs.getObject("direction"), dbo.estate_type2);
//建筑结构
m.build_structure_code = getBuildStructureCode(dbo.build_date, dbo.build_structure);
//建筑类别
m.build_type_code = getBuildTypeCode(dbo.name, dbo.build_type, height);
if(m.build_type_code=='3' && m.height>6){
m.height = null
}
//内部装修与成新度
m.fitment_code = getFitmentCode(rs.getObject("fitment"));
//是否有电梯
m.elevator_code = getElevatorCode(height, m.build_type_code);
//房价网评估价
//m.evaluation_price = getEvaluationPrice(m);
//println "[${index++}] "+new Date();
//评估时间
m.evaluation_date = DateUtils.formatDateByPattern(new Date(),"yyyyMMdd");
//挂牌时间
m.publish_date = rs.getObject("p_date").toString()//DateUtils.formatDateByPattern(obj.publish_date,"yyyyMMdd");
m.status=1;
index++;
saveData(m);
}
m.city = rs.getObject("city");
m.region = rs.getObject("region");
m.district_id= dbo.num_id;
}catch(Exception e){
m.house_id = createHouseId(rs.getObject("url"));//rs.getObject("_id").toString();
//建筑物总楼层
m.height = height>0?height:null;
//所在楼层
m.floor = getHouseFloor(floor, m.height);
//建筑面积
m.house_area = area;
//挂牌价
m.total_price = total_price;
def keys = m.region +"-"+ m.name+"-"+m.house_area+"-"+m.height+"-"+m.floor+"-"+m.total_price;
if(!houses.contains(keys)){
houses << keys;
}else{
println "此房源已存在....."
continue;
}
if(index > 30){
break;
}
m.room = rs.getObject("room");
m.hall = rs.getObject("hall");
m.estate_type = rs.getObject("estate_type");
//建成年代
m.build_date = getBuildDate(rs.getObject("build_date"), dbo);
//总使用年限
if(districtIds.contains(m.district_id)){
m.property_life = 70;
}else{
m.property_life = getPropertyLife(dbo.property_life);
}
//朝向
m.direction = getDirectionCode(rs.getObject("direction"), dbo.estate_type2);
//建筑结构
m.build_structure_code = getBuildStructureCode(dbo.build_date, dbo.build_structure);
//建筑类别
m.build_type_code = getBuildTypeCode(dbo.name, dbo.build_type, height);
if(m.build_type_code=='3' && m.height>6){
m.height = null
}
//内部装修与成新度
m.fitment_code = getFitmentCode(rs.getObject("fitment"));
//是否有电梯
m.elevator_code = getElevatorCode(height, m.build_type_code);
//房价网评估价
//m.evaluation_price = getEvaluationPrice(m);
//println "[${index++}] "+new Date();
//评估时间
m.evaluation_date = DateUtils.formatDateByPattern(new Date(),"yyyyMMdd");
//挂牌时间
m.publish_date = rs.getObject("p_date").toString()//DateUtils.formatDateByPattern(obj.publish_date,"yyyyMMdd");
m.status=1;
index++;
saveData(m);
}
return index;
}
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
for(def source :sources){
def count1 = findTotalCount(source, 202010);
def count2 = findTotalCount(source, 202011);
def count1 = findTotalCount(source, 202101);
def count2 = findTotalCount(source, 202102);
println source +" "+count1 +" "+count2
}
}
@@ -4,11 +4,14 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 浙商本地化部署同步脚本
*/
@BatchOwner(Watcher.GaoJuan)
class BasicInfoDataSynch extends BaseBatch{
// final static DBCollection SERVER_AREA_CONN = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo)
@@ -29,10 +32,10 @@ class BasicInfoDataSynch extends BaseBatch{
@Override
public void process(Object args) {
// synchCitys();
// synchRegion()
// synchTown()
// synchDict()
synchCitys();
synchRegion()
synchTown()
synchDict()
synchDistrict("district")
synchDistrict("office")
}
@@ -47,46 +50,49 @@ class BasicInfoDataSynch extends BaseBatch{
zsCitysMap.put(zsCitys._id.toString(), zsCitys)
zsDatas << zsCitys
}
//数据存在则修改,不存在则保存
//数据存在则修改,不存在则保存
if(fangjiaCitys){
for(def fjCity : fangjiaCitys){
if(fjCity.display!="全国"){
if(fjCity.display!="全国"){
fjCitysMap.put(fjCity._id.toString(), fjCity)
}
def zsCity = zsCitysMap.get(fjCity._id.toString())
if(zsCity){//修改
if(zsCity){//修改
if(zsCity.province != fjCity.province || zsCity.alias!=fjCity.alias){
def udata = new BasicDBObject()
udata.put("province",fjCity.province)
udata.put("alias",fjCity.alias)
udata.put("status",0)
udata.put("update_date",new Date().getTime())
ZS_CITYS_CONN.update(new BasicDBObject("_id",zsCity._id), new BasicDBObject('$set',udata))
}
}else{//保存
def addUata = new BasicDBObject()
}else{//保存
def addUata = new BasicDBObject()
addUata.put("_id",fjCity._id)
addUata.put("display",fjCity.display)
addUata.put("province",fjCity.province)
addUata.put("alias",fjCity.alias)
addUata.put("status",0)
addUata.put("update_date",new Date().getTime())
ZS_CITYS_CONN.save(addUata);
}
}
}
//老数据不在新数据中,被删除时,状态修改为1
//老数据不在新数据中,被删除时,状态修改为1
if(zsDatas){
for(def zsCity : zsDatas){
def fjCity = fjCitysMap.get(zsCity._id.toString())
if(!fjCity){
def udata = new BasicDBObject()
udata.put("status",1)
udata.put("update_date",new Date().getTime())
ZS_CITYS_CONN.update(new BasicDBObject("_id",zsCity._id), new BasicDBObject('$set',udata))
}
}
}
println "[${fangjiaCitys.size()}]个城市同步完成"
println "[${fangjiaCitys.size()}]个城市同步完成"
}
def synchRegion(){
@@ -110,6 +116,7 @@ class BasicInfoDataSynch extends BaseBatch{
udata.put("pattern",fjRegion.pattern)
udata.put("org_name",fjRegion.org_name)
udata.put("status",0)
udata.put("update_date",new Date().getTime())
if(zsRegionObj){
ZS_REGION_CONN.update(new BasicDBObject("_id",zsRegionObj._id), new BasicDBObject('$set',udata))
}else{
@@ -125,11 +132,12 @@ class BasicInfoDataSynch extends BaseBatch{
if(!fjRegion){
def udata = new BasicDBObject()
udata.put("status",1)
udata.put("update_date",new Date().getTime())
ZS_REGION_CONN.update(new BasicDBObject("_id",zsRegion._id), new BasicDBObject('$set',udata))
}
}
}
println "[${fjRegions.size()}]个行政区域同步完成"
println "[${fjRegions.size()}]个行政区域同步完成"
}
def synchTown(){
@@ -152,6 +160,7 @@ class BasicInfoDataSynch extends BaseBatch{
udata.put("short_name",fjTown.short_name)
udata.put("name",fjTown.name)
udata.put("status",0)
udata.put("update_date",new Date().getTime())
if(zsTownObj){
if(fjTown.city!=zsTownObj.city||fjTown.region!=zsTownObj.region||fjTown.short_name!=zsTownObj.short_name||fjTown.name!=zsTownObj.name||zsTownObj.status!=0){
ZS_TOWN_CONN.update(new BasicDBObject("_id",zsTownObj._id), new BasicDBObject('$set',udata))
@@ -169,11 +178,12 @@ class BasicInfoDataSynch extends BaseBatch{
if(!fjTown){
def udata = new BasicDBObject()
udata.put("status",1)
udata.put("update_date",new Date().getTime())
ZS_TOWN_CONN.update(new BasicDBObject("_id",zsTown._id), new BasicDBObject('$set',udata))
}
}
}
println "[${fjTowns.size()}]个乡镇同步完成"
println "[${fjTowns.size()}]个乡镇同步完成"
}
def synchDict(){
def fjDicts = CommonDatautils.getAllDictionary()
@@ -194,6 +204,7 @@ class BasicInfoDataSynch extends BaseBatch{
udata.put("cat",fjDict.cat)
udata.put("name",fjDict.name)
udata.put("status",0)
udata.put("update_date",new Date().getTime())
if(zsDictObj){
if(fjDict.cat!=zsDictObj.cat||fjDict.name!=zsDictObj.name||zsDictObj.status!=0){
ZS_DICT_CONN.update(new BasicDBObject("_id",zsDictObj._id), new BasicDBObject('$set',udata))
@@ -220,11 +231,12 @@ class BasicInfoDataSynch extends BaseBatch{
if(!fjDict){
def udata = new BasicDBObject()
udata.put("status",1)
udata.put("update_date",new Date().getTime())
ZS_DICT_CONN.update(new BasicDBObject("_id",zsDict._id), new BasicDBObject('$set',udata))
}
}
}
println "[${fjDicts.size()}]个字典数据同步完成"
println "[${fjDicts.size()}]个字典数据同步完成"
}
def synchDistrict(cat){
def fangjiaCitys = CommonDatautils.findAllCity()
@@ -259,18 +271,25 @@ class BasicInfoDataSynch extends BaseBatch{
udata.put("township",fjDistrict.township)
udata.put("alias",fjDistrict.alias)
udata.put("cat",cat)
udata.put("update_date",new Date().getTime())
if(fjDistrict.addr_alias){
udata.put("addr_alias",fjDistrict.addr_alias)
}
udata.put("status",0)
def map = new BasicDBObject()
map.put("lat",fjDistrict.lat/10000000000.0d)
map.put("lon", fjDistrict.lng/10000000000.0d)
def map = null;
if(fjDistrict && fjDistrict.lat){
map = new BasicDBObject()
map.put("lat",fjDistrict.lat/10000000000.0d)
map.put("lon", fjDistrict.lng/10000000000.0d)
}
if(zsDistrictObj){
if(fjDistrict.city!=zsDistrictObj.city||fjDistrict.region!=zsDistrictObj.region||fjDistrict.block!=zsDistrictObj.block||fjDistrict.name!=zsDistrictObj.name||fjDistrict.address!=zsDistrictObj.address||fjDistrict.estate_type2!=zsDistrictObj.estate_type||fjDistrict.township!=zsDistrictObj.township||fjDistrict.alias!=zsDistrictObj.alias||fjDistrict.addr_alias!=zsDistrictObj.addr_alias||map!=zsDistrictObj.location||zsDistrictObj.status!=0){
udata.put("location", map)
ZS_SEAWEED_CONN.update(new BasicDBObject("_id",zsDistrictObj._id), new BasicDBObject('$set',udata))
if(map){
udata.put("location", map)
ZS_SEAWEED_CONN.update(new BasicDBObject("_id",zsDistrictObj._id), new BasicDBObject('$set',udata))
}
}
}else{
udata.put("location", map)
@@ -295,10 +314,11 @@ class BasicInfoDataSynch extends BaseBatch{
if(!fjDistrict){
def udata = new BasicDBObject()
udata.put("status",1)
udata.put("update_date",new Date().getTime())
ZS_SEAWEED_CONN.update(new BasicDBObject("_id",fjDistrict._id), new BasicDBObject('$set',udata))
}
}
}
println "[${fjCity}-${cat}][${districts.size()}]个小区数据同步完成"
println "[${fjCity}-${cat}][${districts.size()}]个小区数据同步完成"
}
}
@@ -9,6 +9,9 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 浙商本地化部署同步脚本
*/
@BatchOwner(Watcher.GaoJuan)
class DistrictLevelBatch extends BaseBatch{
@@ -28,7 +31,7 @@ class DistrictLevelBatch extends BaseBatch{
def datas = addPriceData(dists, priceMap);
datas = calcuteLevel(datas);
saveOrUpdateDB(city, datas);
println city +" Â¥ÅÌÆÀ¼¶Í¬²½Íê³É"
println city +" 楼盘评级同步完成"
}
}
@@ -109,6 +112,7 @@ class DistrictLevelBatch extends BaseBatch{
udata.put("level", d.level)
udata.put("avg_price", d.avg_price)
udata.put("status",0)
udata.put("update_date",new Date().getTime())
if(zsDistLevel){
if(zsDistLevel.city!=d.city||zsDistLevel.region!=d.region||zsDistLevel.name!=d.name||zsDistLevel.level!=d.level||zsDistLevel.avg_price!=d.avg_price||zsDistLevel.status!=0){
ZS_LEVEL_CONN.update(new BasicDBObject("_id",zsDistLevel._id), new BasicDBObject('$set',udata))
@@ -127,10 +131,11 @@ class DistrictLevelBatch extends BaseBatch{
}
if(zsDistLevels.size()>0){
for(def zs : zsDistLevels){
def newData = newDataMap.get(zs._id.toString());
def newData = newDataMap.get(zs._id.toString());
if(!newData){
def udata = new BasicDBObject()
udata.put("status",1)
udata.put("update_date",new Date().getTime())
ZS_LEVEL_CONN.update(new BasicDBObject("_id",zs._id), new BasicDBObject('$set',udata))
}
}