Merge remote-tracking branch 'origin/master'
This commit is contained in:
@@ -100,11 +100,7 @@
|
||||
</pluginManagement>
|
||||
</build>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.fangjia.cloud.api.client</groupId>
|
||||
<artifactId>FangJiaApiSwaggerClient</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
|
||||
+2
-1
@@ -61,7 +61,8 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
|
||||
car_site :'carSite',estate_charge :'estateCharge',floorplan_pics :'floorplanPics',out_pics :'outPics',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
|
||||
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',licence :'licence',build_structure :'buildStructure',
|
||||
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure'
|
||||
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
|
||||
alias:'alias',addr_alias:'addrAlias'
|
||||
]
|
||||
public static final def OFFICE_FIELDS_MAP = [
|
||||
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',
|
||||
|
||||
+2
-2
@@ -31,7 +31,7 @@ class LandDetailCrawl {
|
||||
def task = new LandDetailCrawl();
|
||||
// task.run()
|
||||
task.updateData();
|
||||
task.undateOldData();
|
||||
// task.undateOldData();
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -65,7 +65,7 @@ class LandDetailCrawl {
|
||||
}
|
||||
|
||||
def updateData(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\友达\\土地--12月.xlsx",0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\土地\\土地成交3月(2).xlsx",0, 1)
|
||||
if(datas && datas.size()>0){
|
||||
for(def data : datas){
|
||||
def id = data['A']
|
||||
|
||||
@@ -44,7 +44,7 @@ class OpenApiUtils {
|
||||
|
||||
|
||||
static def matchDistrict2(def city, def address, def preciseLevelflag){
|
||||
address = address.replace("(","(").replace(")",")").replace(" ","")
|
||||
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","")
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||
if(preciseLevelflag){
|
||||
url = url +"&preciseLevel=strict"
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.client.MongoCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.collections.CollectionUtils
|
||||
import org.apache.commons.lang.math.RandomUtils
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.bson.Document
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 抓取区域边框
|
||||
*/
|
||||
class RegionPointCrawl {
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
|
||||
|
||||
static main(args) {
|
||||
def task = new RegionPointCrawl();
|
||||
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
|
||||
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
|
||||
def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||
for(def p : provinces){
|
||||
task.crawlCityAndDistrictCode(p);
|
||||
task.run(p);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def crawlCityAndDistrictCode(def province){
|
||||
crawlCity(province)
|
||||
}
|
||||
|
||||
def run(city){
|
||||
def cityCodes = findCityCode(city);
|
||||
if(cityCodes){
|
||||
for(def cityMap : cityCodes){
|
||||
def codeMap = findAllDistrictCode(cityMap.city_code);
|
||||
def index = 0
|
||||
for(def m : codeMap){
|
||||
index ++
|
||||
def ret = polygon(m.ad_code, "0", "all");
|
||||
if(!ret){
|
||||
println m.name +" " + m.ad_code + "无结果"
|
||||
}
|
||||
println index +" "+cityMap.name+" "+m.name+" "+ret.size()
|
||||
if(ret.size()>0){
|
||||
saveData(cityMap.name, m.name, m.ad_code , ret);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def crawlCity(def province){
|
||||
def ret = polygon(province, "2", "base");
|
||||
if(ret.size()>0){
|
||||
for(def map : ret){
|
||||
if(map.level == "province"){
|
||||
def cityList = map.districts
|
||||
if(cityList && cityList.size()>0){
|
||||
for(def cityMap : cityList){
|
||||
if(cityMap.level == "city"){
|
||||
saveGaoDeCityRegionCode(province, cityMap.name, cityMap.citycode, cityMap.adcode, cityMap.center, cityMap.level);
|
||||
saveGaoDeRegionCode(province, cityMap.districts);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def saveGaoDeRegionCode(province, districts){
|
||||
if(districts){
|
||||
for(def dist : districts){
|
||||
if(dist.level=="district"){
|
||||
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findCityCode(city){
|
||||
Pattern pattern = Pattern.compile(city, Pattern.CASE_INSENSITIVE);
|
||||
def q = null;
|
||||
if(city.contains("省") || city.endsWith("自治区")){
|
||||
q = new BasicDBObject("province", pattern);
|
||||
}else{
|
||||
q = new BasicDBObject("name", pattern);
|
||||
}
|
||||
q.put("level", "city")
|
||||
def dbcursor = GAODECITYCODECOLL.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
def city_code = d.get("city_code")
|
||||
if(!rets.contains(city_code)){
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
private BasicDBObject getLikeStr(String findStr) {
|
||||
Pattern pattern = Pattern.compile(findStr + ".*\$", Pattern.MULTILINE);
|
||||
return new BasicDBObject("$regex", pattern);
|
||||
}
|
||||
|
||||
def findAllDistrictCode(def citycode){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_code", citycode)
|
||||
q.put("level", "district")
|
||||
def dbcursor = GAODECITYCODECOLL.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def saveGaoDeCityRegionCode(province, name, citycode, adcode, center, level){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_code", citycode)
|
||||
q.put("province", province)
|
||||
q.put("level", level)
|
||||
q.put("name", name)
|
||||
if(adcode){
|
||||
q.put("ad_code", adcode)
|
||||
}
|
||||
|
||||
def o = GAODECITYCODECOLL.findOne(q)
|
||||
if(!o){
|
||||
q.put("c_date", new Date())
|
||||
q.put("center", center)
|
||||
GAODECITYCODECOLL.save(q);
|
||||
}else{
|
||||
println name +" "+adcode+" 已存在"
|
||||
}
|
||||
}
|
||||
|
||||
def saveData(city, region, code, ret){
|
||||
def q = new BasicDBObject()
|
||||
q.put("code", code)
|
||||
def dbo = POLYLINECOLL.findOne(q)
|
||||
if(!dbo){
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", city)
|
||||
o.put("region", region)
|
||||
o.put("code", code)
|
||||
o.put("center", ret.center.get(0))
|
||||
o.put("level", ret.level.get(0))
|
||||
o.put("polyline", ret.polyline.get(0));
|
||||
POLYLINECOLL.save(o);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static def polygon(def keyword, def subdistrict, def extensions) {
|
||||
String url = "https://restapi.amap.com/v3/config/district";
|
||||
|
||||
def params = [:]
|
||||
params.put("keywords", keyword)
|
||||
params.put("subdistrict",subdistrict)
|
||||
params.put("extensions",extensions)
|
||||
|
||||
String html = null;
|
||||
def jo = null;
|
||||
int errorCount = 0;
|
||||
String status = "error"
|
||||
def resultMap = [:]
|
||||
try {
|
||||
HttpRequest http = new HttpRequest();
|
||||
while (errorCount<5){//"2af0b8533415a612b98d59bb93d664b9" 公司key
|
||||
def key = getRandomKey2() //
|
||||
params.put("key", key)
|
||||
html = http.get(url, params,"utf-8")
|
||||
if (StringUtils.isNotEmpty(html)) {
|
||||
jo = JSONObject.fromObject(html)
|
||||
if(jo.infocode!="10000"){
|
||||
errorCount ++
|
||||
Thread.sleep(30*1000)
|
||||
// println " 不可用Key: "+key
|
||||
}else{
|
||||
errorCount = 100
|
||||
status = "normal"
|
||||
// println " 可用Key: "+key + " "+html
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
return jo.districts;
|
||||
}
|
||||
|
||||
static String getRandomKey2() {
|
||||
def KeyList = [
|
||||
"2af0b8533415a612b98d59bb93d664b9",
|
||||
"0246c57bbcf24aa246c207b3c99dfe41",
|
||||
"042328b3f0fa5daf426802d379521d87",
|
||||
"055d4f1b5b64a2bce18f4fe1f15c2639",
|
||||
"d5484f5170502405d88bbaffe1b496f0",
|
||||
"e74ccc4258079d2200199695110cceb1",
|
||||
"f58629fc799fbc8537ca85046c6d44a5",
|
||||
"291073f17ee0b963ccb1927ed92bf265",
|
||||
"2a1fb75990f2b39f6221e50892dd3988"
|
||||
]
|
||||
|
||||
def count = RandomUtils.nextInt(KeyList.size())
|
||||
return KeyList.get(count);
|
||||
}
|
||||
}
|
||||
H
|
||||
+2
-2
@@ -47,7 +47,7 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2020-11-01"
|
||||
static def statDate = "2021-02-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2020年11月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2021年3月\\二手房\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+25
-1
@@ -60,9 +60,14 @@ class HouseExport {
|
||||
static main(args) {
|
||||
def task = new HouseExport();
|
||||
|
||||
task.run()
|
||||
// task.run()
|
||||
// task.marge()
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
// "重庆",
|
||||
def citys = ["北京"]
|
||||
@@ -777,4 +782,23 @@ def GenNonDuplicateID(randomLength){
|
||||
|
||||
|
||||
|
||||
def marge(){
|
||||
File file = new File("D:\\数据\\建行\\新建文本文档.txt");
|
||||
def ids = new HashSet();
|
||||
def index = 0
|
||||
|
||||
file.eachLine{it->
|
||||
index++
|
||||
if(index>1){
|
||||
def arrs = it.split("\\|@\\|")
|
||||
def id = arrs[1]
|
||||
if(!ids.contains(id)){
|
||||
ids << id
|
||||
}
|
||||
}
|
||||
}
|
||||
println index
|
||||
println ids.size()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class FieldStat {
|
||||
|
||||
static main(args) {
|
||||
def task = new FieldStat();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区融合结果概况2.xlsx", { row ->
|
||||
def city = row['A']
|
||||
if(city){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]);
|
||||
def total = districts.size()
|
||||
|
||||
def aliasIndex = 0
|
||||
def aliasTotalCount = 0
|
||||
for(def d : districts){
|
||||
def alias = d.alias
|
||||
if(alias){
|
||||
aliasIndex++
|
||||
aliasTotalCount = aliasTotalCount + alias.size()
|
||||
}
|
||||
}
|
||||
row['J'] = aliasTotalCount
|
||||
row['K'] = aliasIndex
|
||||
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
|
||||
row['L'] = p+"%"
|
||||
row['M'] = total
|
||||
println city +" "+p
|
||||
}
|
||||
|
||||
});
|
||||
println "finish"
|
||||
}
|
||||
}
|
||||
+34
-1
@@ -23,7 +23,9 @@ class AnjukeCrawlToDB {
|
||||
|
||||
// task.run()
|
||||
|
||||
task.stat()
|
||||
// task.stat()
|
||||
|
||||
task.addAliasStat()
|
||||
}
|
||||
|
||||
def stat(){
|
||||
@@ -77,6 +79,7 @@ class AnjukeCrawlToDB {
|
||||
}
|
||||
}
|
||||
|
||||
println estate_chargeCount
|
||||
|
||||
def estate_charge_pre = DataHandleUtils.getDoubleRound(2, estate_chargeCount*100/total)
|
||||
def build_area_pre = DataHandleUtils.getDoubleRound(2, build_areaCount*100/total)
|
||||
@@ -104,6 +107,36 @@ class AnjukeCrawlToDB {
|
||||
|
||||
}
|
||||
|
||||
def addAliasStat(){
|
||||
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\城市覆盖率统计(2021-03-14) (1).xls",{row->
|
||||
def city = row['B']
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["alias", "addr_alias"]);
|
||||
|
||||
def total = districts.size()
|
||||
|
||||
def aliasIndex = 0
|
||||
def addrAliasIndex = 0
|
||||
for(def d : districts){
|
||||
def alias = d.alias
|
||||
if(alias){
|
||||
aliasIndex++
|
||||
}
|
||||
|
||||
def addrAlias = d.addr_alias
|
||||
if(addrAlias){
|
||||
addrAliasIndex ++
|
||||
}
|
||||
}
|
||||
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
|
||||
row['BA'] = p
|
||||
|
||||
def p1 = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, addrAliasIndex*100/total)
|
||||
row['BB'] = p1
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+292
@@ -0,0 +1,292 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class JiangShuBlankDataStat {
|
||||
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new JiangShuBlankDataStat()
|
||||
// task.run()
|
||||
// task.matchCredit();
|
||||
// task.repeat();
|
||||
// task.matchHouse();
|
||||
|
||||
// task.villa()
|
||||
task.addressParse()
|
||||
}
|
||||
|
||||
def repeat(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\江苏农行1484条.xlsx", 0,1)
|
||||
def keys = []
|
||||
for(def data : datas){
|
||||
def key = data['W']+"##"+data['X']+"##"+data['Y']
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
}
|
||||
}
|
||||
println keys.size();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def esDatas = findLogs();
|
||||
def keys = []
|
||||
def datas = []
|
||||
for(def d : esDatas){
|
||||
def key = d.fjw_city +"##"+d.fjw_region+" "+d.fjw_name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
datas << [d.parameter_city, d.parameter_region, d.parameter_address, d.parameter_size, d.fjw_city, d.fjw_region, d.fjw_name]
|
||||
}else{
|
||||
println key
|
||||
}
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\江苏农行\\全数据.xlsx", datas, "数据")
|
||||
|
||||
// ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\江苏农行930条.xlsx",{row->
|
||||
//// def address = row['G']
|
||||
//// if(!row['X']){
|
||||
//// def area = row['H']
|
||||
//// def retMap = findRow(address, area, esDatas);
|
||||
//// if(retMap){
|
||||
//// row['V'] = retMap.fjw_city
|
||||
//// row['W'] = retMap.fjw_region
|
||||
//// row['X'] = retMap.fjw_name
|
||||
//// row['Y'] = retMap.fjw_avgPrice
|
||||
//// row['Z'] = retMap.fjw_totalPrice
|
||||
////
|
||||
//// row['AG'] = retMap.parameter_city
|
||||
//// row['AH'] = retMap.parameter_region
|
||||
//// }
|
||||
// def inputRegion = row['X']
|
||||
// def matchRegion = row['M']
|
||||
// if( inputRegion&& matchRegion && inputRegion.contains(matchRegion)){
|
||||
// row['AA'] = "包含"
|
||||
// }
|
||||
//// def matchName = row['Y']
|
||||
//// if(matchName && address && address.contains(matchName)){
|
||||
//// row['AL'] = "包含名称"
|
||||
//// }
|
||||
//// }
|
||||
// },0)
|
||||
}
|
||||
|
||||
def matchCredit(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\全数据.xlsx",{row->
|
||||
def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G'], "district")
|
||||
if(obj){
|
||||
row['H'] = obj.estate_type2
|
||||
}
|
||||
// def dbo = CommonDatautils.queryDistrictPrice(row['E'], row['F'], row['G'], DateUtils.parseFromTightByPattern("2021-03-14","yyyy-MM-dd"))
|
||||
// if(dbo){
|
||||
// row['H'] = dbo.avg_price
|
||||
// row['I'] = dbo.credit
|
||||
// row['J'] = dbo.value_type
|
||||
// }
|
||||
//
|
||||
// def lianjiaObj = findThirdPrice(row['E'], row['F'], row['G'], "lianjia_esf_price");
|
||||
// if(lianjiaObj){
|
||||
// row['k'] = lianjiaObj.lianjia_esf_price
|
||||
// row['L'] = lianjiaObj.s_date
|
||||
// }
|
||||
// def beikeObj = findThirdPrice(row['E'], row['F'], row['G'], "beike_esf_price");
|
||||
// if(beikeObj){
|
||||
// row['M'] = beikeObj.beike_esf_price
|
||||
// row['N'] = beikeObj.s_date
|
||||
// }
|
||||
},0)
|
||||
}
|
||||
|
||||
def findThirdPrice(def city, def region, def name, def fieldName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1))
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d;
|
||||
break;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
def findRow(address, area, esDatas){
|
||||
def retMap = [:]
|
||||
for(def d : esDatas){
|
||||
|
||||
if(d.parameter_address ==address && area==d.parameter_size.toDouble()){
|
||||
retMap = d;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
def findLogs(){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007753')).must(QueryBuilders.termQuery("request_uri", "/property/addressEvaluate"))
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
|
||||
def rets = []
|
||||
def infos = []
|
||||
if(houses){
|
||||
StringBuilder sbu=new StringBuilder();
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
println k
|
||||
def map = houses[k];
|
||||
println map
|
||||
if(map.parameter_map && map.res_map.result && map.code==200){
|
||||
def pMap = map.parameter_map;
|
||||
def rMap = map.res_map.result
|
||||
|
||||
|
||||
rets << ["parameter_city":pMap.city, "parameter_region":pMap.district, "parameter_address":pMap.address, "parameter_size":pMap.size,"fjw_city":rMap.city, "fjw_region":rMap.district, "fjw_name":rMap.name,"fjw_avgPrice":rMap.avgPrice, "fjw_totalPrice":rMap.totalPrice]
|
||||
|
||||
sbu.append(pMap.address+"\n");
|
||||
}
|
||||
}
|
||||
//FileUtils.writeStringToFile(new File("D:\\数据\\江苏农行\\log.txt"), sbu.toString(), "GBK")
|
||||
}
|
||||
|
||||
return rets;
|
||||
}
|
||||
|
||||
def matchHouse(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\全数据.xlsx",{row->
|
||||
def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G'], "district")
|
||||
def mod = row['D'].toDouble()*0.2
|
||||
def startArea = row['D'].toDouble()- mod
|
||||
def endArea = row['D'].toDouble() +mod
|
||||
def prices = findHouses(row['E'], row['F'], "sell", startArea, endArea, row['G'], obj.alias);
|
||||
if(!prices){
|
||||
if(row['D'].toDouble()<=100){
|
||||
prices = findHouses(row['E'], row['F'], "sell", 30, 100, row['G'], obj.alias);
|
||||
}else if(row['D'].toDouble()<=200){
|
||||
prices = findHouses(row['E'], row['F'], "sell", 100, 200, row['G'], obj.alias);
|
||||
}else{
|
||||
prices = findHouses(row['E'], row['F'], "sell", 200, 800, row['G'], obj.alias);
|
||||
}
|
||||
}
|
||||
def avg_price = null
|
||||
if(prices){
|
||||
avg_price = getAvgPrice(prices);
|
||||
}
|
||||
if(avg_price){
|
||||
row['Q'] = avg_price.toInteger()
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
def getAvgPrice(def prices){
|
||||
def avg_price = 0
|
||||
if(prices){
|
||||
def sum = prices.sum();
|
||||
avg_price = sum/prices.size()
|
||||
}
|
||||
return avg_price
|
||||
}
|
||||
|
||||
def findHouses(def city, def region, def cat, def startArea, def endArea, def name, def alias) {
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(20200301).to(20210318).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(startArea).to(endArea).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if (!alias) {
|
||||
alias = []
|
||||
}
|
||||
if (!alias.contains(name)) {
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
if (alias && alias.size() > 0) {
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for (def s : alias) {
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
inBuilder.should(QueryBuilders.matchQuery("root", "lianjia.com"));
|
||||
inBuilder.should(QueryBuilders.matchQuery("root", "ke.com"));
|
||||
queryBuilder.must(inBuilder)
|
||||
|
||||
def prices = []
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
if (houses) {
|
||||
for (int k = 0; k < houses.size(); k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
prices << seawater.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
println city +" "+region+" "+name+" "+ prices.size()
|
||||
return prices;
|
||||
}
|
||||
|
||||
def villa(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以上.xlsx", { row ->
|
||||
def obj = CommonDatautils.findDistrict(row['E'], row['F'], row['G']+"别墅", "district")
|
||||
if(obj){
|
||||
def dbo = CommonDatautils.queryDistrictPrice(obj.city, obj.region, obj.name, DateUtils.parseFromTightByPattern("2021-03-14","yyyy-MM-dd"))
|
||||
if(dbo){
|
||||
row['R'] = obj.name
|
||||
row['S'] = dbo.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
def addressParse(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以上.xlsx", { row ->
|
||||
def resp = OpenApiUtils.matchDistrict2(row['E'], row['C'], true)
|
||||
if(resp){
|
||||
println resp
|
||||
row['T'] = resp.roomNumber
|
||||
row['U'] = getFloor(resp.roomNumber);
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
def getFloor(def s){
|
||||
def floor = null
|
||||
if(s){
|
||||
Pattern pa=Pattern.compile("^(\\d{1,2})0\\d{1}")
|
||||
Matcher ma=pa.matcher(s)
|
||||
if(ma.find()){
|
||||
floor = ma.group(1)
|
||||
} else {
|
||||
floor = null
|
||||
}
|
||||
}
|
||||
return floor;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+111
-106
@@ -630,128 +630,133 @@ class CZBlankDataExport {
|
||||
sqlBuffer.append(" and m_date >='"+startDate+"'").append(" and m_date <'"+endDate+"'");
|
||||
// sqlBuffer.append(" and m_date >='"+pDate+"'").append(" and m_date <='"+pDate+"'");
|
||||
sqlBuffer.append(" ORDER BY p_date desc limit 200")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
|
||||
ResultSet rs = null
|
||||
def index = 0;
|
||||
def houses = [];
|
||||
while (rs.next()) {
|
||||
try{
|
||||
rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
|
||||
def area = rs.getObject("area")
|
||||
while (rs.next()) {
|
||||
|
||||
|
||||
|
||||
if(area && (area>1000||area<30)){
|
||||
continue;
|
||||
}
|
||||
|
||||
def floor = rs.getObject("floor")
|
||||
if(floor && floor>100){
|
||||
continue;
|
||||
}
|
||||
|
||||
def height = rs.getObject("height")
|
||||
if(height && (height>100||height==0)){
|
||||
continue;
|
||||
}
|
||||
|
||||
def total_price = rs.getObject("total_price")
|
||||
def house_area = area;
|
||||
def avg_price = total_price/house_area;
|
||||
|
||||
def distPrice = distPriceMaps.get(dbo.city +"##"+dbo.region+"##"+dbo.name);
|
||||
if(distPrice){
|
||||
def maxValue = distPrice + distPrice*0.15;
|
||||
def minValue = distPrice - distPrice*0.15;
|
||||
if(avg_price>maxValue.toInteger() || avg_price<minValue.toInteger()){
|
||||
def area = rs.getObject("area")
|
||||
if(area && (area>1000||area<30)){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
def district_name = rs.getObject("district_name")
|
||||
if(district_name.endsWith("别墅") && height && height>=10){
|
||||
continue;
|
||||
}
|
||||
def floor = rs.getObject("floor")
|
||||
if(floor && floor>100){
|
||||
continue;
|
||||
}
|
||||
|
||||
def m = [:];
|
||||
//小区名称
|
||||
m.name = dbo.name;
|
||||
if(dbo.alias){
|
||||
def newAlias = [];
|
||||
for(def tname : dbo.alias){
|
||||
if(tname && tname.length()>=2){
|
||||
newAlias << tname;
|
||||
def height = rs.getObject("height")
|
||||
if(height && (height>100||height==0)){
|
||||
continue;
|
||||
}
|
||||
|
||||
def total_price = rs.getObject("total_price")
|
||||
def house_area = area;
|
||||
def avg_price = total_price/house_area;
|
||||
|
||||
def distPrice = distPriceMaps.get(dbo.city +"##"+dbo.region+"##"+dbo.name);
|
||||
if(distPrice){
|
||||
def maxValue = distPrice + distPrice*0.15;
|
||||
def minValue = distPrice - distPrice*0.15;
|
||||
if(avg_price>maxValue.toInteger() || avg_price<minValue.toInteger()){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if(newAlias){
|
||||
m.all_name = dbo.name + "("+newAlias.join(",")+")"
|
||||
|
||||
def district_name = rs.getObject("district_name")
|
||||
if(district_name.endsWith("别墅") && height && height>=10){
|
||||
continue;
|
||||
}
|
||||
|
||||
def m = [:];
|
||||
//小区名称
|
||||
m.name = dbo.name;
|
||||
if(dbo.alias){
|
||||
def newAlias = [];
|
||||
for(def tname : dbo.alias){
|
||||
if(tname && tname.length()>=2){
|
||||
newAlias << tname;
|
||||
}
|
||||
}
|
||||
if(newAlias){
|
||||
m.all_name = dbo.name + "("+newAlias.join(",")+")"
|
||||
}else{
|
||||
m.all_name = dbo.name;
|
||||
}
|
||||
}else{
|
||||
m.all_name = dbo.name;
|
||||
}
|
||||
}else{
|
||||
m.all_name = dbo.name;
|
||||
m.city = rs.getObject("city");
|
||||
m.region = rs.getObject("region");
|
||||
m.district_id= dbo.num_id;
|
||||
|
||||
m.house_id = createHouseId(rs.getObject("url"));//rs.getObject("_id").toString();
|
||||
//建筑物总楼层
|
||||
m.height = height>0?height:null;
|
||||
//所在楼层
|
||||
m.floor = getHouseFloor(floor, m.height);
|
||||
//建筑面积
|
||||
m.house_area = area;
|
||||
//挂牌价
|
||||
m.total_price = total_price;
|
||||
|
||||
def keys = m.region +"-"+ m.name+"-"+m.house_area+"-"+m.height+"-"+m.floor+"-"+m.total_price;
|
||||
if(!houses.contains(keys)){
|
||||
houses << keys;
|
||||
}else{
|
||||
println "此房源已存在....."
|
||||
continue;
|
||||
}
|
||||
|
||||
if(index > 30){
|
||||
break;
|
||||
}
|
||||
m.room = rs.getObject("room");
|
||||
m.hall = rs.getObject("hall");
|
||||
m.estate_type = rs.getObject("estate_type");
|
||||
//建成年代
|
||||
m.build_date = getBuildDate(rs.getObject("build_date"), dbo);
|
||||
//总使用年限
|
||||
if(districtIds.contains(m.district_id)){
|
||||
m.property_life = 70;
|
||||
}else{
|
||||
m.property_life = getPropertyLife(dbo.property_life);
|
||||
}
|
||||
//朝向
|
||||
m.direction = getDirectionCode(rs.getObject("direction"), dbo.estate_type2);
|
||||
//建筑结构
|
||||
m.build_structure_code = getBuildStructureCode(dbo.build_date, dbo.build_structure);
|
||||
//建筑类别
|
||||
m.build_type_code = getBuildTypeCode(dbo.name, dbo.build_type, height);
|
||||
if(m.build_type_code=='3' && m.height>6){
|
||||
m.height = null
|
||||
}
|
||||
|
||||
//内部装修与成新度
|
||||
m.fitment_code = getFitmentCode(rs.getObject("fitment"));
|
||||
//是否有电梯
|
||||
m.elevator_code = getElevatorCode(height, m.build_type_code);
|
||||
//房价网评估价
|
||||
//m.evaluation_price = getEvaluationPrice(m);
|
||||
//println "[${index++}] "+new Date();
|
||||
//评估时间
|
||||
m.evaluation_date = DateUtils.formatDateByPattern(new Date(),"yyyyMMdd");
|
||||
//挂牌时间
|
||||
m.publish_date = rs.getObject("p_date").toString()//DateUtils.formatDateByPattern(obj.publish_date,"yyyyMMdd");
|
||||
|
||||
m.status=1;
|
||||
|
||||
index++;
|
||||
saveData(m);
|
||||
}
|
||||
m.city = rs.getObject("city");
|
||||
m.region = rs.getObject("region");
|
||||
m.district_id= dbo.num_id;
|
||||
}catch(Exception e){
|
||||
|
||||
m.house_id = createHouseId(rs.getObject("url"));//rs.getObject("_id").toString();
|
||||
//建筑物总楼层
|
||||
m.height = height>0?height:null;
|
||||
//所在楼层
|
||||
m.floor = getHouseFloor(floor, m.height);
|
||||
//建筑面积
|
||||
m.house_area = area;
|
||||
//挂牌价
|
||||
m.total_price = total_price;
|
||||
|
||||
def keys = m.region +"-"+ m.name+"-"+m.house_area+"-"+m.height+"-"+m.floor+"-"+m.total_price;
|
||||
if(!houses.contains(keys)){
|
||||
houses << keys;
|
||||
}else{
|
||||
println "此房源已存在....."
|
||||
continue;
|
||||
}
|
||||
|
||||
if(index > 30){
|
||||
break;
|
||||
}
|
||||
m.room = rs.getObject("room");
|
||||
m.hall = rs.getObject("hall");
|
||||
m.estate_type = rs.getObject("estate_type");
|
||||
//建成年代
|
||||
m.build_date = getBuildDate(rs.getObject("build_date"), dbo);
|
||||
//总使用年限
|
||||
if(districtIds.contains(m.district_id)){
|
||||
m.property_life = 70;
|
||||
}else{
|
||||
m.property_life = getPropertyLife(dbo.property_life);
|
||||
}
|
||||
//朝向
|
||||
m.direction = getDirectionCode(rs.getObject("direction"), dbo.estate_type2);
|
||||
//建筑结构
|
||||
m.build_structure_code = getBuildStructureCode(dbo.build_date, dbo.build_structure);
|
||||
//建筑类别
|
||||
m.build_type_code = getBuildTypeCode(dbo.name, dbo.build_type, height);
|
||||
if(m.build_type_code=='3' && m.height>6){
|
||||
m.height = null
|
||||
}
|
||||
|
||||
//内部装修与成新度
|
||||
m.fitment_code = getFitmentCode(rs.getObject("fitment"));
|
||||
//是否有电梯
|
||||
m.elevator_code = getElevatorCode(height, m.build_type_code);
|
||||
//房价网评估价
|
||||
//m.evaluation_price = getEvaluationPrice(m);
|
||||
//println "[${index++}] "+new Date();
|
||||
//评估时间
|
||||
m.evaluation_date = DateUtils.formatDateByPattern(new Date(),"yyyyMMdd");
|
||||
//挂牌时间
|
||||
m.publish_date = rs.getObject("p_date").toString()//DateUtils.formatDateByPattern(obj.publish_date,"yyyyMMdd");
|
||||
|
||||
m.status=1;
|
||||
|
||||
index++;
|
||||
saveData(m);
|
||||
}
|
||||
|
||||
return index;
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202010);
|
||||
def count2 = findTotalCount(source, 202011);
|
||||
def count1 = findTotalCount(source, 202101);
|
||||
def count2 = findTotalCount(source, 202102);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+41
-21
@@ -4,11 +4,14 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
/**
|
||||
* 浙商本地化部署同步脚本
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BasicInfoDataSynch extends BaseBatch{
|
||||
// final static DBCollection SERVER_AREA_CONN = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo)
|
||||
@@ -29,10 +32,10 @@ class BasicInfoDataSynch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
// synchCitys();
|
||||
// synchRegion()
|
||||
// synchTown()
|
||||
// synchDict()
|
||||
synchCitys();
|
||||
synchRegion()
|
||||
synchTown()
|
||||
synchDict()
|
||||
synchDistrict("district")
|
||||
synchDistrict("office")
|
||||
}
|
||||
@@ -47,46 +50,49 @@ class BasicInfoDataSynch extends BaseBatch{
|
||||
zsCitysMap.put(zsCitys._id.toString(), zsCitys)
|
||||
zsDatas << zsCitys
|
||||
}
|
||||
//数据存在则修改,不存在则保存
|
||||
//数据存在则修改,不存在则保存
|
||||
if(fangjiaCitys){
|
||||
for(def fjCity : fangjiaCitys){
|
||||
if(fjCity.display!="全国"){
|
||||
if(fjCity.display!="全国"){
|
||||
fjCitysMap.put(fjCity._id.toString(), fjCity)
|
||||
}
|
||||
def zsCity = zsCitysMap.get(fjCity._id.toString())
|
||||
if(zsCity){//修改
|
||||
if(zsCity){//修改
|
||||
if(zsCity.province != fjCity.province || zsCity.alias!=fjCity.alias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province",fjCity.province)
|
||||
udata.put("alias",fjCity.alias)
|
||||
udata.put("status",0)
|
||||
udata.put("update_date",new Date().getTime())
|
||||
ZS_CITYS_CONN.update(new BasicDBObject("_id",zsCity._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}else{//保存
|
||||
def addUata = new BasicDBObject()
|
||||
}else{//保存
|
||||
def addUata = new BasicDBObject()
|
||||
addUata.put("_id",fjCity._id)
|
||||
addUata.put("display",fjCity.display)
|
||||
addUata.put("province",fjCity.province)
|
||||
addUata.put("alias",fjCity.alias)
|
||||
addUata.put("status",0)
|
||||
addUata.put("update_date",new Date().getTime())
|
||||
ZS_CITYS_CONN.save(addUata);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//老数据不在新数据中,被删除时,状态修改为1
|
||||
//老数据不在新数据中,被删除时,状态修改为1
|
||||
if(zsDatas){
|
||||
for(def zsCity : zsDatas){
|
||||
def fjCity = fjCitysMap.get(zsCity._id.toString())
|
||||
if(!fjCity){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status",1)
|
||||
udata.put("update_date",new Date().getTime())
|
||||
ZS_CITYS_CONN.update(new BasicDBObject("_id",zsCity._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "[${fangjiaCitys.size()}]个城市同步完成"
|
||||
println "[${fangjiaCitys.size()}]个城市同步完成"
|
||||
|
||||
}
|
||||
def synchRegion(){
|
||||
@@ -110,6 +116,7 @@ class BasicInfoDataSynch extends BaseBatch{
|
||||
udata.put("pattern",fjRegion.pattern)
|
||||
udata.put("org_name",fjRegion.org_name)
|
||||
udata.put("status",0)
|
||||
udata.put("update_date",new Date().getTime())
|
||||
if(zsRegionObj){
|
||||
ZS_REGION_CONN.update(new BasicDBObject("_id",zsRegionObj._id), new BasicDBObject('$set',udata))
|
||||
}else{
|
||||
@@ -125,11 +132,12 @@ class BasicInfoDataSynch extends BaseBatch{
|
||||
if(!fjRegion){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status",1)
|
||||
udata.put("update_date",new Date().getTime())
|
||||
ZS_REGION_CONN.update(new BasicDBObject("_id",zsRegion._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
println "[${fjRegions.size()}]个行政区域同步完成"
|
||||
println "[${fjRegions.size()}]个行政区域同步完成"
|
||||
}
|
||||
|
||||
def synchTown(){
|
||||
@@ -152,6 +160,7 @@ class BasicInfoDataSynch extends BaseBatch{
|
||||
udata.put("short_name",fjTown.short_name)
|
||||
udata.put("name",fjTown.name)
|
||||
udata.put("status",0)
|
||||
udata.put("update_date",new Date().getTime())
|
||||
if(zsTownObj){
|
||||
if(fjTown.city!=zsTownObj.city||fjTown.region!=zsTownObj.region||fjTown.short_name!=zsTownObj.short_name||fjTown.name!=zsTownObj.name||zsTownObj.status!=0){
|
||||
ZS_TOWN_CONN.update(new BasicDBObject("_id",zsTownObj._id), new BasicDBObject('$set',udata))
|
||||
@@ -169,11 +178,12 @@ class BasicInfoDataSynch extends BaseBatch{
|
||||
if(!fjTown){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status",1)
|
||||
udata.put("update_date",new Date().getTime())
|
||||
ZS_TOWN_CONN.update(new BasicDBObject("_id",zsTown._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
println "[${fjTowns.size()}]个乡镇同步完成"
|
||||
println "[${fjTowns.size()}]个乡镇同步完成"
|
||||
}
|
||||
def synchDict(){
|
||||
def fjDicts = CommonDatautils.getAllDictionary()
|
||||
@@ -194,6 +204,7 @@ class BasicInfoDataSynch extends BaseBatch{
|
||||
udata.put("cat",fjDict.cat)
|
||||
udata.put("name",fjDict.name)
|
||||
udata.put("status",0)
|
||||
udata.put("update_date",new Date().getTime())
|
||||
if(zsDictObj){
|
||||
if(fjDict.cat!=zsDictObj.cat||fjDict.name!=zsDictObj.name||zsDictObj.status!=0){
|
||||
ZS_DICT_CONN.update(new BasicDBObject("_id",zsDictObj._id), new BasicDBObject('$set',udata))
|
||||
@@ -220,11 +231,12 @@ class BasicInfoDataSynch extends BaseBatch{
|
||||
if(!fjDict){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status",1)
|
||||
udata.put("update_date",new Date().getTime())
|
||||
ZS_DICT_CONN.update(new BasicDBObject("_id",zsDict._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
println "[${fjDicts.size()}]个字典数据同步完成"
|
||||
println "[${fjDicts.size()}]个字典数据同步完成"
|
||||
}
|
||||
def synchDistrict(cat){
|
||||
def fangjiaCitys = CommonDatautils.findAllCity()
|
||||
@@ -259,18 +271,25 @@ class BasicInfoDataSynch extends BaseBatch{
|
||||
udata.put("township",fjDistrict.township)
|
||||
udata.put("alias",fjDistrict.alias)
|
||||
udata.put("cat",cat)
|
||||
udata.put("update_date",new Date().getTime())
|
||||
if(fjDistrict.addr_alias){
|
||||
udata.put("addr_alias",fjDistrict.addr_alias)
|
||||
}
|
||||
|
||||
udata.put("status",0)
|
||||
def map = new BasicDBObject()
|
||||
map.put("lat",fjDistrict.lat/10000000000.0d)
|
||||
map.put("lon", fjDistrict.lng/10000000000.0d)
|
||||
def map = null;
|
||||
if(fjDistrict && fjDistrict.lat){
|
||||
map = new BasicDBObject()
|
||||
map.put("lat",fjDistrict.lat/10000000000.0d)
|
||||
map.put("lon", fjDistrict.lng/10000000000.0d)
|
||||
}
|
||||
|
||||
if(zsDistrictObj){
|
||||
if(fjDistrict.city!=zsDistrictObj.city||fjDistrict.region!=zsDistrictObj.region||fjDistrict.block!=zsDistrictObj.block||fjDistrict.name!=zsDistrictObj.name||fjDistrict.address!=zsDistrictObj.address||fjDistrict.estate_type2!=zsDistrictObj.estate_type||fjDistrict.township!=zsDistrictObj.township||fjDistrict.alias!=zsDistrictObj.alias||fjDistrict.addr_alias!=zsDistrictObj.addr_alias||map!=zsDistrictObj.location||zsDistrictObj.status!=0){
|
||||
udata.put("location", map)
|
||||
ZS_SEAWEED_CONN.update(new BasicDBObject("_id",zsDistrictObj._id), new BasicDBObject('$set',udata))
|
||||
if(map){
|
||||
udata.put("location", map)
|
||||
ZS_SEAWEED_CONN.update(new BasicDBObject("_id",zsDistrictObj._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}else{
|
||||
udata.put("location", map)
|
||||
@@ -295,10 +314,11 @@ class BasicInfoDataSynch extends BaseBatch{
|
||||
if(!fjDistrict){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status",1)
|
||||
udata.put("update_date",new Date().getTime())
|
||||
ZS_SEAWEED_CONN.update(new BasicDBObject("_id",fjDistrict._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
println "[${fjCity}-${cat}][${districts.size()}]个小区数据同步完成"
|
||||
println "[${fjCity}-${cat}][${districts.size()}]个小区数据同步完成"
|
||||
}
|
||||
}
|
||||
|
||||
+7
-2
@@ -9,6 +9,9 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 浙商本地化部署同步脚本
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class DistrictLevelBatch extends BaseBatch{
|
||||
|
||||
@@ -28,7 +31,7 @@ class DistrictLevelBatch extends BaseBatch{
|
||||
def datas = addPriceData(dists, priceMap);
|
||||
datas = calcuteLevel(datas);
|
||||
saveOrUpdateDB(city, datas);
|
||||
println city +" Â¥ÅÌÆÀ¼¶Í¬²½Íê³É"
|
||||
println city +" 楼盘评级同步完成"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -109,6 +112,7 @@ class DistrictLevelBatch extends BaseBatch{
|
||||
udata.put("level", d.level)
|
||||
udata.put("avg_price", d.avg_price)
|
||||
udata.put("status",0)
|
||||
udata.put("update_date",new Date().getTime())
|
||||
if(zsDistLevel){
|
||||
if(zsDistLevel.city!=d.city||zsDistLevel.region!=d.region||zsDistLevel.name!=d.name||zsDistLevel.level!=d.level||zsDistLevel.avg_price!=d.avg_price||zsDistLevel.status!=0){
|
||||
ZS_LEVEL_CONN.update(new BasicDBObject("_id",zsDistLevel._id), new BasicDBObject('$set',udata))
|
||||
@@ -127,10 +131,11 @@ class DistrictLevelBatch extends BaseBatch{
|
||||
}
|
||||
if(zsDistLevels.size()>0){
|
||||
for(def zs : zsDistLevels){
|
||||
def newData = newDataMap.get(zs._id.toString());
|
||||
def newData = newDataMap.get(zs._id.toString());
|
||||
if(!newData){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status",1)
|
||||
udata.put("update_date",new Date().getTime())
|
||||
ZS_LEVEL_CONN.update(new BasicDBObject("_id",zs._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user