Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj
This commit is contained in:
+91
@@ -832,6 +832,33 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def findNewUpdateDistricts(def city, def cat, def fields){
|
||||
|
||||
def startDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status",0)
|
||||
// q.put("status",new BasicDBObject('$ne', 0))
|
||||
q.put("cat", cat);
|
||||
q.put("city", city)
|
||||
q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
|
||||
// q.put("loop_line_info",new BasicDBObject('$exists',false))
|
||||
// q.put("name","扬州北街8号")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
public static def findAllRegionDistricts(def city, def region, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
@@ -5818,4 +5845,68 @@ class CommonDatautils {
|
||||
}
|
||||
return address;
|
||||
}
|
||||
|
||||
|
||||
static def findSeawater(city, lng, lat, distance, p_date, cat){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.206:9200/seawater-*/_search"
|
||||
def body = """
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"geo_distance": {
|
||||
"distance": "${distance}",
|
||||
"location": {
|
||||
"lat": ${lat},
|
||||
"lon": ${lng}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"term": {
|
||||
"city": "${city}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"term": {
|
||||
"cat": "${cat}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"range": {
|
||||
"publish_date": {
|
||||
"gte": "${p_date}",
|
||||
"format": "yyyy-MM-dd"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"sort": [
|
||||
{
|
||||
"publish_date": {
|
||||
"order": "desc"
|
||||
}
|
||||
}
|
||||
],
|
||||
"size":30
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = new JsonSlurper().parseText(res)
|
||||
|
||||
def esObj = null
|
||||
if(response && response.hits){
|
||||
esObj = response.hits.hits
|
||||
}
|
||||
return esObj
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+12
-10
@@ -1,21 +1,16 @@
|
||||
package com.fangjia.script.house
|
||||
import com.fangjia.core.util.DateUtil
|
||||
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
|
||||
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
@@ -48,14 +43,17 @@ class DistrictSellHouseStatBatch extends BaseBatch{
|
||||
for(def city : citys){
|
||||
index ++ ;
|
||||
def districts = CommonDatautils.getAllDistrictBaseInfo(city, "district")
|
||||
def sellRet = houseStatCount(districts, startDate, endDate, sDate);
|
||||
|
||||
String start = DateUtils.formatDateByPattern(startDate, "yyyyMMdd")
|
||||
|
||||
def sellRet = houseStatCount(districts, start.toInteger(), sDate);
|
||||
saveResult(city, sDate, sellRet);
|
||||
|
||||
println "[${city} ${index}/${citys.size()}]" +new Date().toString();
|
||||
}
|
||||
}
|
||||
|
||||
def houseStatCount(def districts, def startDate, def endDate, def sDate){
|
||||
def houseStatCount(def districts, def startDate, def sDate){
|
||||
|
||||
def index = 0;
|
||||
|
||||
@@ -75,8 +73,12 @@ class DistrictSellHouseStatBatch extends BaseBatch{
|
||||
void run() {
|
||||
index++;
|
||||
def map = [:];
|
||||
map.sell_count = findHouseCount(dist, "sell", startDate, endDate);
|
||||
map.rent_count = findHouseCount(dist, "rent", startDate, endDate);
|
||||
// map.sell_count = findHouseCount(dist, "sell", startDate, endDate);
|
||||
// map.rent_count = findHouseCount(dist, "rent", startDate, endDate);
|
||||
|
||||
map.sell_count = CommonDatautils.findHouseCountByDate(dist.city, dist.region, dist.name, dist.alias, "sell", startDate)
|
||||
map.rent_count = CommonDatautils.findHouseCountByDate(dist.city, dist.region, dist.name, dist.alias, "rent", startDate)
|
||||
|
||||
map.city = dist.city;
|
||||
map.region = dist.region;
|
||||
map.name = dist.name;
|
||||
|
||||
+64
-15
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.jiangnanblank
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -7,24 +8,17 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.rabbitmq.client.AMQP
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.poi.hssf.usermodel.HSSFWorkbook
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.text.DecimalFormat
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* Created by gaojuan on 2017/12/11.
|
||||
* 批量全国小区及商铺字段转换
|
||||
@@ -33,7 +27,7 @@ import java.util.regex.Pattern
|
||||
class CZDataBatch extends BaseBatch{
|
||||
|
||||
public static def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
def fields = ["city","region","name", "address", "cat","lat2","lng2","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"]
|
||||
def fields = ["city","region","name", "address", "cat","lat2","lng2","lat","lng","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"]
|
||||
public static DBCollection DIST_COLL = MongoUtils.getCollectionByName2("business", "jn_bank_district", dbUrl)
|
||||
|
||||
//final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls");
|
||||
@@ -47,6 +41,8 @@ class CZDataBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
println "args 参数:"+args.toString()
|
||||
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
|
||||
citys.remove("常州");
|
||||
@@ -57,12 +53,21 @@ class CZDataBatch extends BaseBatch{
|
||||
def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend)
|
||||
def commids = findAllJnBankDatas(city)
|
||||
//
|
||||
def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
|
||||
FzDataUtils.removeNoUpdate(city, districts)
|
||||
def districts = null;
|
||||
def shops = null;
|
||||
if(args && args[0]== "all"){
|
||||
districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
|
||||
FzDataUtils.removeNoUpdate(city, districts)
|
||||
}else{
|
||||
districts = converList(CommonDatautils.findNewUpdateDistricts(city, "district", fields));
|
||||
}
|
||||
districtRun(districts, statMap, commids);
|
||||
|
||||
|
||||
def shops = CommonDatautils.findAllDistricts(city, "shop", fields);
|
||||
if(args && args[0]== "all"){
|
||||
shops = CommonDatautils.findAllDistricts(city, "shop", fields);
|
||||
}else{
|
||||
shops = CommonDatautils.findNewUpdateDistricts(city, "shop", fields);
|
||||
}
|
||||
shops = mergeShop(shops);
|
||||
shopRun(shops);
|
||||
}
|
||||
@@ -227,11 +232,14 @@ class CZDataBatch extends BaseBatch{
|
||||
map.build_structure = shop.build_structure
|
||||
// 楼盘均价
|
||||
map.avg_price = FzDataUtils.getShopPrice(priceString, shop.city, shop.region, shop.name);
|
||||
if(!map.avg_price){
|
||||
map.avg_price = getSellPrice(shop.city, shop.lng/10000000000.0, shop.lat/10000000000.0)
|
||||
}
|
||||
// 省 市 区
|
||||
if(shop.city == "常州"){
|
||||
map.province_code = regionMap.get("江苏");
|
||||
map.city_code = regionMap.get(shop.city);
|
||||
map.region_code = regionMap.get(shop.region);
|
||||
map.province_code = FzDataUtils.regionMap.get("江苏");
|
||||
map.city_code = FzDataUtils.regionMap.get(shop.city);
|
||||
map.region_code = FzDataUtils.regionMap.get(shop.region);
|
||||
map.province = "江苏";
|
||||
map.price_old_string = priceString;
|
||||
map.heights = heights;
|
||||
@@ -259,6 +267,9 @@ class CZDataBatch extends BaseBatch{
|
||||
map.street_code = FzDataUtils.getStreetCode(street);
|
||||
// 租金
|
||||
map.shop_rent = FzDataUtils.getRentNumber(rentPrice, shop.city, shop.region, shop.name);
|
||||
if(!map.shop_rent){
|
||||
map.shop_rent = getRentPrice(shop.city, shop.lng/10000000000.0, shop.lat/10000000000.0)
|
||||
}
|
||||
// 物业公司
|
||||
map.estate_company = FzDataUtils.getEstateCompany(shop.estate_company);
|
||||
// 物业费(每月每平米)
|
||||
@@ -267,6 +278,44 @@ class CZDataBatch extends BaseBatch{
|
||||
saveShop(map);
|
||||
}
|
||||
|
||||
def getSellPrice(def city, def lng, def lat){
|
||||
def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd')
|
||||
def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_sell")
|
||||
if(rets){
|
||||
def prices = []
|
||||
for(def r : rets){
|
||||
prices << r._source.avg_price
|
||||
}
|
||||
if(prices){
|
||||
def price = (int)(prices.sum()/prices.size())
|
||||
return price
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def getRentPrice(def city, def lng, def lat){
|
||||
def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd')
|
||||
def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_rent")
|
||||
if(!rets){
|
||||
rets = CommonDatautils.findSeawater(city, lng, lat, "2km", p_date, "shop_rent")
|
||||
if(!rets){
|
||||
rets = CommonDatautils.findSeawater(city, lng, lat, "3km", p_date, "shop_rent")
|
||||
}
|
||||
}
|
||||
if(rets){
|
||||
def prices = []
|
||||
for(def r : rets){
|
||||
prices << r._source.avg_price
|
||||
}
|
||||
if(prices){
|
||||
def day_price = (prices.sum()/prices.size())/30
|
||||
return DataHandleUtils.getDoubleRound(2, day_price)
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def saveShop(def m){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city",m.city)
|
||||
|
||||
+1
@@ -809,4 +809,5 @@ class FzDataUtils {
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user