Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj
This commit is contained in:
+91
@@ -832,6 +832,33 @@ class CommonDatautils {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static def findNewUpdateDistricts(def city, def cat, def fields){
|
||||||
|
|
||||||
|
def startDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
|
||||||
|
DBObject q = new BasicDBObject();
|
||||||
|
q.put("status",0)
|
||||||
|
// q.put("status",new BasicDBObject('$ne', 0))
|
||||||
|
q.put("cat", cat);
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
|
||||||
|
// q.put("loop_line_info",new BasicDBObject('$exists',false))
|
||||||
|
// q.put("name","扬州北街8号")
|
||||||
|
|
||||||
|
DBObject keys = new BasicDBObject();
|
||||||
|
if(fields){
|
||||||
|
for(def field : fields){
|
||||||
|
keys.put(field, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
|
||||||
|
def rets = []
|
||||||
|
for(def d : dbsour){
|
||||||
|
rets << d
|
||||||
|
}
|
||||||
|
return rets;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
public static def findAllRegionDistricts(def city, def region, def cat, def fields){
|
public static def findAllRegionDistricts(def city, def region, def cat, def fields){
|
||||||
|
|
||||||
DBObject q = new BasicDBObject();
|
DBObject q = new BasicDBObject();
|
||||||
@@ -5818,4 +5845,68 @@ class CommonDatautils {
|
|||||||
}
|
}
|
||||||
return address;
|
return address;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static def findSeawater(city, lng, lat, distance, p_date, cat){
|
||||||
|
def header = [:]
|
||||||
|
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
|
||||||
|
String seawaterUrl = "http://192.168.0.206:9200/seawater-*/_search"
|
||||||
|
def body = """
|
||||||
|
{
|
||||||
|
"query": {
|
||||||
|
"bool": {
|
||||||
|
"must": [
|
||||||
|
{
|
||||||
|
"geo_distance": {
|
||||||
|
"distance": "${distance}",
|
||||||
|
"location": {
|
||||||
|
"lat": ${lat},
|
||||||
|
"lon": ${lng}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"term": {
|
||||||
|
"city": "${city}"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"term": {
|
||||||
|
"cat": "${cat}"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"range": {
|
||||||
|
"publish_date": {
|
||||||
|
"gte": "${p_date}",
|
||||||
|
"format": "yyyy-MM-dd"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"sort": [
|
||||||
|
{
|
||||||
|
"publish_date": {
|
||||||
|
"order": "desc"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"size":30
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
HttpRequest request = new HttpRequest()
|
||||||
|
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||||
|
def response = new JsonSlurper().parseText(res)
|
||||||
|
|
||||||
|
def esObj = null
|
||||||
|
if(response && response.hits){
|
||||||
|
esObj = response.hits.hits
|
||||||
|
}
|
||||||
|
return esObj
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
+12
-10
@@ -1,21 +1,16 @@
|
|||||||
package com.fangjia.script.house
|
package com.fangjia.script.house
|
||||||
import com.fangjia.core.util.DateUtil
|
|
||||||
import com.fangjia.datastore.driver.DSCollection
|
import com.fangjia.datastore.driver.DSCollection
|
||||||
import com.fangjia.datastore.driver.DataStore
|
import com.fangjia.datastore.driver.DataStore
|
||||||
import com.fangjia.script.BaseBatch
|
import com.fangjia.script.BaseBatch
|
||||||
import com.fangjia.script.BatchOwner
|
import com.fangjia.script.BatchOwner
|
||||||
import com.fangjia.script.Watcher
|
import com.fangjia.script.Watcher
|
||||||
import com.fangjia.script.commData.CommonDatautils
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
import com.fangjia.script.util.CommonUtils
|
|
||||||
import com.fangjia.script.util.DateUtils
|
import com.fangjia.script.util.DateUtils
|
||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.Bytes
|
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
import com.mongodb.DBObject
|
import com.mongodb.DBObject
|
||||||
import com.mongodb.QueryOperators
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
import java.util.concurrent.ExecutorService
|
import java.util.concurrent.ExecutorService
|
||||||
import java.util.concurrent.Executors
|
import java.util.concurrent.Executors
|
||||||
@@ -48,14 +43,17 @@ class DistrictSellHouseStatBatch extends BaseBatch{
|
|||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
index ++ ;
|
index ++ ;
|
||||||
def districts = CommonDatautils.getAllDistrictBaseInfo(city, "district")
|
def districts = CommonDatautils.getAllDistrictBaseInfo(city, "district")
|
||||||
def sellRet = houseStatCount(districts, startDate, endDate, sDate);
|
|
||||||
|
String start = DateUtils.formatDateByPattern(startDate, "yyyyMMdd")
|
||||||
|
|
||||||
|
def sellRet = houseStatCount(districts, start.toInteger(), sDate);
|
||||||
saveResult(city, sDate, sellRet);
|
saveResult(city, sDate, sellRet);
|
||||||
|
|
||||||
println "[${city} ${index}/${citys.size()}]" +new Date().toString();
|
println "[${city} ${index}/${citys.size()}]" +new Date().toString();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def houseStatCount(def districts, def startDate, def endDate, def sDate){
|
def houseStatCount(def districts, def startDate, def sDate){
|
||||||
|
|
||||||
def index = 0;
|
def index = 0;
|
||||||
|
|
||||||
@@ -75,8 +73,12 @@ class DistrictSellHouseStatBatch extends BaseBatch{
|
|||||||
void run() {
|
void run() {
|
||||||
index++;
|
index++;
|
||||||
def map = [:];
|
def map = [:];
|
||||||
map.sell_count = findHouseCount(dist, "sell", startDate, endDate);
|
// map.sell_count = findHouseCount(dist, "sell", startDate, endDate);
|
||||||
map.rent_count = findHouseCount(dist, "rent", startDate, endDate);
|
// map.rent_count = findHouseCount(dist, "rent", startDate, endDate);
|
||||||
|
|
||||||
|
map.sell_count = CommonDatautils.findHouseCountByDate(dist.city, dist.region, dist.name, dist.alias, "sell", startDate)
|
||||||
|
map.rent_count = CommonDatautils.findHouseCountByDate(dist.city, dist.region, dist.name, dist.alias, "rent", startDate)
|
||||||
|
|
||||||
map.city = dist.city;
|
map.city = dist.city;
|
||||||
map.region = dist.region;
|
map.region = dist.region;
|
||||||
map.name = dist.name;
|
map.name = dist.name;
|
||||||
|
|||||||
+64
-15
@@ -1,5 +1,6 @@
|
|||||||
package com.fangjia.script.jiangnanblank
|
package com.fangjia.script.jiangnanblank
|
||||||
|
|
||||||
|
import com.fangjia.core.util.DateUtil
|
||||||
import com.fangjia.script.BaseBatch
|
import com.fangjia.script.BaseBatch
|
||||||
import com.fangjia.script.BatchOwner
|
import com.fangjia.script.BatchOwner
|
||||||
import com.fangjia.script.Watcher
|
import com.fangjia.script.Watcher
|
||||||
@@ -7,24 +8,17 @@ import com.fangjia.script.commData.CommonDatautils
|
|||||||
import com.fangjia.script.common.DataHandleUtils
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
import com.fangjia.script.util.DateUtils
|
import com.fangjia.script.util.DateUtils
|
||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
import com.fangjia.script.util.OpenApiClient
|
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
import com.mongodb.DBObject
|
import com.mongodb.DBObject
|
||||||
import com.rabbitmq.client.AMQP
|
|
||||||
import net.sf.json.JSONObject
|
|
||||||
import org.apache.poi.hssf.usermodel.HSSFWorkbook
|
import org.apache.poi.hssf.usermodel.HSSFWorkbook
|
||||||
import org.bson.types.ObjectId
|
import org.bson.types.ObjectId
|
||||||
|
|
||||||
import java.text.DecimalFormat
|
|
||||||
import java.util.concurrent.CountDownLatch
|
import java.util.concurrent.CountDownLatch
|
||||||
import java.util.concurrent.ExecutorService
|
import java.util.concurrent.ExecutorService
|
||||||
import java.util.concurrent.Executors
|
import java.util.concurrent.Executors
|
||||||
import java.util.concurrent.TimeUnit
|
import java.util.concurrent.TimeUnit
|
||||||
import java.util.concurrent.atomic.AtomicLong
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
import java.util.regex.Matcher
|
|
||||||
import java.util.regex.Pattern
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Created by gaojuan on 2017/12/11.
|
* Created by gaojuan on 2017/12/11.
|
||||||
* 批量全国小区及商铺字段转换
|
* 批量全国小区及商铺字段转换
|
||||||
@@ -33,7 +27,7 @@ import java.util.regex.Pattern
|
|||||||
class CZDataBatch extends BaseBatch{
|
class CZDataBatch extends BaseBatch{
|
||||||
|
|
||||||
public static def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
public static def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||||
def fields = ["city","region","name", "address", "cat","lat2","lng2","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"]
|
def fields = ["city","region","name", "address", "cat","lat2","lng2","lat","lng","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"]
|
||||||
public static DBCollection DIST_COLL = MongoUtils.getCollectionByName2("business", "jn_bank_district", dbUrl)
|
public static DBCollection DIST_COLL = MongoUtils.getCollectionByName2("business", "jn_bank_district", dbUrl)
|
||||||
|
|
||||||
//final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls");
|
//final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls");
|
||||||
@@ -47,6 +41,8 @@ class CZDataBatch extends BaseBatch{
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
void process(Object args) {
|
void process(Object args) {
|
||||||
|
println "args 参数:"+args.toString()
|
||||||
|
|
||||||
def citys = CommonDatautils.findAllCity();
|
def citys = CommonDatautils.findAllCity();
|
||||||
|
|
||||||
citys.remove("常州");
|
citys.remove("常州");
|
||||||
@@ -57,12 +53,21 @@ class CZDataBatch extends BaseBatch{
|
|||||||
def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend)
|
def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend)
|
||||||
def commids = findAllJnBankDatas(city)
|
def commids = findAllJnBankDatas(city)
|
||||||
//
|
//
|
||||||
def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
|
def districts = null;
|
||||||
FzDataUtils.removeNoUpdate(city, districts)
|
def shops = null;
|
||||||
|
if(args && args[0]== "all"){
|
||||||
|
districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
|
||||||
|
FzDataUtils.removeNoUpdate(city, districts)
|
||||||
|
}else{
|
||||||
|
districts = converList(CommonDatautils.findNewUpdateDistricts(city, "district", fields));
|
||||||
|
}
|
||||||
districtRun(districts, statMap, commids);
|
districtRun(districts, statMap, commids);
|
||||||
|
|
||||||
|
if(args && args[0]== "all"){
|
||||||
def shops = CommonDatautils.findAllDistricts(city, "shop", fields);
|
shops = CommonDatautils.findAllDistricts(city, "shop", fields);
|
||||||
|
}else{
|
||||||
|
shops = CommonDatautils.findNewUpdateDistricts(city, "shop", fields);
|
||||||
|
}
|
||||||
shops = mergeShop(shops);
|
shops = mergeShop(shops);
|
||||||
shopRun(shops);
|
shopRun(shops);
|
||||||
}
|
}
|
||||||
@@ -227,11 +232,14 @@ class CZDataBatch extends BaseBatch{
|
|||||||
map.build_structure = shop.build_structure
|
map.build_structure = shop.build_structure
|
||||||
// 楼盘均价
|
// 楼盘均价
|
||||||
map.avg_price = FzDataUtils.getShopPrice(priceString, shop.city, shop.region, shop.name);
|
map.avg_price = FzDataUtils.getShopPrice(priceString, shop.city, shop.region, shop.name);
|
||||||
|
if(!map.avg_price){
|
||||||
|
map.avg_price = getSellPrice(shop.city, shop.lng/10000000000.0, shop.lat/10000000000.0)
|
||||||
|
}
|
||||||
// 省 市 区
|
// 省 市 区
|
||||||
if(shop.city == "常州"){
|
if(shop.city == "常州"){
|
||||||
map.province_code = regionMap.get("江苏");
|
map.province_code = FzDataUtils.regionMap.get("江苏");
|
||||||
map.city_code = regionMap.get(shop.city);
|
map.city_code = FzDataUtils.regionMap.get(shop.city);
|
||||||
map.region_code = regionMap.get(shop.region);
|
map.region_code = FzDataUtils.regionMap.get(shop.region);
|
||||||
map.province = "江苏";
|
map.province = "江苏";
|
||||||
map.price_old_string = priceString;
|
map.price_old_string = priceString;
|
||||||
map.heights = heights;
|
map.heights = heights;
|
||||||
@@ -259,6 +267,9 @@ class CZDataBatch extends BaseBatch{
|
|||||||
map.street_code = FzDataUtils.getStreetCode(street);
|
map.street_code = FzDataUtils.getStreetCode(street);
|
||||||
// 租金
|
// 租金
|
||||||
map.shop_rent = FzDataUtils.getRentNumber(rentPrice, shop.city, shop.region, shop.name);
|
map.shop_rent = FzDataUtils.getRentNumber(rentPrice, shop.city, shop.region, shop.name);
|
||||||
|
if(!map.shop_rent){
|
||||||
|
map.shop_rent = getRentPrice(shop.city, shop.lng/10000000000.0, shop.lat/10000000000.0)
|
||||||
|
}
|
||||||
// 物业公司
|
// 物业公司
|
||||||
map.estate_company = FzDataUtils.getEstateCompany(shop.estate_company);
|
map.estate_company = FzDataUtils.getEstateCompany(shop.estate_company);
|
||||||
// 物业费(每月每平米)
|
// 物业费(每月每平米)
|
||||||
@@ -267,6 +278,44 @@ class CZDataBatch extends BaseBatch{
|
|||||||
saveShop(map);
|
saveShop(map);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def getSellPrice(def city, def lng, def lat){
|
||||||
|
def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd')
|
||||||
|
def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_sell")
|
||||||
|
if(rets){
|
||||||
|
def prices = []
|
||||||
|
for(def r : rets){
|
||||||
|
prices << r._source.avg_price
|
||||||
|
}
|
||||||
|
if(prices){
|
||||||
|
def price = (int)(prices.sum()/prices.size())
|
||||||
|
return price
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
def getRentPrice(def city, def lng, def lat){
|
||||||
|
def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd')
|
||||||
|
def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_rent")
|
||||||
|
if(!rets){
|
||||||
|
rets = CommonDatautils.findSeawater(city, lng, lat, "2km", p_date, "shop_rent")
|
||||||
|
if(!rets){
|
||||||
|
rets = CommonDatautils.findSeawater(city, lng, lat, "3km", p_date, "shop_rent")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(rets){
|
||||||
|
def prices = []
|
||||||
|
for(def r : rets){
|
||||||
|
prices << r._source.avg_price
|
||||||
|
}
|
||||||
|
if(prices){
|
||||||
|
def day_price = (prices.sum()/prices.size())/30
|
||||||
|
return DataHandleUtils.getDoubleRound(2, day_price)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
def saveShop(def m){
|
def saveShop(def m){
|
||||||
DBObject q = new BasicDBObject();
|
DBObject q = new BasicDBObject();
|
||||||
q.put("city",m.city)
|
q.put("city",m.city)
|
||||||
|
|||||||
+1
@@ -809,4 +809,5 @@ class FzDataUtils {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user