Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj

This commit is contained in:
2025-12-04 11:11:51 +08:00
4 changed files with 168 additions and 25 deletions
@@ -832,6 +832,33 @@ class CommonDatautils {
}
public static def findNewUpdateDistricts(def city, def cat, def fields){
def startDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
DBObject q = new BasicDBObject();
q.put("status",0)
// q.put("status",new BasicDBObject('$ne', 0))
q.put("cat", cat);
q.put("city", city)
q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
// q.put("loop_line_info",new BasicDBObject('$exists',false))
// q.put("name","扬州北街8号")
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
def rets = []
for(def d : dbsour){
rets << d
}
return rets;
}
public static def findAllRegionDistricts(def city, def region, def cat, def fields){
DBObject q = new BasicDBObject();
@@ -5818,4 +5845,68 @@ class CommonDatautils {
}
return address;
}
static def findSeawater(city, lng, lat, distance, p_date, cat){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.206:9200/seawater-*/_search"
def body = """
{
"query": {
"bool": {
"must": [
{
"geo_distance": {
"distance": "${distance}",
"location": {
"lat": ${lat},
"lon": ${lng}
}
}
},
{
"term": {
"city": "${city}"
}
},
{
"term": {
"cat": "${cat}"
}
},
{
"range": {
"publish_date": {
"gte": "${p_date}",
"format": "yyyy-MM-dd"
}
}
}
]
}
},
"sort": [
{
"publish_date": {
"order": "desc"
}
}
],
"size":30
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = new JsonSlurper().parseText(res)
def esObj = null
if(response && response.hits){
esObj = response.hits.hits
}
return esObj
}
}
@@ -1,21 +1,16 @@
package com.fangjia.script.house
import com.fangjia.core.util.DateUtil
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
@@ -48,14 +43,17 @@ class DistrictSellHouseStatBatch extends BaseBatch{
for(def city : citys){
index ++ ;
def districts = CommonDatautils.getAllDistrictBaseInfo(city, "district")
def sellRet = houseStatCount(districts, startDate, endDate, sDate);
String start = DateUtils.formatDateByPattern(startDate, "yyyyMMdd")
def sellRet = houseStatCount(districts, start.toInteger(), sDate);
saveResult(city, sDate, sellRet);
println "[${city} ${index}/${citys.size()}]" +new Date().toString();
}
}
def houseStatCount(def districts, def startDate, def endDate, def sDate){
def houseStatCount(def districts, def startDate, def sDate){
def index = 0;
@@ -75,8 +73,12 @@ class DistrictSellHouseStatBatch extends BaseBatch{
void run() {
index++;
def map = [:];
map.sell_count = findHouseCount(dist, "sell", startDate, endDate);
map.rent_count = findHouseCount(dist, "rent", startDate, endDate);
// map.sell_count = findHouseCount(dist, "sell", startDate, endDate);
// map.rent_count = findHouseCount(dist, "rent", startDate, endDate);
map.sell_count = CommonDatautils.findHouseCountByDate(dist.city, dist.region, dist.name, dist.alias, "sell", startDate)
map.rent_count = CommonDatautils.findHouseCountByDate(dist.city, dist.region, dist.name, dist.alias, "rent", startDate)
map.city = dist.city;
map.region = dist.region;
map.name = dist.name;
@@ -1,5 +1,6 @@
package com.fangjia.script.jiangnanblank
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -7,24 +8,17 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import org.apache.poi.hssf.usermodel.HSSFWorkbook
import org.bson.types.ObjectId
import java.text.DecimalFormat
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* Created by gaojuan on 2017/12/11.
* 批量全国小区及商铺字段转换
@@ -33,7 +27,7 @@ import java.util.regex.Pattern
class CZDataBatch extends BaseBatch{
public static def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
def fields = ["city","region","name", "address", "cat","lat2","lng2","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"]
def fields = ["city","region","name", "address", "cat","lat2","lng2","lat","lng","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"]
public static DBCollection DIST_COLL = MongoUtils.getCollectionByName2("business", "jn_bank_district", dbUrl)
//final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls");
@@ -47,6 +41,8 @@ class CZDataBatch extends BaseBatch{
@Override
void process(Object args) {
println "args 参数:"+args.toString()
def citys = CommonDatautils.findAllCity();
citys.remove("常州");
@@ -57,12 +53,21 @@ class CZDataBatch extends BaseBatch{
def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend)
def commids = findAllJnBankDatas(city)
//
def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
FzDataUtils.removeNoUpdate(city, districts)
def districts = null;
def shops = null;
if(args && args[0]== "all"){
districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
FzDataUtils.removeNoUpdate(city, districts)
}else{
districts = converList(CommonDatautils.findNewUpdateDistricts(city, "district", fields));
}
districtRun(districts, statMap, commids);
def shops = CommonDatautils.findAllDistricts(city, "shop", fields);
if(args && args[0]== "all"){
shops = CommonDatautils.findAllDistricts(city, "shop", fields);
}else{
shops = CommonDatautils.findNewUpdateDistricts(city, "shop", fields);
}
shops = mergeShop(shops);
shopRun(shops);
}
@@ -227,11 +232,14 @@ class CZDataBatch extends BaseBatch{
map.build_structure = shop.build_structure
// 楼盘均价
map.avg_price = FzDataUtils.getShopPrice(priceString, shop.city, shop.region, shop.name);
if(!map.avg_price){
map.avg_price = getSellPrice(shop.city, shop.lng/10000000000.0, shop.lat/10000000000.0)
}
// 省 市 区
if(shop.city == "常州"){
map.province_code = regionMap.get("江苏");
map.city_code = regionMap.get(shop.city);
map.region_code = regionMap.get(shop.region);
map.province_code = FzDataUtils.regionMap.get("江苏");
map.city_code = FzDataUtils.regionMap.get(shop.city);
map.region_code = FzDataUtils.regionMap.get(shop.region);
map.province = "江苏";
map.price_old_string = priceString;
map.heights = heights;
@@ -259,6 +267,9 @@ class CZDataBatch extends BaseBatch{
map.street_code = FzDataUtils.getStreetCode(street);
// 租金
map.shop_rent = FzDataUtils.getRentNumber(rentPrice, shop.city, shop.region, shop.name);
if(!map.shop_rent){
map.shop_rent = getRentPrice(shop.city, shop.lng/10000000000.0, shop.lat/10000000000.0)
}
// 物业公司
map.estate_company = FzDataUtils.getEstateCompany(shop.estate_company);
// 物业费(每月每平米)
@@ -267,6 +278,44 @@ class CZDataBatch extends BaseBatch{
saveShop(map);
}
def getSellPrice(def city, def lng, def lat){
def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd')
def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_sell")
if(rets){
def prices = []
for(def r : rets){
prices << r._source.avg_price
}
if(prices){
def price = (int)(prices.sum()/prices.size())
return price
}
}
return null;
}
def getRentPrice(def city, def lng, def lat){
def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd')
def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_rent")
if(!rets){
rets = CommonDatautils.findSeawater(city, lng, lat, "2km", p_date, "shop_rent")
if(!rets){
rets = CommonDatautils.findSeawater(city, lng, lat, "3km", p_date, "shop_rent")
}
}
if(rets){
def prices = []
for(def r : rets){
prices << r._source.avg_price
}
if(prices){
def day_price = (prices.sum()/prices.size())/30
return DataHandleUtils.getDoubleRound(2, day_price)
}
}
return null;
}
def saveShop(def m){
DBObject q = new BasicDBObject();
q.put("city",m.city)
@@ -809,4 +809,5 @@ class FzDataUtils {
}
}