Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj

This commit is contained in:
2025-12-04 11:11:51 +08:00
4 changed files with 168 additions and 25 deletions
@@ -832,6 +832,33 @@ class CommonDatautils {
} }
public static def findNewUpdateDistricts(def city, def cat, def fields){
def startDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
DBObject q = new BasicDBObject();
q.put("status",0)
// q.put("status",new BasicDBObject('$ne', 0))
q.put("cat", cat);
q.put("city", city)
q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
// q.put("loop_line_info",new BasicDBObject('$exists',false))
// q.put("name","扬州北街8号")
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
def rets = []
for(def d : dbsour){
rets << d
}
return rets;
}
public static def findAllRegionDistricts(def city, def region, def cat, def fields){ public static def findAllRegionDistricts(def city, def region, def cat, def fields){
DBObject q = new BasicDBObject(); DBObject q = new BasicDBObject();
@@ -5818,4 +5845,68 @@ class CommonDatautils {
} }
return address; return address;
} }
static def findSeawater(city, lng, lat, distance, p_date, cat){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.206:9200/seawater-*/_search"
def body = """
{
"query": {
"bool": {
"must": [
{
"geo_distance": {
"distance": "${distance}",
"location": {
"lat": ${lat},
"lon": ${lng}
}
}
},
{
"term": {
"city": "${city}"
}
},
{
"term": {
"cat": "${cat}"
}
},
{
"range": {
"publish_date": {
"gte": "${p_date}",
"format": "yyyy-MM-dd"
}
}
}
]
}
},
"sort": [
{
"publish_date": {
"order": "desc"
}
}
],
"size":30
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = new JsonSlurper().parseText(res)
def esObj = null
if(response && response.hits){
esObj = response.hits.hits
}
return esObj
}
} }
@@ -1,21 +1,16 @@
package com.fangjia.script.house package com.fangjia.script.house
import com.fangjia.core.util.DateUtil
import com.fangjia.datastore.driver.DSCollection import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.BaseBatch import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import com.mongodb.QueryOperators
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
@@ -48,14 +43,17 @@ class DistrictSellHouseStatBatch extends BaseBatch{
for(def city : citys){ for(def city : citys){
index ++ ; index ++ ;
def districts = CommonDatautils.getAllDistrictBaseInfo(city, "district") def districts = CommonDatautils.getAllDistrictBaseInfo(city, "district")
def sellRet = houseStatCount(districts, startDate, endDate, sDate);
String start = DateUtils.formatDateByPattern(startDate, "yyyyMMdd")
def sellRet = houseStatCount(districts, start.toInteger(), sDate);
saveResult(city, sDate, sellRet); saveResult(city, sDate, sellRet);
println "[${city} ${index}/${citys.size()}]" +new Date().toString(); println "[${city} ${index}/${citys.size()}]" +new Date().toString();
} }
} }
def houseStatCount(def districts, def startDate, def endDate, def sDate){ def houseStatCount(def districts, def startDate, def sDate){
def index = 0; def index = 0;
@@ -75,8 +73,12 @@ class DistrictSellHouseStatBatch extends BaseBatch{
void run() { void run() {
index++; index++;
def map = [:]; def map = [:];
map.sell_count = findHouseCount(dist, "sell", startDate, endDate); // map.sell_count = findHouseCount(dist, "sell", startDate, endDate);
map.rent_count = findHouseCount(dist, "rent", startDate, endDate); // map.rent_count = findHouseCount(dist, "rent", startDate, endDate);
map.sell_count = CommonDatautils.findHouseCountByDate(dist.city, dist.region, dist.name, dist.alias, "sell", startDate)
map.rent_count = CommonDatautils.findHouseCountByDate(dist.city, dist.region, dist.name, dist.alias, "rent", startDate)
map.city = dist.city; map.city = dist.city;
map.region = dist.region; map.region = dist.region;
map.name = dist.name; map.name = dist.name;
@@ -1,5 +1,6 @@
package com.fangjia.script.jiangnanblank package com.fangjia.script.jiangnanblank
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher import com.fangjia.script.Watcher
@@ -7,24 +8,17 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import org.apache.poi.hssf.usermodel.HSSFWorkbook import org.apache.poi.hssf.usermodel.HSSFWorkbook
import org.bson.types.ObjectId import org.bson.types.ObjectId
import java.text.DecimalFormat
import java.util.concurrent.CountDownLatch import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
/** /**
* Created by gaojuan on 2017/12/11. * Created by gaojuan on 2017/12/11.
* 批量全国小区及商铺字段转换 * 批量全国小区及商铺字段转换
@@ -33,7 +27,7 @@ import java.util.regex.Pattern
class CZDataBatch extends BaseBatch{ class CZDataBatch extends BaseBatch{
public static def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; public static def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
def fields = ["city","region","name", "address", "cat","lat2","lng2","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"] def fields = ["city","region","name", "address", "cat","lat2","lng2","lat","lng","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"]
public static DBCollection DIST_COLL = MongoUtils.getCollectionByName2("business", "jn_bank_district", dbUrl) public static DBCollection DIST_COLL = MongoUtils.getCollectionByName2("business", "jn_bank_district", dbUrl)
//final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls"); //final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls");
@@ -47,6 +41,8 @@ class CZDataBatch extends BaseBatch{
@Override @Override
void process(Object args) { void process(Object args) {
println "args 参数:"+args.toString()
def citys = CommonDatautils.findAllCity(); def citys = CommonDatautils.findAllCity();
citys.remove("常州"); citys.remove("常州");
@@ -57,12 +53,21 @@ class CZDataBatch extends BaseBatch{
def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend) def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend)
def commids = findAllJnBankDatas(city) def commids = findAllJnBankDatas(city)
// //
def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields)); def districts = null;
FzDataUtils.removeNoUpdate(city, districts) def shops = null;
if(args && args[0]== "all"){
districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
FzDataUtils.removeNoUpdate(city, districts)
}else{
districts = converList(CommonDatautils.findNewUpdateDistricts(city, "district", fields));
}
districtRun(districts, statMap, commids); districtRun(districts, statMap, commids);
if(args && args[0]== "all"){
def shops = CommonDatautils.findAllDistricts(city, "shop", fields); shops = CommonDatautils.findAllDistricts(city, "shop", fields);
}else{
shops = CommonDatautils.findNewUpdateDistricts(city, "shop", fields);
}
shops = mergeShop(shops); shops = mergeShop(shops);
shopRun(shops); shopRun(shops);
} }
@@ -227,11 +232,14 @@ class CZDataBatch extends BaseBatch{
map.build_structure = shop.build_structure map.build_structure = shop.build_structure
// 楼盘均价 // 楼盘均价
map.avg_price = FzDataUtils.getShopPrice(priceString, shop.city, shop.region, shop.name); map.avg_price = FzDataUtils.getShopPrice(priceString, shop.city, shop.region, shop.name);
if(!map.avg_price){
map.avg_price = getSellPrice(shop.city, shop.lng/10000000000.0, shop.lat/10000000000.0)
}
// 省 市 区 // 省 市 区
if(shop.city == "常州"){ if(shop.city == "常州"){
map.province_code = regionMap.get("江苏"); map.province_code = FzDataUtils.regionMap.get("江苏");
map.city_code = regionMap.get(shop.city); map.city_code = FzDataUtils.regionMap.get(shop.city);
map.region_code = regionMap.get(shop.region); map.region_code = FzDataUtils.regionMap.get(shop.region);
map.province = "江苏"; map.province = "江苏";
map.price_old_string = priceString; map.price_old_string = priceString;
map.heights = heights; map.heights = heights;
@@ -259,6 +267,9 @@ class CZDataBatch extends BaseBatch{
map.street_code = FzDataUtils.getStreetCode(street); map.street_code = FzDataUtils.getStreetCode(street);
// 租金 // 租金
map.shop_rent = FzDataUtils.getRentNumber(rentPrice, shop.city, shop.region, shop.name); map.shop_rent = FzDataUtils.getRentNumber(rentPrice, shop.city, shop.region, shop.name);
if(!map.shop_rent){
map.shop_rent = getRentPrice(shop.city, shop.lng/10000000000.0, shop.lat/10000000000.0)
}
// 物业公司 // 物业公司
map.estate_company = FzDataUtils.getEstateCompany(shop.estate_company); map.estate_company = FzDataUtils.getEstateCompany(shop.estate_company);
// 物业费(每月每平米) // 物业费(每月每平米)
@@ -267,6 +278,44 @@ class CZDataBatch extends BaseBatch{
saveShop(map); saveShop(map);
} }
def getSellPrice(def city, def lng, def lat){
def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd')
def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_sell")
if(rets){
def prices = []
for(def r : rets){
prices << r._source.avg_price
}
if(prices){
def price = (int)(prices.sum()/prices.size())
return price
}
}
return null;
}
def getRentPrice(def city, def lng, def lat){
def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd')
def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_rent")
if(!rets){
rets = CommonDatautils.findSeawater(city, lng, lat, "2km", p_date, "shop_rent")
if(!rets){
rets = CommonDatautils.findSeawater(city, lng, lat, "3km", p_date, "shop_rent")
}
}
if(rets){
def prices = []
for(def r : rets){
prices << r._source.avg_price
}
if(prices){
def day_price = (prices.sum()/prices.size())/30
return DataHandleUtils.getDoubleRound(2, day_price)
}
}
return null;
}
def saveShop(def m){ def saveShop(def m){
DBObject q = new BasicDBObject(); DBObject q = new BasicDBObject();
q.put("city",m.city) q.put("city",m.city)
@@ -809,4 +809,5 @@ class FzDataUtils {
} }
} }