Merge branch 'script_gj' into 'master'

Script gj

See merge request !1200
This commit is contained in:
2025-12-04 19:47:26 +08:00
5 changed files with 272 additions and 50 deletions
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份成交数据.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳10月份成交数据11.20.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -443,7 +443,7 @@ class CcbCommonUtils {
}
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
// q.put("name","龙源星城")
// q.put("name","登高西路安置小区C区")
// q.put("name_cat", 0);
// q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"]))
@@ -4,11 +4,13 @@ import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.MongoClient
import java.text.DecimalFormat
import java.util.concurrent.CopyOnWriteArrayList
@@ -26,6 +28,8 @@ class DistrictSellStatBatch extends BaseBatch {
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com")
public static final DBCollection COMMAND_COLL = MongoUtils.getCollectionByName("fangjia_log", "calc_command", "log.mongo.db.fangjia.com",52002)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static final DBCollection SEAWEED_MONGO_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
@@ -37,14 +41,46 @@ class DistrictSellStatBatch extends BaseBatch {
@Override
void process(Object args) {
def months = DateUtils.getMonthList(202510)
months = months[0..(months.size()-1)]
println months
for(def month : months){
def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd")
calcMonthlyPrice(curDate)
// def months = DateUtils.getMonthList(202510)
// months = months[0..(months.size()-1)]
// println months
// for(def month : months){
// def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd")
// calcMonthlyPrice(curDate)
// }
def date = new Date()
def day = date.getDate()
def hours = date.getHours()
if(day == 4 && hours<24){
calcMonthlyPrice() //正常算一期
}else{
//根据calc_type算历史
def q = new BasicDBObject("category" ,"sell_month")
def commands = COMMAND_COLL.find(q)
println "共 【"+commands.size()+"】 个小区重算 "+new Date()
for(def command : commands){
def calc_type = command.calc_type
def id = command._id
if(calc_type==1){
//找出第一次均价计算的时间
def obj = CommonDatautils.queryFirstOrLastDistrictPrice(command.city, command.region, command.name, 1)
if(obj){
def weekend = obj.weekend
def startMonth = DateUtils.formatDateByPattern(weekend, "yyyyMM").toInteger()
def months = DateUtils.getMonthList(startMonth)
months = months[1..(months.size()-1)]
println months
for(def month : months){
def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd")
calcMonthlyPriceByName(curDate, command.city, command.region, command.name)
}
//删除命令
COMMAND_COLL.remove(new BasicDBObject("_id", id))
}
}
}
}
// calcMonthlyPrice()
}
@@ -69,7 +105,7 @@ class DistrictSellStatBatch extends BaseBatch {
if(!curDate){
curDate = new Date()
}
def citys = ["上海"]//CommonUtils.findAllCity()
def citys = CommonUtils.findAllCity()
// citys.remove("上海")
// citys.remove("福州")
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1)
@@ -92,15 +128,15 @@ class DistrictSellStatBatch extends BaseBatch {
def districts = []
if(city == "福州"){
if(day == 29){
// if(day == 15){
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList()
}else{
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)), keys).toList()
}
// }else{
// districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)), keys).toList()
// }
}else{
def q = new BasicDBObject("city", city).append("cat", "district").append("status", 0)
// q.put("name", "越华路小区")
// if(day == 30){
// if(day == 2){
districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000).toList()
// }else{
// q.put("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate))
@@ -113,10 +149,10 @@ class DistrictSellStatBatch extends BaseBatch {
def startDate = DateUtils.getFirstDayOfMonth(curDate, -1);
def endDate = DateUtils.getLastDayOfMonth(curDate, -1);
if(districts.size()>0){
def weekendPriceMap = findPriceMap(city, startDate, endDate)//上个月小区均价
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价
def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价
def weekendPriceMap = findPriceMap(city, startDate, endDate, null, null)//上个月小区均价
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth, null, null)//当月小区月均价
def lastMonthlyPriceMap = findMonthlyPrice(city, monDate, null, null)//上个月小区月均价
def lastYearPriceMap = findMonthlyPrice(city, yoyDate, null, null)//去年小区月均价
if(weekendPriceMap){
running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap)
}
@@ -124,6 +160,44 @@ class DistrictSellStatBatch extends BaseBatch {
}
}
def calcMonthlyPriceByName(curDate, city, region, name) {
//只有福州使用tob价格,改为所有城市都使用2B价格,除人工指导价除外
if (!curDate) {
curDate = new Date()
}
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1)
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(curDate, "yyyyMM"), -1, "yyyyMM"));
def monDate = getMomDate(curMonth)
def yoyDate = getYoyDate(curMonth)
def q = new BasicDBObject("city", city).append("cat", "district").append("status", 0).append("region", region).append("name", name)
def districts = SEAWEED_MONGO_COLL.find(q, keys).toList()
def startDate = DateUtils.getFirstDayOfMonth(curDate, -1);
def endDate = DateUtils.getLastDayOfMonth(curDate, -1);
if (districts.size() > 0) {
def weekendPriceMap = findPriceMap(city, startDate, endDate, region, name)//上个月小区均价
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth, region, name)//当月小区月均价
def lastMonthlyPriceMap = findMonthlyPrice(city, monDate, region, name)//上个月小区月均价
def lastYearPriceMap = findMonthlyPrice(city, yoyDate, region, name)//去年小区月均价
if (weekendPriceMap) {
running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap)
}
}
}
def running(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
@@ -284,10 +358,13 @@ class DistrictSellStatBatch extends BaseBatch {
}
def findPriceMap(city, startDate, endDate){
def findPriceMap(city, startDate, endDate, region, name){
def q = new BasicDBObject()
q.put("city", city)
// q.put("district_name","长春铺")
if(name){
q.put("region", region)
q.put("district_name", name)
}
q.put("weekend", new BasicDBObject('$lte', endDate).append('$gte', startDate))
def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject().append("weekend", -1))
def priceMap = [:]
@@ -308,10 +385,14 @@ class DistrictSellStatBatch extends BaseBatch {
return priceMap
}
def findMonthlyPrice(city, monDate){
def findMonthlyPrice(city, monDate, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("date", monDate)
if(name){
q.put("region", region)
q.put("district_name", name)
}
def dbcursor = DISTRICT_SELL_STAT.find(q)
def priceMap = [:]
for(def d : dbcursor){
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
@@ -13,40 +14,36 @@ import org.bson.types.ObjectId
class DistrictLocationGeo {
final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018)
final DBCollection allSeaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "all_seaweed_gaode_pois", "192.168.0.235", 27018)
final DBCollection JD_SEAWEED_COLL = MongoUtils.getCollectionByName("business", "jingdong_community", "192.168.0.206",27020)
final DBCollection JD_GEO_COLL = MongoUtils.getCollectionByName("business", "jingdong_geo", "192.168.0.206",27020)
static main(args) {
def task = new DistrictLocationGeo()
task.run()//通过经纬度,调用高德逆解析,并保存数据
// task.run2();//通过地址,调用高德逆解析
// def distance = GeoUtils.calculateDistance(130.573501, 45.752712, 130.573527,45.753287);
// println distance
}
def run(){
ExcelUtils.modify("D:\\数据\\京东\\失败.xlsx",{ d->
if(d['E']){
def jsonObject = GaoDeApiUtils.geocode(d['E'])
if(jsonObject){
d["F"] = "高德"
def dbo = new BasicDBObject()
dbo.put("old_id", d['A'])
dbo.put("city", d['B'])
dbo.put("region", d['C'])
dbo.put("name", d['D'])
dbo.put("old_location", d['E'])
dbo.put("formatted_address", jsonObject.regeocode.formatted_address)
def addrComp = jsonObject.regeocode.addressComponent
// addrComp.remove("neighborhood")
// addrComp.remove("building")
dbo.put("addressComponent", addrComp)
dbo.put("pois", jsonObject.regeocode.pois)
dbo.put("roads", jsonObject.regeocode.roads)
dbo.put("roadinters", jsonObject.regeocode.roadinters)
dbo.put("aois", jsonObject.regeocode.aois)
dbo = cleanGeoData(dbo)
dbo = convertToDBObject(dbo)
println(dbo)
allSeaweedGaodePoisColl.save(dbo)
ExcelUtils.multiThreadingModify("D:\\数据\\京东\\京东新增数据5.6万.xlsx",{ d->
if(d['A']){
def dbo = JD_SEAWEED_COLL.findOne(new BasicDBObject("_id", new ObjectId(d['A'])))
if(dbo && dbo.longitude){
def jsonObject = GaoDeApiUtils.geocode(dbo.longitude+","+dbo.latitude)
if(jsonObject){
d["F"] = "高德"
// def dbo = new BasicDBObject()
dbo.put("formatted_address", jsonObject.regeocode.formatted_address)
def addrComp = jsonObject.regeocode.addressComponent
dbo.put("addressComponent", addrComp)
dbo.put("pois", jsonObject.regeocode.pois)
dbo.put("roads", jsonObject.regeocode.roads)
dbo.put("roadinters", jsonObject.regeocode.roadinters)
dbo.put("aois", jsonObject.regeocode.aois)
dbo = cleanGeoData(dbo)
dbo = convertToDBObject(dbo)
JD_GEO_COLL.save(dbo)
}
}
}
})
@@ -0,0 +1,144 @@
package com.fangjia.script.gj.district
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONNull
class JindDongGeoTest {
final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018)
final DBCollection allSeaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "all_seaweed_gaode_pois", "192.168.0.235", 27018)
static main(args) {
def task = new JindDongGeoTest()
task.run()//通过经纬度,调用高德逆解析,并保存数据
}
def run(){
ExcelUtils.modify("D:\\数据\\京东\\失败.xlsx",{ d->
if(d['E']){
def jsonObject = GaoDeApiUtils.geocode(d['E'])
if(jsonObject){
d["F"] = "高德"
def dbo = new BasicDBObject()
dbo.put("old_id", d['A'])
dbo.put("city", d['B'])
dbo.put("region", d['C'])
dbo.put("name", d['D'])
dbo.put("old_location", d['E'])
dbo.put("formatted_address", jsonObject.regeocode.formatted_address)
def addrComp = jsonObject.regeocode.addressComponent
// addrComp.remove("neighborhood")
// addrComp.remove("building")
dbo.put("addressComponent", addrComp)
dbo.put("pois", jsonObject.regeocode.pois)
dbo.put("roads", jsonObject.regeocode.roads)
dbo.put("roadinters", jsonObject.regeocode.roadinters)
dbo.put("aois", jsonObject.regeocode.aois)
dbo = cleanGeoData(dbo)
dbo = convertToDBObject(dbo)
println(dbo)
allSeaweedGaodePoisColl.save(dbo)
}
}
})
}
static BasicDBObject convertToDBObject(Map map) {
if (!map) return null
BasicDBObject dbObject = new BasicDBObject()
map.each { key, value ->
def convertedValue = convertValue(value)
if (convertedValue != null) {
dbObject.put(key, convertedValue)
}
}
return dbObject
}
static Object convertValue(Object value) {
if (value == null) return null
switch (value) {
case Map:
return convertToDBObject(value as Map)
case List:
return convertList(value as List)
default:
return value
}
}
static BasicDBList convertList(List list) {
BasicDBList dbList = new BasicDBList()
list.each { item ->
def convertedItem = convertValue(item)
if (convertedItem != null) {
dbList.add(convertedItem)
}
}
return dbList
}
static def cleanGeoData(data) {
if (data == null) return null
def result = [:]
data.each { key, value ->
def cleanedValue = cleanValue(value)
if (cleanedValue != null && !isEmptyCollection(cleanedValue)) {
result[key] = cleanedValue
}
}
return result
}
static def cleanValue(value) {
if (value == null) return null
switch (value) {
case Map:
def cleanedMap = [:]
value.each { k, v ->
def cleaned = cleanValue(v)
if (cleaned != null && !isEmptyCollection(cleaned)) {
cleanedMap[k] = cleaned
}
}
return cleanedMap ? cleanedMap : null
case List:
def cleanedList = value.collect { cleanValue(it) }
.findAll { it != null && !isEmptyCollection(it) }
return cleanedList ? cleanedList : null
case JSONNull:
return null
default:
// 检查是否是包含JSONNull的字符串表示
if (value instanceof String && value.contains("JSONNull")) {
return null
}
return value
}
}
static def isEmptyCollection(obj) {
if (obj instanceof Map) return obj.isEmpty()
if (obj instanceof List) return obj.isEmpty()
return false
}
}