Merge branch 'script_gj' into 'master'
Script gj See merge request !1200
This commit is contained in:
+1
-1
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份成交数据.xlsx", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳10月份成交数据11.20.xlsx", 0, 1)
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
+1
-1
@@ -443,7 +443,7 @@ class CcbCommonUtils {
|
||||
}
|
||||
|
||||
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
|
||||
// q.put("name","龙源星城")
|
||||
// q.put("name","登高西路安置小区C区")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"]))
|
||||
|
||||
|
||||
+101
-20
@@ -4,11 +4,13 @@ import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.MongoClient
|
||||
|
||||
import java.text.DecimalFormat
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
@@ -26,6 +28,8 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection COMMAND_COLL = MongoUtils.getCollectionByName("fangjia_log", "calc_command", "log.mongo.db.fangjia.com",52002)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
public static final DBCollection SEAWEED_MONGO_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
|
||||
@@ -37,14 +41,46 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def months = DateUtils.getMonthList(202510)
|
||||
months = months[0..(months.size()-1)]
|
||||
println months
|
||||
for(def month : months){
|
||||
def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd")
|
||||
calcMonthlyPrice(curDate)
|
||||
// def months = DateUtils.getMonthList(202510)
|
||||
// months = months[0..(months.size()-1)]
|
||||
// println months
|
||||
// for(def month : months){
|
||||
// def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd")
|
||||
// calcMonthlyPrice(curDate)
|
||||
// }
|
||||
|
||||
def date = new Date()
|
||||
def day = date.getDate()
|
||||
def hours = date.getHours()
|
||||
if(day == 4 && hours<24){
|
||||
calcMonthlyPrice() //正常算一期
|
||||
}else{
|
||||
//根据calc_type算历史
|
||||
def q = new BasicDBObject("category" ,"sell_month")
|
||||
def commands = COMMAND_COLL.find(q)
|
||||
println "共 【"+commands.size()+"】 个小区重算 "+new Date()
|
||||
for(def command : commands){
|
||||
def calc_type = command.calc_type
|
||||
def id = command._id
|
||||
if(calc_type==1){
|
||||
//找出第一次均价计算的时间
|
||||
def obj = CommonDatautils.queryFirstOrLastDistrictPrice(command.city, command.region, command.name, 1)
|
||||
if(obj){
|
||||
def weekend = obj.weekend
|
||||
def startMonth = DateUtils.formatDateByPattern(weekend, "yyyyMM").toInteger()
|
||||
def months = DateUtils.getMonthList(startMonth)
|
||||
months = months[1..(months.size()-1)]
|
||||
println months
|
||||
for(def month : months){
|
||||
def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd")
|
||||
calcMonthlyPriceByName(curDate, command.city, command.region, command.name)
|
||||
}
|
||||
//删除命令
|
||||
COMMAND_COLL.remove(new BasicDBObject("_id", id))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// calcMonthlyPrice()
|
||||
}
|
||||
|
||||
|
||||
@@ -69,7 +105,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
if(!curDate){
|
||||
curDate = new Date()
|
||||
}
|
||||
def citys = ["上海"]//CommonUtils.findAllCity()
|
||||
def citys = CommonUtils.findAllCity()
|
||||
// citys.remove("上海")
|
||||
// citys.remove("福州")
|
||||
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1)
|
||||
@@ -92,15 +128,15 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
|
||||
def districts = []
|
||||
if(city == "福州"){
|
||||
if(day == 29){
|
||||
// if(day == 15){
|
||||
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList()
|
||||
}else{
|
||||
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)), keys).toList()
|
||||
}
|
||||
// }else{
|
||||
// districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)), keys).toList()
|
||||
// }
|
||||
}else{
|
||||
def q = new BasicDBObject("city", city).append("cat", "district").append("status", 0)
|
||||
// q.put("name", "越华路小区")
|
||||
// if(day == 30){
|
||||
// if(day == 2){
|
||||
districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000).toList()
|
||||
// }else{
|
||||
// q.put("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate))
|
||||
@@ -113,10 +149,10 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
def startDate = DateUtils.getFirstDayOfMonth(curDate, -1);
|
||||
def endDate = DateUtils.getLastDayOfMonth(curDate, -1);
|
||||
if(districts.size()>0){
|
||||
def weekendPriceMap = findPriceMap(city, startDate, endDate)//上个月小区均价
|
||||
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
|
||||
def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价
|
||||
def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价
|
||||
def weekendPriceMap = findPriceMap(city, startDate, endDate, null, null)//上个月小区均价
|
||||
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth, null, null)//当月小区月均价
|
||||
def lastMonthlyPriceMap = findMonthlyPrice(city, monDate, null, null)//上个月小区月均价
|
||||
def lastYearPriceMap = findMonthlyPrice(city, yoyDate, null, null)//去年小区月均价
|
||||
if(weekendPriceMap){
|
||||
running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap)
|
||||
}
|
||||
@@ -124,6 +160,44 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def calcMonthlyPriceByName(curDate, city, region, name) {
|
||||
//只有福州使用tob价格,改为所有城市都使用2B价格,除人工指导价除外
|
||||
if (!curDate) {
|
||||
curDate = new Date()
|
||||
}
|
||||
|
||||
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1)
|
||||
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(curDate, "yyyyMM"), -1, "yyyyMM"));
|
||||
def monDate = getMomDate(curMonth)
|
||||
def yoyDate = getYoyDate(curMonth)
|
||||
|
||||
def q = new BasicDBObject("city", city).append("cat", "district").append("status", 0).append("region", region).append("name", name)
|
||||
|
||||
def districts = SEAWEED_MONGO_COLL.find(q, keys).toList()
|
||||
|
||||
|
||||
|
||||
def startDate = DateUtils.getFirstDayOfMonth(curDate, -1);
|
||||
def endDate = DateUtils.getLastDayOfMonth(curDate, -1);
|
||||
if (districts.size() > 0) {
|
||||
def weekendPriceMap = findPriceMap(city, startDate, endDate, region, name)//上个月小区均价
|
||||
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth, region, name)//当月小区月均价
|
||||
def lastMonthlyPriceMap = findMonthlyPrice(city, monDate, region, name)//上个月小区月均价
|
||||
def lastYearPriceMap = findMonthlyPrice(city, yoyDate, region, name)//去年小区月均价
|
||||
if (weekendPriceMap) {
|
||||
running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def running(districts){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
@@ -284,10 +358,13 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
|
||||
def findPriceMap(city, startDate, endDate){
|
||||
def findPriceMap(city, startDate, endDate, region, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
// q.put("district_name","长春铺")
|
||||
if(name){
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
}
|
||||
q.put("weekend", new BasicDBObject('$lte', endDate).append('$gte', startDate))
|
||||
def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject().append("weekend", -1))
|
||||
def priceMap = [:]
|
||||
@@ -308,10 +385,14 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
return priceMap
|
||||
}
|
||||
|
||||
def findMonthlyPrice(city, monDate){
|
||||
def findMonthlyPrice(city, monDate, region, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("date", monDate)
|
||||
if(name){
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
}
|
||||
def dbcursor = DISTRICT_SELL_STAT.find(q)
|
||||
def priceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
|
||||
+25
-28
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
@@ -13,40 +14,36 @@ import org.bson.types.ObjectId
|
||||
|
||||
class DistrictLocationGeo {
|
||||
|
||||
final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018)
|
||||
final DBCollection allSeaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "all_seaweed_gaode_pois", "192.168.0.235", 27018)
|
||||
final DBCollection JD_SEAWEED_COLL = MongoUtils.getCollectionByName("business", "jingdong_community", "192.168.0.206",27020)
|
||||
final DBCollection JD_GEO_COLL = MongoUtils.getCollectionByName("business", "jingdong_geo", "192.168.0.206",27020)
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictLocationGeo()
|
||||
task.run()//通过经纬度,调用高德逆解析,并保存数据
|
||||
// task.run2();//通过地址,调用高德逆解析
|
||||
// def distance = GeoUtils.calculateDistance(130.573501, 45.752712, 130.573527,45.753287);
|
||||
// println distance
|
||||
|
||||
}
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\京东\\失败.xlsx",{ d->
|
||||
if(d['E']){
|
||||
def jsonObject = GaoDeApiUtils.geocode(d['E'])
|
||||
if(jsonObject){
|
||||
d["F"] = "高德"
|
||||
def dbo = new BasicDBObject()
|
||||
dbo.put("old_id", d['A'])
|
||||
dbo.put("city", d['B'])
|
||||
dbo.put("region", d['C'])
|
||||
dbo.put("name", d['D'])
|
||||
dbo.put("old_location", d['E'])
|
||||
dbo.put("formatted_address", jsonObject.regeocode.formatted_address)
|
||||
def addrComp = jsonObject.regeocode.addressComponent
|
||||
// addrComp.remove("neighborhood")
|
||||
// addrComp.remove("building")
|
||||
dbo.put("addressComponent", addrComp)
|
||||
dbo.put("pois", jsonObject.regeocode.pois)
|
||||
dbo.put("roads", jsonObject.regeocode.roads)
|
||||
dbo.put("roadinters", jsonObject.regeocode.roadinters)
|
||||
dbo.put("aois", jsonObject.regeocode.aois)
|
||||
dbo = cleanGeoData(dbo)
|
||||
dbo = convertToDBObject(dbo)
|
||||
println(dbo)
|
||||
|
||||
allSeaweedGaodePoisColl.save(dbo)
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\京东\\京东新增数据5.6万.xlsx",{ d->
|
||||
if(d['A']){
|
||||
def dbo = JD_SEAWEED_COLL.findOne(new BasicDBObject("_id", new ObjectId(d['A'])))
|
||||
if(dbo && dbo.longitude){
|
||||
def jsonObject = GaoDeApiUtils.geocode(dbo.longitude+","+dbo.latitude)
|
||||
if(jsonObject){
|
||||
d["F"] = "高德"
|
||||
// def dbo = new BasicDBObject()
|
||||
dbo.put("formatted_address", jsonObject.regeocode.formatted_address)
|
||||
def addrComp = jsonObject.regeocode.addressComponent
|
||||
dbo.put("addressComponent", addrComp)
|
||||
dbo.put("pois", jsonObject.regeocode.pois)
|
||||
dbo.put("roads", jsonObject.regeocode.roads)
|
||||
dbo.put("roadinters", jsonObject.regeocode.roadinters)
|
||||
dbo.put("aois", jsonObject.regeocode.aois)
|
||||
dbo = cleanGeoData(dbo)
|
||||
dbo = convertToDBObject(dbo)
|
||||
JD_GEO_COLL.save(dbo)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
+144
@@ -0,0 +1,144 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBList
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONNull
|
||||
|
||||
class JindDongGeoTest {
|
||||
final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018)
|
||||
final DBCollection allSeaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "all_seaweed_gaode_pois", "192.168.0.235", 27018)
|
||||
|
||||
static main(args) {
|
||||
def task = new JindDongGeoTest()
|
||||
task.run()//通过经纬度,调用高德逆解析,并保存数据
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\京东\\失败.xlsx",{ d->
|
||||
if(d['E']){
|
||||
def jsonObject = GaoDeApiUtils.geocode(d['E'])
|
||||
if(jsonObject){
|
||||
d["F"] = "高德"
|
||||
def dbo = new BasicDBObject()
|
||||
dbo.put("old_id", d['A'])
|
||||
dbo.put("city", d['B'])
|
||||
dbo.put("region", d['C'])
|
||||
dbo.put("name", d['D'])
|
||||
dbo.put("old_location", d['E'])
|
||||
dbo.put("formatted_address", jsonObject.regeocode.formatted_address)
|
||||
def addrComp = jsonObject.regeocode.addressComponent
|
||||
// addrComp.remove("neighborhood")
|
||||
// addrComp.remove("building")
|
||||
dbo.put("addressComponent", addrComp)
|
||||
dbo.put("pois", jsonObject.regeocode.pois)
|
||||
dbo.put("roads", jsonObject.regeocode.roads)
|
||||
dbo.put("roadinters", jsonObject.regeocode.roadinters)
|
||||
dbo.put("aois", jsonObject.regeocode.aois)
|
||||
dbo = cleanGeoData(dbo)
|
||||
dbo = convertToDBObject(dbo)
|
||||
println(dbo)
|
||||
|
||||
allSeaweedGaodePoisColl.save(dbo)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
}
|
||||
|
||||
static BasicDBObject convertToDBObject(Map map) {
|
||||
if (!map) return null
|
||||
|
||||
BasicDBObject dbObject = new BasicDBObject()
|
||||
map.each { key, value ->
|
||||
def convertedValue = convertValue(value)
|
||||
if (convertedValue != null) {
|
||||
dbObject.put(key, convertedValue)
|
||||
}
|
||||
}
|
||||
return dbObject
|
||||
}
|
||||
|
||||
static Object convertValue(Object value) {
|
||||
if (value == null) return null
|
||||
|
||||
switch (value) {
|
||||
case Map:
|
||||
return convertToDBObject(value as Map)
|
||||
case List:
|
||||
return convertList(value as List)
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
static BasicDBList convertList(List list) {
|
||||
BasicDBList dbList = new BasicDBList()
|
||||
list.each { item ->
|
||||
def convertedItem = convertValue(item)
|
||||
if (convertedItem != null) {
|
||||
dbList.add(convertedItem)
|
||||
}
|
||||
}
|
||||
return dbList
|
||||
}
|
||||
|
||||
static def cleanGeoData(data) {
|
||||
if (data == null) return null
|
||||
|
||||
def result = [:]
|
||||
|
||||
data.each { key, value ->
|
||||
def cleanedValue = cleanValue(value)
|
||||
if (cleanedValue != null && !isEmptyCollection(cleanedValue)) {
|
||||
result[key] = cleanedValue
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
static def cleanValue(value) {
|
||||
if (value == null) return null
|
||||
|
||||
switch (value) {
|
||||
case Map:
|
||||
def cleanedMap = [:]
|
||||
value.each { k, v ->
|
||||
def cleaned = cleanValue(v)
|
||||
if (cleaned != null && !isEmptyCollection(cleaned)) {
|
||||
cleanedMap[k] = cleaned
|
||||
}
|
||||
}
|
||||
return cleanedMap ? cleanedMap : null
|
||||
|
||||
case List:
|
||||
def cleanedList = value.collect { cleanValue(it) }
|
||||
.findAll { it != null && !isEmptyCollection(it) }
|
||||
return cleanedList ? cleanedList : null
|
||||
|
||||
case JSONNull:
|
||||
return null
|
||||
|
||||
default:
|
||||
// 检查是否是包含JSONNull的字符串表示
|
||||
if (value instanceof String && value.contains("JSONNull")) {
|
||||
return null
|
||||
}
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
static def isEmptyCollection(obj) {
|
||||
if (obj instanceof Map) return obj.isEmpty()
|
||||
if (obj instanceof List) return obj.isEmpty()
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user