updadte
This commit is contained in:
+47
-13
@@ -31,26 +31,60 @@ class DataMigration {
|
|||||||
public static final DBCollection STA = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com");
|
public static final DBCollection STA = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com");
|
||||||
public static final DBCollection UN_RELATION = MongoUtils.getCollectionByName("fangjia", "un_relation", "mongo.db.fangjia.com");
|
public static final DBCollection UN_RELATION = MongoUtils.getCollectionByName("fangjia", "un_relation", "mongo.db.fangjia.com");
|
||||||
static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
|
public static DBCollection COMMAND_COLL = MongoUtils.getCollectionByName("fangjia_log", "calc_command", "192.168.0.42", 52002)
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
DBCursor cursor = DEAL_PRICE_COLL.find().batchSize(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT);
|
|
||||||
DBObject dbo
|
DBCursor cursor = COMMAND_COLL.find().addOption(Bytes.QUERYOPTION_NOTIMEOUT);
|
||||||
int i =0
|
def list = []
|
||||||
while(cursor.hasNext()){
|
for(def o : cursor){
|
||||||
i++
|
def key = o.city +"##"+o.region+"##"+o.name
|
||||||
dbo = cursor.next()
|
if(!list.contains(key)){
|
||||||
if(dbo.get("total_price") instanceof Double){
|
list << key
|
||||||
dbo.put("total_price", dbo.get("total_price").toString().toDouble().toInteger())
|
def q = new BasicDBObject("city", o.city).append("region", o.region).append("district_name", o.name)
|
||||||
dbo.put("c_date",new Date())
|
DBCursor dealpriceCursor = DEAL_PRICE_COLL.find(q)
|
||||||
DEAL_PRICE_COLL.save(dbo);
|
for(def dl : dealpriceCursor){
|
||||||
|
if(dl.get("hall")){
|
||||||
|
if(dl.get("hall") instanceof String){
|
||||||
|
println dl._id.toString() +" hall 为字符串"
|
||||||
|
def dbo = new BasicDBObject()
|
||||||
|
dbo.put("hall", dl.get("hall").toString().toInteger())
|
||||||
|
DEAL_PRICE_COLL.update(new BasicDBObject("_id", dl._id), new BasicDBObject('$set', dbo))
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
if(!dl.get("area")){
|
||||||
|
println dl._id.toString() +" 面积为空"
|
||||||
|
DEAL_PRICE_COLL.remove(new BasicDBObject("_id", dl._id))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if(i%10000==0){
|
|
||||||
println i
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
// DBCursor cursor = DEAL_PRICE_COLL.find().batchSize(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT);
|
||||||
|
// DBObject dbo
|
||||||
|
// int i =0
|
||||||
|
// while(cursor.hasNext()){
|
||||||
|
// i++
|
||||||
|
// dbo = cursor.next()
|
||||||
|
// if(dbo.get("total_price") instanceof Double){
|
||||||
|
// dbo.put("total_price", dbo.get("total_price").toString().toDouble().toInteger())
|
||||||
|
// dbo.put("c_date",new Date())
|
||||||
|
// DEAL_PRICE_COLL.save(dbo);
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// if(i%10000==0){
|
||||||
|
// println i
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
}
|
}
|
||||||
// migration();
|
// migration();
|
||||||
|
|
||||||
|
|||||||
+11
-11
@@ -1,6 +1,6 @@
|
|||||||
package com.fangjia.script.ccbevaluation
|
package com.fangjia.script.ccbevaluation
|
||||||
|
|
||||||
|
import com.fangjia.script.util.DateUtils
|
||||||
import com.fangjia.script.util.ExcelUtils
|
import com.fangjia.script.util.ExcelUtils
|
||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
|||||||
* 深圳公众号成交数据录入
|
* 深圳公众号成交数据录入
|
||||||
*/
|
*/
|
||||||
|
|
||||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年5月深圳成交明细.xlsx", 0, 1)
|
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳6月份成交数据.xlsx", 0, 1)
|
||||||
|
|
||||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||||
@@ -73,18 +73,18 @@ for(def d : datas){
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
map.area = d['F'].replace("平","").toString().toDouble()
|
map.area = d['F'].toString().toDouble()
|
||||||
map.total_price = (int)(d['G'].replace("万","").toString().toDouble().toInteger()*10000)
|
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
|
||||||
map.avg_price = (int)(d['H'].replace("元/平","").toString().toDouble())
|
map.avg_price = (int)(d['H'].toString().toDouble())
|
||||||
if(d['I']){
|
if(d['I']){
|
||||||
map.sell_price = (int)(d['I'].replace("万","").toString().toDouble()*10000)
|
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
|
||||||
}
|
}
|
||||||
|
|
||||||
map.trade_date = d['J']
|
// map.trade_date = d['J']
|
||||||
// def trade_date = d['J']
|
def trade_date = d['J']
|
||||||
// if(trade_date){
|
if(trade_date){
|
||||||
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
|
map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
|
||||||
// }
|
}
|
||||||
// else{
|
// else{
|
||||||
// def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
|
// def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
|
||||||
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||||
|
|||||||
+32
-26
@@ -43,6 +43,13 @@ class DistrictSellStatBatch extends BaseBatch {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
void process(Object args) {
|
void process(Object args) {
|
||||||
|
// def months = DateUtils.getMonthList(202206)
|
||||||
|
// months = months[0..(months.size()-2)]
|
||||||
|
// println months
|
||||||
|
// for(def month : months){
|
||||||
|
// def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd")
|
||||||
|
// calcMonthlyPrice(curDate)
|
||||||
|
// }
|
||||||
calcMonthlyPrice()
|
calcMonthlyPrice()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -63,21 +70,24 @@ class DistrictSellStatBatch extends BaseBatch {
|
|||||||
return calendar.getTime().format("yyyyMM").toInteger()
|
return calendar.getTime().format("yyyyMM").toInteger()
|
||||||
}
|
}
|
||||||
|
|
||||||
def calcMonthlyPrice(){
|
def calcMonthlyPrice(curDate){
|
||||||
//只有福州使用tob价格,改为所有城市都使用2B价格,除人工指导价除外
|
//只有福州使用tob价格,改为所有城市都使用2B价格,除人工指导价除外
|
||||||
def citys = CommonUtils.findAllCity()
|
if(!curDate){
|
||||||
|
curDate = new Date()
|
||||||
|
}
|
||||||
|
def citys = ["福州"]//CommonUtils.findAllCity()
|
||||||
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1)
|
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1)
|
||||||
|
|
||||||
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date()))
|
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(curDate))
|
||||||
|
|
||||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(curDate, "yyyyMM"), -1, "yyyyMM"));
|
||||||
def monDate = getMomDate(curMonth)
|
def monDate = getMomDate(curMonth)
|
||||||
def yoyDate = getYoyDate(curMonth)
|
def yoyDate = getYoyDate(curMonth)
|
||||||
|
|
||||||
|
|
||||||
def day = new Date().getDate()
|
def day = new Date().getDate()
|
||||||
def startMDate = DateUtils.truncateDay(new Date()-7)
|
def startMDate = DateUtils.truncateDay(curDate-7)
|
||||||
def endMDate = DateUtils.truncateDay(new Date())
|
def endMDate = DateUtils.truncateDay(curDate)
|
||||||
|
|
||||||
def index = 0
|
def index = 0
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
@@ -85,7 +95,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
|||||||
index++
|
index++
|
||||||
def districts = []
|
def districts = []
|
||||||
if(city == "福州"){
|
if(city == "福州"){
|
||||||
if(day == 2){
|
if(day == 12){
|
||||||
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList()
|
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList()
|
||||||
}else{
|
}else{
|
||||||
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)), keys).toList()
|
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)), keys).toList()
|
||||||
@@ -101,10 +111,10 @@ class DistrictSellStatBatch extends BaseBatch {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
println index+" "+city +" "+districts.size()+" "+ (new Date()).toString()
|
println index+" "+city +" "+districts.size()+" "+ curDate.toString()
|
||||||
|
|
||||||
def startDate = DateUtils.getFirstDayOfMonth(new Date(), -1);
|
def startDate = DateUtils.getFirstDayOfMonth(curDate, -1);
|
||||||
def endDate = DateUtils.getLastDayOfMonth(new Date(), -1);
|
def endDate = DateUtils.getLastDayOfMonth(curDate, -1);
|
||||||
if(districts.size()>0){
|
if(districts.size()>0){
|
||||||
def weekendPriceMap = findPriceMap(city, startDate, endDate)//上个月小区均价
|
def weekendPriceMap = findPriceMap(city, startDate, endDate)//上个月小区均价
|
||||||
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
|
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
|
||||||
@@ -141,13 +151,13 @@ class DistrictSellStatBatch extends BaseBatch {
|
|||||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||||
def rets = new CopyOnWriteArrayList()
|
def rets = new CopyOnWriteArrayList()
|
||||||
|
|
||||||
districts.each{dist->
|
for(def i=0;i<districts.size();i++){
|
||||||
|
def dist = districts[i]
|
||||||
fixedThreadPool.execute(new Runnable() {
|
fixedThreadPool.execute(new Runnable() {
|
||||||
void run() {
|
void run() {
|
||||||
try {
|
try {
|
||||||
totalCount.addAndGet(1)
|
totalCount.addAndGet(1)
|
||||||
def type = dist.estate_type2=="别墅"?"别墅":"小区"
|
def key = dist.region +"##" + dist.name
|
||||||
def key = dist.region +"##" + dist.name+"##"+ type
|
|
||||||
|
|
||||||
def weekendPriceObj = weekendPriceMap.get(key)
|
def weekendPriceObj = weekendPriceMap.get(key)
|
||||||
if(weekendPriceObj){
|
if(weekendPriceObj){
|
||||||
@@ -169,8 +179,8 @@ class DistrictSellStatBatch extends BaseBatch {
|
|||||||
|
|
||||||
if(weekendPriceObj.value_type!=1){
|
if(weekendPriceObj.value_type!=1){
|
||||||
if(dist.city == "福州"){
|
if(dist.city == "福州"){
|
||||||
if(Math.abs(mom)*100>3){
|
if(Math.abs(mom)*100>8){
|
||||||
def randMarkup = nextDouble(0, 3)
|
def randMarkup = nextDouble(5, 8)
|
||||||
if(mom < 0){
|
if(mom < 0){
|
||||||
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
|
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
|
||||||
}else{
|
}else{
|
||||||
@@ -192,7 +202,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
|||||||
}
|
}
|
||||||
|
|
||||||
double yoy = 0d
|
double yoy = 0d
|
||||||
if(lastYearPriceObj){
|
if(lastYearPriceObj && lastYearPriceObj.avg_price>0){
|
||||||
yoy = DataUtil.roundDouble(((curPrice - lastYearPriceObj.avg_price)/(double)lastYearPriceObj.avg_price),4)
|
yoy = DataUtil.roundDouble(((curPrice - lastYearPriceObj.avg_price)/(double)lastYearPriceObj.avg_price),4)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -215,7 +225,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
|||||||
q.put("commid", dist._id.toString())
|
q.put("commid", dist._id.toString())
|
||||||
DISTRICT_SELL_STAT.update(q , new BasicDBObject('$set', data),true,false)
|
DISTRICT_SELL_STAT.update(q , new BasicDBObject('$set', data),true,false)
|
||||||
// DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
|
// DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
|
||||||
println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name
|
println curMonth+" [update]"+totalCount.get()+" "+ dist.city+" "+dist.name
|
||||||
}
|
}
|
||||||
}else{
|
}else{
|
||||||
def data = new BasicDBObject()
|
def data = new BasicDBObject()
|
||||||
@@ -232,7 +242,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
|||||||
data.put("mom", mom)
|
data.put("mom", mom)
|
||||||
data.put("yoy", yoy)
|
data.put("yoy", yoy)
|
||||||
DISTRICT_SELL_STAT.save(data)
|
DISTRICT_SELL_STAT.save(data)
|
||||||
println "[save]"+totalCount.get()+" "+ dist.city+" "+dist.name
|
println curMonth+" [save]"+totalCount.get()+" "+ dist.city+" "+dist.name
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -265,18 +275,18 @@ class DistrictSellStatBatch extends BaseBatch {
|
|||||||
def findPriceMap(city, startDate, endDate){
|
def findPriceMap(city, startDate, endDate){
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
q.put("city", city)
|
q.put("city", city)
|
||||||
// q.put("district_name","金福花园")
|
// q.put("district_name","长春铺")
|
||||||
q.put("weekend", new BasicDBObject('$lte', endDate).append('$gte', startDate))
|
q.put("weekend", new BasicDBObject('$lte', endDate).append('$gte', startDate))
|
||||||
def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject().append("weekend", -1))
|
def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject().append("weekend", -1))
|
||||||
def priceMap = [:]
|
def priceMap = [:]
|
||||||
for(def d : dbcursor){
|
for(def d : dbcursor){
|
||||||
if(!priceMap.containsKey(d.region +"##"+d.district_name+"##"+d.estate_type)){
|
if(!priceMap.containsKey(d.region +"##"+d.district_name)){
|
||||||
if(city == "福州"){
|
if(city == "福州"){
|
||||||
if(d.avg_price_2b && d.value_type!=1){
|
if(d.avg_price_2b && d.value_type!=1){
|
||||||
d.avg_price = d.avg_price_2b
|
d.avg_price = d.avg_price_2b
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
priceMap.put(d.region +"##"+d.district_name+"##"+d.estate_type, d)
|
priceMap.put(d.region +"##"+d.district_name, d)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return priceMap
|
return priceMap
|
||||||
@@ -289,11 +299,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
|||||||
def dbcursor = DISTRICT_SELL_STAT.find(q)
|
def dbcursor = DISTRICT_SELL_STAT.find(q)
|
||||||
def priceMap = [:]
|
def priceMap = [:]
|
||||||
for(def d : dbcursor){
|
for(def d : dbcursor){
|
||||||
if(d.estate_type=="别墅"){
|
priceMap.put(d.region +"##"+d.district_name, d)
|
||||||
priceMap.put(d.region +"##"+d.district_name+"##别墅", d)
|
|
||||||
}else{
|
|
||||||
priceMap.put(d.region +"##"+d.district_name+"##小区", d)
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
return priceMap
|
return priceMap
|
||||||
|
|||||||
+4
-4
@@ -29,11 +29,11 @@ class FzDataResultStat {
|
|||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new FzDataResultStat()
|
def task = new FzDataResultStat()
|
||||||
// task.priceMarkupRun();
|
task.priceMarkupRun();
|
||||||
// task.regionPriceTrendRun()
|
// task.regionPriceTrendRun()
|
||||||
//// task.margeExcel()
|
//// task.margeExcel()
|
||||||
// task.saveCsvData()
|
// task.saveCsvData()
|
||||||
task.testNewPrice()
|
// task.testNewPrice()
|
||||||
}
|
}
|
||||||
|
|
||||||
def margeExcel(){
|
def margeExcel(){
|
||||||
@@ -82,7 +82,7 @@ class FzDataResultStat {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
||||||
def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -4, "yyyyMM")))
|
def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")))
|
||||||
def map2 = [:]
|
def map2 = [:]
|
||||||
for(def obj : list2) {
|
for(def obj : list2) {
|
||||||
def key = obj.city + "##" + obj.region + "##" + obj.name
|
def key = obj.city + "##" + obj.region + "##" + obj.name
|
||||||
@@ -119,7 +119,7 @@ class FzDataResultStat {
|
|||||||
index++
|
index++
|
||||||
println row
|
println row
|
||||||
}
|
}
|
||||||
ExcelUtils.write("D:\\数据\\福州\\价格比对5_${curMonth}.xlsx", rows, "数据")
|
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
|
||||||
|
|
||||||
|
|
||||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
|
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import org.apache.commons.io.FileUtils
|
|||||||
import org.bson.types.ObjectId
|
import org.bson.types.ObjectId
|
||||||
|
|
||||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||||
|
DBCollection dist_stat_coll = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
import com.fangjia.core.util.HttpRequest
|
import com.fangjia.core.util.HttpRequest
|
||||||
|
|
||||||
@@ -117,15 +118,15 @@ import com.fangjia.core.util.HttpRequest
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
//
|
||||||
def findThirdPrice(city, region, name, AVERAGE_STATISTICS){
|
//def findThirdPrice(city, region, name, AVERAGE_STATISTICS){
|
||||||
def q = new BasicDBObject()
|
// def q = new BasicDBObject()
|
||||||
q.put("city", city)
|
// q.put("city", city)
|
||||||
q.put("region", region)
|
// q.put("region", region)
|
||||||
q.put("name", name)
|
// q.put("name", name)
|
||||||
q.put("s_date", 202304)
|
// q.put("s_date", 202304)
|
||||||
return AVERAGE_STATISTICS.findOne(q)
|
// return AVERAGE_STATISTICS.findOne(q)
|
||||||
}
|
//}
|
||||||
//DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
//DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
//def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv")
|
//def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv")
|
||||||
@@ -236,25 +237,32 @@ def findThirdPrice(city, region, name, AVERAGE_STATISTICS){
|
|||||||
//},0)
|
//},0)
|
||||||
|
|
||||||
|
|
||||||
def files = CommonDatautils.getAllFiles("D:\\数据\\福州\\原始数据")
|
//def files = CommonDatautils.getAllFiles("D:\\数据\\福州\\原始数据")
|
||||||
if(files){
|
//if(files){
|
||||||
for(def filepath : files){
|
// for(def filepath : files){
|
||||||
def filename = filepath.getPath()
|
// def filename = filepath.getPath()
|
||||||
def rets = read(filename);
|
// def rets = read(filename);
|
||||||
def date = filename.substring(filename.toString().lastIndexOf("_")+1, filename.toString().indexOf("."))
|
// def date = filename.substring(filename.toString().lastIndexOf("_")+1, filename.toString().indexOf("."))
|
||||||
println date +" "+rets.size()
|
// println date +" "+rets.size()
|
||||||
|
// }
|
||||||
|
//}
|
||||||
|
//
|
||||||
|
//
|
||||||
|
//def read(path){
|
||||||
|
// File file = new File(path);
|
||||||
|
// def rets = []
|
||||||
|
// file.eachLine{it->
|
||||||
|
// rets << it
|
||||||
|
// }
|
||||||
|
// return rets;
|
||||||
|
//}
|
||||||
|
|
||||||
|
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州价格比对.xlsx",{d->
|
||||||
|
def obj = dist_stat_coll.findOne(new BasicDBObject("city", d['B']).append("region",d['C']).append("district_name",d['D']).append("weekend",DateUtils.parseFromTightYYYYMMDD("2025-06-29")))
|
||||||
|
if(obj){
|
||||||
|
d['H'] = obj.avg_price_2b
|
||||||
}
|
}
|
||||||
}
|
})
|
||||||
|
|
||||||
|
|
||||||
def read(path){
|
|
||||||
File file = new File(path);
|
|
||||||
def rets = []
|
|
||||||
file.eachLine{it->
|
|
||||||
rets << it
|
|
||||||
}
|
|
||||||
return rets;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
|||||||
//福州第三方均价统计,发送钉钉消息
|
//福州第三方均价统计,发送钉钉消息
|
||||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202403.csv")
|
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202506.csv")
|
||||||
def count0 = 0
|
def count0 = 0
|
||||||
def count1 = 0
|
def count1 = 0
|
||||||
def count2 = 0
|
def count2 = 0
|
||||||
|
|||||||
+3
-3
@@ -189,7 +189,7 @@ class AddDevShapeData {
|
|||||||
|
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\623西安'
|
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\0812'
|
||||||
def dir = new File(directoryPath)
|
def dir = new File(directoryPath)
|
||||||
if (dir.isDirectory()) {
|
if (dir.isDirectory()) {
|
||||||
dir.eachFileRecurse(FileType.FILES) { file ->
|
dir.eachFileRecurse(FileType.FILES) { file ->
|
||||||
@@ -300,7 +300,7 @@ class AddDevShapeData {
|
|||||||
|
|
||||||
areaMap.count = coordinates.size()
|
areaMap.count = coordinates.size()
|
||||||
def areas = []
|
def areas = []
|
||||||
if(name.contains("陕西蓝田工业园")){
|
// if(name.contains("陕西蓝田工业园")){
|
||||||
for(def pointList : coordinates){
|
for(def pointList : coordinates){
|
||||||
index ++
|
index ++
|
||||||
def gaode_coordinates = []
|
def gaode_coordinates = []
|
||||||
@@ -343,7 +343,7 @@ class AddDevShapeData {
|
|||||||
|
|
||||||
}
|
}
|
||||||
areaMap.area_info = areas.join("\n")
|
areaMap.area_info = areas.join("\n")
|
||||||
}
|
// }
|
||||||
|
|
||||||
}else{
|
}else{
|
||||||
println file +" 有多个"
|
println file +" 有多个"
|
||||||
|
|||||||
+2
-2
@@ -33,8 +33,8 @@ class DevUpdateTestBatch {
|
|||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new DevUpdateTestBatch()
|
def task = new DevUpdateTestBatch()
|
||||||
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
|
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
|
||||||
task.addDev(path)
|
// task.addDev(path)
|
||||||
// task.addDevAddress(path)
|
task.addDevAddress(path)
|
||||||
//
|
//
|
||||||
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
|
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
|
||||||
|
|
||||||
|
|||||||
+22
-3
@@ -26,14 +26,16 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
|
|||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
def indexName = "administrative_mapping"
|
def indexName = "administrative_mapping"
|
||||||
def retMap = synchAdminiMappingTo100()
|
synchAdminiMappingTo100()
|
||||||
|
|
||||||
|
def retMap = getAdminiMappingTo100()
|
||||||
synchToEs(retMap, "192.168.0.89", indexName)
|
synchToEs(retMap, "192.168.0.89", indexName)
|
||||||
synchToEs(retMap, "192.168.0.52", indexName)
|
synchToEs(retMap, "192.168.0.52", indexName)
|
||||||
println(" 运行完成")
|
println(" 运行完成")
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def synchAdminiMappingTo100(city){
|
def synchAdminiMappingTo100(){
|
||||||
//删除旧的,再同步新的
|
//删除旧的,再同步新的
|
||||||
ADMINISTRATIVE_MAPPING_COLL_100.remove(new BasicDBObject())
|
ADMINISTRATIVE_MAPPING_COLL_100.remove(new BasicDBObject())
|
||||||
|
|
||||||
@@ -45,7 +47,7 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
|
|||||||
for(def d : dbcursor){
|
for(def d : dbcursor){
|
||||||
datas << d
|
datas << d
|
||||||
retMap.put(d._id.toString(), d)
|
retMap.put(d._id.toString(), d)
|
||||||
if (datas.size() >= 2000) {
|
if (datas.size() >=1000) {
|
||||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||||
datas.clear();
|
datas.clear();
|
||||||
}
|
}
|
||||||
@@ -59,6 +61,23 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
|
|||||||
return retMap;
|
return retMap;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def getAdminiMappingTo100(){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
|
||||||
|
def datas = []
|
||||||
|
def retMap = [:]
|
||||||
|
for(def d : dbcursor){
|
||||||
|
datas << d
|
||||||
|
retMap.put(d._id.toString(), d)
|
||||||
|
if(retMap.size()%10000==0){
|
||||||
|
print "."
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
return retMap;
|
||||||
|
}
|
||||||
|
|
||||||
def synchToEs(retMap, host, indexName){
|
def synchToEs(retMap, host, indexName){
|
||||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||||
// queryBuilder.must(QueryBuilders.termQuery("street", "喀拉玛盖镇"))
|
// queryBuilder.must(QueryBuilders.termQuery("street", "喀拉玛盖镇"))
|
||||||
|
|||||||
+38
-17
@@ -22,9 +22,9 @@ import java.util.concurrent.atomic.AtomicLong
|
|||||||
class AddPriceTest {
|
class AddPriceTest {
|
||||||
|
|
||||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||||
|
//
|
||||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
// def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||||
"postgres", "postgres", "org.postgresql.Driver")
|
// "postgres", "postgres", "org.postgresql.Driver")
|
||||||
|
|
||||||
//235为房价网内部计算均价的库
|
//235为房价网内部计算均价的库
|
||||||
def pgsqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
def pgsqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||||
@@ -38,15 +38,15 @@ class AddPriceTest {
|
|||||||
task.run();
|
task.run();
|
||||||
}
|
}
|
||||||
def run(){
|
def run(){
|
||||||
def filePath = "D:\\数据\\价格\\惠州.xlsx"
|
def filePath = "D:\\数据\\福州\\福州价格比对.xlsx"
|
||||||
// addGXDPrice(filePath)
|
// addGXDPrice(filePath)
|
||||||
// addCcbPrice(filePath)
|
addCcbPrice(filePath)
|
||||||
// addThirdPrice(filePath)
|
// addThirdPrice(filePath)
|
||||||
// addFjwPrice(filePath)
|
// addFjwPrice(filePath)
|
||||||
// exportPkPrice()
|
// exportPkPrice()
|
||||||
// addRegion()
|
// addRegion()
|
||||||
// addTradePrice(filePath)
|
// addTradePrice(filePath)
|
||||||
addThirdSamplePrice("D:\\数据\\建行\\房产估值\\指导价\\级联、国信达价格导入_0313.xlsx")
|
// addThirdSamplePrice("D:\\数据\\建行\\房产估值\\指导价\\级联、国信达价格导入_0313.xlsx")
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -218,19 +218,19 @@ class AddPriceTest {
|
|||||||
ExcelUtils.multiThreadingModify(filepath,{ d->
|
ExcelUtils.multiThreadingModify(filepath,{ d->
|
||||||
if(d['A']){
|
if(d['A']){
|
||||||
def cityId = cityCodeDataMap.get(d['B'])
|
def cityId = cityCodeDataMap.get(d['B'])
|
||||||
def map = findAvgPrice(d['A'], cityId)
|
// def map = findAvgPrice(d['A'], cityId)
|
||||||
if(map){
|
// if(map){
|
||||||
d['I'] = map.avg_price
|
// d['I'] = map.avg_price
|
||||||
d['J'] = map.month
|
// d['J'] = map.month
|
||||||
// d['H'] = map.calc_type
|
//// d['H'] = map.calc_type
|
||||||
// d['AM'] = map.credit_value
|
//// d['AM'] = map.credit_value
|
||||||
}
|
// }
|
||||||
|
|
||||||
def newPriceMap = findNewFjwAvgPrice(d['A'], cityId)
|
def newPriceMap = findNewFjwAvgPrice2(d['A'], cityId, 202506)
|
||||||
if(newPriceMap){
|
if(newPriceMap){
|
||||||
d['G'] = newPriceMap.avg_price
|
d['I'] = newPriceMap.avg_price
|
||||||
d['H'] = newPriceMap.month
|
// d['H'] = newPriceMap.month
|
||||||
d['H'] = map.calc_type
|
// d['H'] = map.calc_type
|
||||||
// d['AM'] = map.credit_value
|
// d['AM'] = map.credit_value
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -277,6 +277,27 @@ class AddPriceTest {
|
|||||||
return map
|
return map
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def findNewFjwAvgPrice2(def commid, def cityid, def sdate){
|
||||||
|
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=${sdate}"
|
||||||
|
def map = [:]
|
||||||
|
pgsqlInstance235.eachRow(sql, { r ->
|
||||||
|
map.avg_price = r.gujia_bds_clc_avprc
|
||||||
|
map.type = r.gujia_bds_clmtd
|
||||||
|
map.month = r.gujiabds_clc_yrmo_val
|
||||||
|
map.credit_value = r.gujia_bds_cfdc_dgr
|
||||||
|
if(r.gujia_bds_clmtd == 1){
|
||||||
|
map.calc_type = "人工估值"
|
||||||
|
}else if(r.gujia_bds_clmtd == 2){
|
||||||
|
map.calc_type = "样本计算"
|
||||||
|
}else if(r.gujia_bds_clmtd == 3){
|
||||||
|
map.calc_type = "神经网络"
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return map
|
||||||
|
}
|
||||||
|
|
||||||
def addThirdPrice(filepath){
|
def addThirdPrice(filepath){
|
||||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||||
if (!d['X']) {
|
if (!d['X']) {
|
||||||
|
|||||||
+2
-1
@@ -296,7 +296,8 @@ class AddressUtil {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static void main(String[] args){
|
public static void main(String[] args){
|
||||||
def addr = AddressUtil.formatAddress("福州", "北三环路201号琴亭信和小区")
|
AddressUtil.initCityMap()
|
||||||
|
// def addr = AddressUtil.formatAddress("福州", "北三环路201号琴亭信和小区")
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user