Merge branch 'script_gj' into 'master'

福州脚本修改

See merge request !695
This commit is contained in:
2023-07-21 18:36:46 +08:00
7 changed files with 109 additions and 33 deletions
@@ -141,7 +141,7 @@ class BigdataDealBatch extends BaseBatch{
totalCount.addAndGet(1)
if(dealIds.contains(trade._id.toString())){
println "重复:"+trade._id.toString() + " "+ trade.district_name
println "重复:"+trade._id.toString() + " "+ trade.city+ " " +trade.district_name
}else{
if(source == "youda"){
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, ccbPriceMap)
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-06-26", "2023-07-20")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-06-26", "2023-07-22")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -252,7 +252,6 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name", "交发风景")
DBObject keys = new BasicDBObject();
if(fields){
@@ -270,6 +269,29 @@ class CommonDatautils {
}
public static def findAllUpdateDistricts(def city, def cat, def fields, def startMdate, def endMdate){
DBObject q = new BasicDBObject();
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
q.put("m_date", new BasicDBObject('$gte', startMdate).append('$lt', endMdate))
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000)
List<DBObject> rets = []
for(def obj : cursor){
println obj.name +" "+obj.m_date
rets << obj
}
return rets;
}
public static def findRemoveDistricts(def city, def cat, def fields, def m_date){
DBObject q = new BasicDBObject();
q.put("status", 1);
@@ -25,7 +25,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class DistrictInfoExportBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def charset = "GBK"
@@ -41,6 +41,9 @@ class DistrictInfoExportBatch extends BaseBatch{
@Override
void process(Object args) {
def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"));
def headers = ["id","city","district","name","lng","lat","property_type","build_type","households",
"property_fee","car_site_count","property_life","house_property","open_date","finish_date",
"delivery_date","floor_area","build_area","plot_ratio","landscaping_ratio","developer","property_company",
@@ -74,19 +77,35 @@ class DistrictInfoExportBatch extends BaseBatch{
def createDatas(def city){
def result = []
def districts = CommonDatautils.findAllDistricts(city, "district", null);
def districts = null
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
def day = new Date().getDate()
def startMDate = DateUtils.truncateDay(new Date()-1)
def endMDate = DateUtils.truncateDay(new Date())
if(day == 15){
districts = CommonDatautils.findAllDistricts(city, "district", null);
}else{
districts = CommonDatautils.findAllUpdateDistricts(city, "district", null, startMDate, endMDate);
}
def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1)
def startDate = DateUtils.getFirstDayOfMonth(lastDate);
// def startDate = DateUtils.getFirstDayOfMonth(lastDate);
def fzAddrMap = getFzAddress();
//因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据
saveOrUpdateFzDist(districts)
remarkRemoveDist(removeDists)
def fzDistricts = getFzDistricts(city)
if(day == 15){
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
remarkRemoveDist(removeDists)
}
def fzDistricts = null
if(day==15){
fzDistricts = getFzDistricts(city, null, null)
}else{
fzDistricts = getFzDistricts(city, startMDate, endMDate)
}
for(def rMap : fzDistricts){
//价格没有也提供
@@ -157,7 +176,6 @@ class DistrictInfoExportBatch extends BaseBatch{
udate.put(k,v)
}
udate.removeField("_id")
udate.put("m_date", new Date())
FZ_DIST_COLL.update(new BasicDBObject('_id',d.get('_id')), new BasicDBObject('$set', udate));
}else{
FZ_DIST_COLL.save(d)
@@ -184,9 +202,12 @@ class DistrictInfoExportBatch extends BaseBatch{
}
}
def getFzDistricts(def city){
def getFzDistricts(def city, def mdate, def endMDate){
def q = new BasicDBObject()
q.put("city", city)
if(mdate){
q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate))
}
def rets = []
FZ_DIST_COLL.find(q).each{it->
rets << it
@@ -24,7 +24,7 @@ import java.util.concurrent.TimeUnit
@BatchOwner(Watcher.GaoJuan)
class DistrictPriceExportBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def charset = "GBK"
@@ -41,6 +41,7 @@ class DistrictPriceExportBatch extends BaseBatch{
@Override
void process(Object args) {
def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"));
def headers = ["id","comm_id","city","district","name","avg_price","year_month"]
def ret = createNewDatas("福州")
@@ -88,9 +89,13 @@ class DistrictPriceExportBatch extends BaseBatch{
return sb.toString()
}
def getFzDistricts(def city){
def getFzDistricts(def city, def mdate, def endMDate){
def q = new BasicDBObject()
q.put("city", city)
if(mdate){
q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate))
}
def rets = []
FZ_DIST_COLL.find(q).each{it->
rets << it
@@ -100,7 +105,17 @@ class DistrictPriceExportBatch extends BaseBatch{
def createNewDatas(def city){
def districts = getFzDistricts(city)
def districts = null
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def day = new Date().getDate()
def startMDate = DateUtils.truncateDay(new Date()-1)
def endMDate = DateUtils.truncateDay(new Date())
if(day == 15){
districts = getFzDistricts(city, null, null)
}else{
districts = getFzDistricts(city, startMDate, endMDate)
}
def curPriceMap = findFzDistrictSellStat("福州",curDate)
def result = []
for(def rMap : districts){
@@ -271,9 +286,10 @@ class DistrictPriceExportBatch extends BaseBatch{
}
def saveToDB(rets){
for(def r : rets){
def q = new BasicDBObject()
q.put("month", curDate.toInteger())
q.put("month", r[6])
q.put("comm_id", r[1])
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
if(obj){
@@ -70,26 +70,43 @@ class DistrictSellStatBatch extends BaseBatch {
def monDate = getMomDate(curMonth)
def yoyDate = getYoyDate(curMonth)
def day = new Date().getDate()
def startMDate = DateUtils.truncateDay(new Date()-1)
def endMDate = DateUtils.truncateDay(new Date())
def index = 0
for(def city : citys){
index++
def districts = []
if(city == "福州"){
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList()
if(day == 2){
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList()
}else{
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)), keys).toList()
}
}else{
districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000)
def q = new BasicDBObject("city", city).append("cat", "district").append("status", 0)
if(day == 2){
districts = DbUtils.seaweed.find(q, keys).batchSize(2000)
}else{
q.put("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate))
districts = DbUtils.seaweed.find(q, keys).batchSize(2000)
}
}
print city +" "+districts.size()
def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价
def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价
if(districts && weekendPriceMap){
running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap)
println index+" "+city +" "+districts.size()+" "+ (new Date()).toString()
if(districts.size()>0){
def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价
def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价
if(weekendPriceMap){
running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap)
}
}
println index+" "+city +" "+ (new Date()).toString()
}
}
@@ -210,7 +227,7 @@ class DistrictSellStatBatch extends BaseBatch {
// };
// })
}
latch.await()
// latch.await()
}
@@ -22,11 +22,11 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\指导价\\202306测试偏差较大案例统计数据——tob价格.xlsx"
addGXDPrice(filePath)
def filePath = "C:\\Users\\gaojuan\\Desktop\\7月成交案例badcase.xlsx"
// addGXDPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
addFjwPrice(filePath)
}
@@ -107,11 +107,11 @@ class AddPriceTest {
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['B']) {
def dist = CommonDatautils.findDistrictById(d['B'])
def dist = CommonDatautils.findDistrictById(d['E'])
if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
d['I'] = gxdPrice
d['L'] = gxdPrice
}
}
})
@@ -136,10 +136,10 @@ class AddPriceTest {
def addFjwPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(!d['K']){
def dist = CommonDatautils.findDistrictById(d['B'])
def dist = CommonDatautils.findDistrictById(d['E'])
if(dist){
// d['N'] = dist.region
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-06-25","yyyy-MM-dd"))
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-07-16","yyyy-MM-dd"))
if(dbo){
// d['K'] = dbo.avg_price
d['K'] = dbo.avg_price_2b