Merge branch 'script_gj' into 'master'

update

See merge request !591
This commit is contained in:
2023-04-07 17:35:02 +08:00
5 changed files with 63 additions and 12 deletions
@@ -440,7 +440,7 @@ class CcbCommonUtils {
}
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000)
println "原始:"+dbsour.count()
println "[${city}] 原始:"+dbsour.count()
def rets = []
for(def d : dbsour){
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
@@ -450,7 +450,7 @@ class CcbCommonUtils {
rets << d
}
}
println "小区量:"+rets.size()
println "[${city}] 小区量:"+rets.size()
return rets;
}
@@ -57,13 +57,12 @@ class BigdataCitypriceBatch extends BaseBatch{
curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMM").toInteger();
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMMdd").toInteger();
def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(curDate,-1), "yyyyMMdd").toInteger();
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
def lastYearDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-12), "yyyyMMdd").toInteger();
// cityRun(citys, startDate, endDate, cityCodeDataMap)
// regionRun(citys, startDate, endDate, cityCodeDataMap)
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeDataMap)
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
}
def cityRun(citys, startDate, endDate, cityCodeDataMap){
@@ -460,14 +459,14 @@ class BigdataCitypriceBatch extends BaseBatch{
def dealYoy = ''
if(allLastMonthMap){
def dealMarkupMap = allLastMonthMap.get(dist._id.toString())
if(dealMarkupMap && dealMarkupMap.deal_count){
if(dealMarkupMap && dealMarkupMap.deal_count && dealMarkupMap.deal_count>0){
def markup = (dealCount-dealMarkupMap.deal_count)*100/dealMarkupMap.deal_count
dealMarkup = DataHandleUtils.getDoubleRound(2, markup)
}
}
if(allLastYearMap){
def dealYoyMap = allLastYearMap.get(dist._id.toString())
if(dealYoyMap && dealYoyMap.deal_count){
if(dealYoyMap && dealYoyMap.deal_count && dealYoyMap.deal_count>0){
def markup = (dealCount-dealYoyMap.deal_count)*100/dealYoyMap.deal_count
dealYoy = DataHandleUtils.getDoubleRound(2, markup)
}
@@ -85,7 +85,7 @@ class ExportDistrictRongHe extends BaseBatch{
exportPgBigdataModelingBase(lastMonth)
//2月8号 临时导出苏州一年的数据
exportPgBigdataForsale("2023-01-01",DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
exportPgBigdataForsale("2023-03-01",DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
}
SSHHelper jschCommand = new SSHHelper();
@@ -1669,7 +1669,7 @@ class ExportDistrictRongHe extends BaseBatch{
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate} ";//
def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE,DEAL_COUNT,DEAL_COUNT_YOY,DEAL_COUNT_MOM,NETLIST_PRICE_YOY,NETLIST_PRICE_MOM,NETLIST_COUNT_YOY,NETLIST_COUNT_MOM,RENT_COUNT_YOY,RENT_COUNT_MOM,CITY_RANK,AREA_RANK,DISTRICT_PRICE_THAN,CITY_PRICE_THAN from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate} ";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
@@ -1697,6 +1697,19 @@ class ExportDistrictRongHe extends BaseBatch{
data << r.RDATERANGE
data << r.DATATYPE
data << r.SOURCECODE
data << r.DEAL_COUNT
data << r.DEAL_COUNT_YOY
data << r.DEAL_COUNT_MOM
data << r.NETLIST_PRICE_YOY
data << r.NETLIST_PRICE_MOM
data << r.NETLIST_COUNT_YOY
data << r.NETLIST_COUNT_MOM
data << r.RENT_COUNT_YOY
data << r.RENT_COUNT_MOM
data << r.CITY_RANK
data << r.AREA_RANK
data << r.DISTRICT_PRICE_THAN
data << r.CITY_PRICE_THAN
allRows << data
});
}
@@ -26,7 +26,7 @@ class ExportDistrictRongHeTest extends BaseBatch{
@Override
void process(Object args) {
citys = CcbCommonUtils.getRunningCitys(args)
def curDate = "20221210"//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def curDate = DateUtils.tjaformatDateByPattern(new Date()+1, "yyyyMMdd")
def new_dists = []//["350604","350605","410307","410308","450181"]
//按天导(每天一个文件夹)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
@@ -38,7 +38,7 @@ class ExportDistrictRongHeTest extends BaseBatch{
exportBigdataCommlayoutpicFjw()
exportBigdataCommunityGJRH()//
exportBigdataCommunityGJRH()/
exportBigdataBuildingFJW()//
exportBigdataCommfeeGj()//
@@ -18,13 +18,13 @@ class AddAvgpriceBatch {
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static {
citys = CommonDatautils.findAllCity()
citys = ["徐州"]//CommonDatautils.findAllCity()
println "加载城市完成:"+citys.size()
}
static main(args) {
def task = new AddAvgpriceBatch();
task.run();
task.run1();
}
def run(){
@@ -63,6 +63,45 @@ class AddAvgpriceBatch {
def run1(){
def totalCount = 0;
citys.each{city->
if(city!="全国"){
def districtKeysMap = CommonDatautils.findAllDistrictKeys(city, "district")
println "------------------------------------------------------------"
println "${city}小区量:"+districtKeysMap.size()
if(districtKeysMap){
def lastWeekendStats = queryAllDistrictPriceByWeekend(city, DateUtils.parseFromTightYYYYMMDD("2023-03-26"))
println "【2023-03-26】共计均价小区:"+lastWeekendStats.size()
//库中小区没有计算过均价的小区
def districtKeys = districtKeysMap.keySet();
def rets1 = findNoCaluteDistrict(districtKeys, lastWeekendStats);
println "【2023-03-26】共 "+rets1.size()+" 个小区无均价"
if(rets1){
for(def disKey : rets1){
def array = disKey.split("##");
def estateType2 = districtKeysMap.get(disKey)
//发重算均价命令
addPrice(array[0], array[1], array[2], estateType2, DateUtils.parseFromTightByPattern('2023-03-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-03','yyyy-MM-dd'))
}
}
}
}
}
println "共补:"+totalCount
}
def addPrice(city, region, name, estateType2,startDate, endDate){
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(city, region, name, estateType2, weekend,false,0)
}
}
def findHavePrice(lastAgoWeekendStats, rets){
def disKeys = [];
for(def key : rets){