Files
airflow/dags/stat/community/region_rate.py
T
2025-10-14 08:22:52 +08:00

118 lines
6.4 KiB
Python

import argparse
import logging
import os
import re
from datetime import datetime
import numpy as np
import pandas as pd
from dateutil.relativedelta import relativedelta
from goojia_utils import ElasticSearchClient
from goojia_utils import MAIN_MONGO_CLIENT, MongoDataFrameUpdater
from goojia_utils import areautils
from goojia_utils import dateutils
from sklearn.mixture import GaussianMixture
from sklearn.neighbors import BallTree
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sqlalchemy import create_engine
import multiprocessing
community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate")
region_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "region_rate")
region_sell_stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "region_sell_stat")
sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent")
def process_city(args):
city = args['city']
calc_year_month = args['calc_year_month']
community_rate = pd.DataFrame(list(community_rate_client.find({"city": city, "date": calc_year_month})))
region_stat = community_rate.groupby(by=["city", "region", "date"]).agg({
"total_score" : 'mean',
"community_quality_score" :'mean',
"community_price_score" : 'mean',
"location_advantage_score" : 'mean',
"flowability_score" : 'mean',
"living_facilities_score" : 'mean',
"education_score" : 'mean',
"hospital_score" : 'mean',
"rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
"community_quality_rating" :lambda x: x.mode().iloc[0] if not x.mode().empty else None,
"community_price_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
"location_advantage_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
"flowability_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
"living_facilities_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
"education_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
"hospital_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
"resistance_coef" : 'mean'
})
region_stat.reset_index(inplace=True)
for col in region_stat.columns:
if 'score' in str(col):
region_stat[col] = region_stat[col].apply(lambda x: round(x, 2))
region_stat['resistance_coef'] = region_stat['resistance_coef'].apply(lambda x: int(x))
region_stat_keys = { "city": 1, "region": 1, "avg_price": 1, "mom": 1, "yoy": 1, "_id": 0 }
region_sell_stat = pd.DataFrame(region_sell_stat_client.find({"city": city, "date": calc_year_month}, region_stat_keys))
if region_sell_stat.empty:
return
region_sell_stat['yoy'] = region_sell_stat['yoy'] * 100
region_sell_stat['mom'] = region_sell_stat['mom'] * 100
region_sell_stat['yoy'] = region_sell_stat['yoy'].apply(lambda x: round(x, 2))
region_sell_stat['mom'] = region_sell_stat['mom'].apply(lambda x: round(x, 2))
three_month_before = int(dateutils.format_date(dateutils.parse_date(str(calc_year_month), fm='%Y%m') - relativedelta(months=3), fm='%Y%m'))
three_month_before_region_sell_stat = pd.DataFrame(region_sell_stat_client.find({"city": city, "date": three_month_before}, region_stat_keys))
three_month_before_region_sell_stat.rename(columns={"avg_price": "three_month_before_avg_price"}, inplace=True)
region_sell_stat = region_sell_stat.merge(three_month_before_region_sell_stat[["city", "region", "three_month_before_avg_price"]], on=["city", "region"])
region_sell_stat['qoq'] = (region_sell_stat['avg_price'] - region_sell_stat['three_month_before_avg_price']) * 100/region_sell_stat['three_month_before_avg_price']
region_sell_stat['qoq'] = region_sell_stat['qoq'].apply(lambda x: round(x, 2))
region_sell_stat = region_sell_stat[["city", "region",'avg_price','mom', 'yoy', 'qoq']]
sqmetre_rent = pd.DataFrame(list(sqmetre_rent_client.find({"s_date": calc_year_month, "cat": "region", "city": city},{"city": 1, "region": 1, "avg_price": 1, "_id": 0})))
sqmetre_rent = sqmetre_rent.rename(columns={"avg_price": "sqmetre_rent"})
region_sell_stat = region_sell_stat.merge(sqmetre_rent, on=['city', 'region'], how='left')
region_sell_stat['roi'] = round(region_sell_stat['sqmetre_rent'] * 12 / region_sell_stat['avg_price'] * 100, 2)
region_sell_stat['rent_to_price_ratio'] =region_sell_stat['sqmetre_rent'] /region_sell_stat['avg_price']
region_sell_stat['rent_to_price_ratio'] = region_sell_stat['rent_to_price_ratio'].apply(lambda x: round(x, 4))
region_stat = region_stat.merge(region_sell_stat, on=["city", "region"])
dtype = {"date": int, 'resistance_coef': int}
updater = MongoDataFrameUpdater(region_rate_client)
updater.bulk_insert_or_update_by_id(region_stat, id_fields=['city', 'region', 'date'], dtype=dtype)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--city', type=str, default='all')
parser.add_argument('--month', type=str, default='')
args = parser.parse_args()
if args.city == 'all':
cities = areautils.get_all_cities()
else:
cities = [city.strip() for city in args.city.split(',') if city.strip()]
if args.month == '':
calc_year_months = [int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), "%Y%m"))]
else:
calc_year_months = [int(calc_year_month.strip()) for calc_year_month in args.month.split(',') if calc_year_month.strip()]
# for city in cities:
# args_list = [{"city": city, "calc_year_month": calc_year_month} for calc_year_month in calc_year_months]
# for args in args_list:
# process_city(args)
with multiprocessing.Pool(processes=min(8, len(calc_year_months))) as pool:
# for calc_year_month in calc_year_months:
# args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities]
# pool.map(process_city, args_list)
# for calc_year_month in calc_year_months:
# args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities]
# pool.map(process_city, args_list)
for city in cities:
args_list = [{"city": city, "calc_year_month": calc_year_month} for calc_year_month in calc_year_months]
pool.map(process_city, args_list)