添加区域财力评级

This commit is contained in:
2025-07-06 11:21:38 +08:00
parent e31aaf6794
commit 7f286359e1
2 changed files with 119 additions and 1 deletions
+114
View File
@@ -0,0 +1,114 @@
import argparse
import logging
import os
import re
from datetime import datetime
import numpy as np
import pandas as pd
from dateutil.relativedelta import relativedelta
from goojia_utils import ElasticSearchClient
from goojia_utils import MAIN_MONGO_CLIENT, MongoDataFrameUpdater
from goojia_utils import areautils
from goojia_utils import dateutils
from sklearn.mixture import GaussianMixture
from sklearn.neighbors import BallTree
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sqlalchemy import create_engine
import multiprocessing
community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate")
region_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "region_rate")
region_sell_stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "region_sell_stat")
sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent")
def process_city(args):
city = args['city']
calc_year_month = args['calc_year_month']
community_rate = pd.DataFrame(list(community_rate_client.find({"city": city, "date": calc_year_month})))
region_stat = community_rate.groupby(by=["city", "region", "date"]).agg({
"total_score" : 'mean',
"community_quality_score" :'mean',
"community_price_score" : 'mean',
"location_advantage_score" : 'mean',
"flowability_score" : 'mean',
"living_facilities_score" : 'mean',
"education_score" : 'mean',
"hospital_score" : 'mean',
"rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
"community_quality_rating" :lambda x: x.mode().iloc[0] if not x.mode().empty else None,
"community_price_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
"location_advantage_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
"flowability_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
"living_facilities_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
"education_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
"hospital_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
"resistance_coef" : 'mean'
})
region_stat.reset_index(inplace=True)
for col in region_stat.columns:
if 'score' in str(col):
region_stat[col] = region_stat[col].apply(lambda x: round(x, 2))
region_stat['resistance_coef'] = region_stat['resistance_coef'].apply(lambda x: int(x))
region_stat_keys = { "city": 1, "region": 1, "avg_price": 1, "mom": 1, "yoy": 1, "_id": 0 }
region_sell_stat = pd.DataFrame(region_sell_stat_client.find({"city": city, "date": calc_year_month}, region_stat_keys))
region_sell_stat['yoy'] = region_sell_stat['yoy'] * 100
region_sell_stat['mom'] = region_sell_stat['mom'] * 100
region_sell_stat['yoy'] = region_sell_stat['yoy'].apply(lambda x: round(x, 2))
region_sell_stat['mom'] = region_sell_stat['mom'].apply(lambda x: round(x, 2))
three_month_before = int(dateutils.format_date(dateutils.parse_date(str(calc_year_month), fm='%Y%m') - relativedelta(months=3), fm='%Y%m'))
three_month_before_region_sell_stat = pd.DataFrame(region_sell_stat_client.find({"city": city, "date": three_month_before}, region_stat_keys))
three_month_before_region_sell_stat.rename(columns={"avg_price": "three_month_before_avg_price"}, inplace=True)
region_sell_stat = region_sell_stat.merge(three_month_before_region_sell_stat[["city", "region", "three_month_before_avg_price"]], on=["city", "region"])
region_sell_stat['qoq'] = (region_sell_stat['avg_price'] - region_sell_stat['three_month_before_avg_price']) * 100/region_sell_stat['three_month_before_avg_price']
region_sell_stat['qoq'] = region_sell_stat['qoq'].apply(lambda x: round(x, 2))
region_sell_stat = region_sell_stat[["city", "region",'avg_price','mom', 'yoy', 'qoq']]
sqmetre_rent = pd.DataFrame(list(sqmetre_rent_client.find({"s_date": calc_year_month, "cat": "region", "city": city},{"city": 1, "region": 1, "avg_price": 1, "_id": 0})))
sqmetre_rent = sqmetre_rent.rename(columns={"avg_price": "sqmetre_rent"})
region_sell_stat = region_sell_stat.merge(sqmetre_rent, on=['city', 'region'], how='left')
region_sell_stat['roi'] = round(region_sell_stat['sqmetre_rent'] * 12 / region_sell_stat['avg_price'] * 100, 2)
region_sell_stat['rent_to_price_ratio'] =region_sell_stat['sqmetre_rent'] /region_sell_stat['avg_price']
region_sell_stat['rent_to_price_ratio'] = region_sell_stat['rent_to_price_ratio'].apply(lambda x: round(x, 4))
region_stat = region_stat.merge(region_sell_stat, on=["city", "region"])
dtype = {"date": int, 'resistance_coef': int}
updater = MongoDataFrameUpdater(region_rate_client)
updater.bulk_insert_or_update_by_id(region_stat, id_fields=['city', 'region', 'date'], dtype=dtype)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--city', type=str, default='all')
parser.add_argument('--month', type=str, default='')
args = parser.parse_args()
if args.city == 'all':
cities = areautils.get_all_cities()
else:
cities = [city.strip() for city in args.city.split(',') if city.strip()]
if args.month == '':
calc_year_months = [int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), "%Y%m"))]
else:
calc_year_months = [int(calc_year_month.strip()) for calc_year_month in args.month.split(',') if calc_year_month.strip()]
with multiprocessing.Pool(processes=min(8, len(calc_year_months))) as pool:
# for calc_year_month in calc_year_months:
# args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities]
# pool.map(process_city, args_list)
# for calc_year_month in calc_year_months:
# args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities]
# pool.map(process_city, args_list)
for city in cities:
args_list = [{"city": city, "calc_year_month": calc_year_month} for calc_year_month in calc_year_months]
pool.map(process_city, args_list)
+5 -1
View File
@@ -14,4 +14,8 @@ with DAG(
task_id='community_rating',
bash_command='python /opt/airflow/dags/repo/dags/stat/community/community_rate.py --city "{{ dag_run.conf.get("city", "all")}}" --month "{{ dag_run.conf.get("month", "")}}"'
)
t1
t2 = BashOperator(
task_id='region_rating',
bash_command='python /opt/airflow/dags/repo/dags/stat/community/region_rate.py --city "{{ dag_run.conf.get("city", "all")}}" --month "{{ dag_run.conf.get("month", "")}}"'
)
t1 >> t2