diff --git a/dags/stat/community/region_rate.py b/dags/stat/community/region_rate.py new file mode 100644 index 0000000..5180c5b --- /dev/null +++ b/dags/stat/community/region_rate.py @@ -0,0 +1,114 @@ +import argparse +import logging +import os +import re +from datetime import datetime + +import numpy as np +import pandas as pd +from dateutil.relativedelta import relativedelta +from goojia_utils import ElasticSearchClient +from goojia_utils import MAIN_MONGO_CLIENT, MongoDataFrameUpdater +from goojia_utils import areautils +from goojia_utils import dateutils +from sklearn.mixture import GaussianMixture +from sklearn.neighbors import BallTree +from sklearn.preprocessing import MinMaxScaler +from sklearn.preprocessing import StandardScaler +from sqlalchemy import create_engine +import multiprocessing + + +community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate") +region_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "region_rate") +region_sell_stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "region_sell_stat") +sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent") + + +def process_city(args): + city = args['city'] + calc_year_month = args['calc_year_month'] + community_rate = pd.DataFrame(list(community_rate_client.find({"city": city, "date": calc_year_month}))) + region_stat = community_rate.groupby(by=["city", "region", "date"]).agg({ + "total_score" : 'mean', + "community_quality_score" :'mean', + "community_price_score" : 'mean', + "location_advantage_score" : 'mean', + "flowability_score" : 'mean', + "living_facilities_score" : 'mean', + "education_score" : 'mean', + "hospital_score" : 'mean', + "rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None, + "community_quality_rating" :lambda x: x.mode().iloc[0] if not x.mode().empty else None, + "community_price_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None, + "location_advantage_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None, + "flowability_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None, + "living_facilities_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None, + "education_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None, + "hospital_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None, + "resistance_coef" : 'mean' + }) + region_stat.reset_index(inplace=True) + for col in region_stat.columns: + if 'score' in str(col): + region_stat[col] = region_stat[col].apply(lambda x: round(x, 2)) + region_stat['resistance_coef'] = region_stat['resistance_coef'].apply(lambda x: int(x)) + + region_stat_keys = { "city": 1, "region": 1, "avg_price": 1, "mom": 1, "yoy": 1, "_id": 0 } + region_sell_stat = pd.DataFrame(region_sell_stat_client.find({"city": city, "date": calc_year_month}, region_stat_keys)) + region_sell_stat['yoy'] = region_sell_stat['yoy'] * 100 + region_sell_stat['mom'] = region_sell_stat['mom'] * 100 + region_sell_stat['yoy'] = region_sell_stat['yoy'].apply(lambda x: round(x, 2)) + region_sell_stat['mom'] = region_sell_stat['mom'].apply(lambda x: round(x, 2)) + + three_month_before = int(dateutils.format_date(dateutils.parse_date(str(calc_year_month), fm='%Y%m') - relativedelta(months=3), fm='%Y%m')) + three_month_before_region_sell_stat = pd.DataFrame(region_sell_stat_client.find({"city": city, "date": three_month_before}, region_stat_keys)) + three_month_before_region_sell_stat.rename(columns={"avg_price": "three_month_before_avg_price"}, inplace=True) + region_sell_stat = region_sell_stat.merge(three_month_before_region_sell_stat[["city", "region", "three_month_before_avg_price"]], on=["city", "region"]) + + + + region_sell_stat['qoq'] = (region_sell_stat['avg_price'] - region_sell_stat['three_month_before_avg_price']) * 100/region_sell_stat['three_month_before_avg_price'] + region_sell_stat['qoq'] = region_sell_stat['qoq'].apply(lambda x: round(x, 2)) + region_sell_stat = region_sell_stat[["city", "region",'avg_price','mom', 'yoy', 'qoq']] + + sqmetre_rent = pd.DataFrame(list(sqmetre_rent_client.find({"s_date": calc_year_month, "cat": "region", "city": city},{"city": 1, "region": 1, "avg_price": 1, "_id": 0}))) + sqmetre_rent = sqmetre_rent.rename(columns={"avg_price": "sqmetre_rent"}) + region_sell_stat = region_sell_stat.merge(sqmetre_rent, on=['city', 'region'], how='left') + region_sell_stat['roi'] = round(region_sell_stat['sqmetre_rent'] * 12 / region_sell_stat['avg_price'] * 100, 2) + region_sell_stat['rent_to_price_ratio'] =region_sell_stat['sqmetre_rent'] /region_sell_stat['avg_price'] + region_sell_stat['rent_to_price_ratio'] = region_sell_stat['rent_to_price_ratio'].apply(lambda x: round(x, 4)) + + region_stat = region_stat.merge(region_sell_stat, on=["city", "region"]) + + + dtype = {"date": int, 'resistance_coef': int} + updater = MongoDataFrameUpdater(region_rate_client) + updater.bulk_insert_or_update_by_id(region_stat, id_fields=['city', 'region', 'date'], dtype=dtype) + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--city', type=str, default='all') + parser.add_argument('--month', type=str, default='') + args = parser.parse_args() + if args.city == 'all': + cities = areautils.get_all_cities() + else: + cities = [city.strip() for city in args.city.split(',') if city.strip()] + if args.month == '': + calc_year_months = [int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), "%Y%m"))] + else: + calc_year_months = [int(calc_year_month.strip()) for calc_year_month in args.month.split(',') if calc_year_month.strip()] + + with multiprocessing.Pool(processes=min(8, len(calc_year_months))) as pool: + # for calc_year_month in calc_year_months: + # args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities] + # pool.map(process_city, args_list) + # for calc_year_month in calc_year_months: + # args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities] + # pool.map(process_city, args_list) + for city in cities: + args_list = [{"city": city, "calc_year_month": calc_year_month} for calc_year_month in calc_year_months] + pool.map(process_city, args_list) \ No newline at end of file diff --git a/dags/stat/dag_community_rating.py b/dags/stat/dag_community_rating.py index 1186583..d2da035 100644 --- a/dags/stat/dag_community_rating.py +++ b/dags/stat/dag_community_rating.py @@ -14,4 +14,8 @@ with DAG( task_id='community_rating', bash_command='python /opt/airflow/dags/repo/dags/stat/community/community_rate.py --city "{{ dag_run.conf.get("city", "all")}}" --month "{{ dag_run.conf.get("month", "")}}"' ) - t1 \ No newline at end of file + t2 = BashOperator( + task_id='region_rating', + bash_command='python /opt/airflow/dags/repo/dags/stat/community/region_rate.py --city "{{ dag_run.conf.get("city", "all")}}" --month "{{ dag_run.conf.get("month", "")}}"' + ) + t1 >> t2 \ No newline at end of file