添加区域财力评级
This commit is contained in:
@@ -0,0 +1,114 @@
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from goojia_utils import ElasticSearchClient
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, MongoDataFrameUpdater
|
||||
from goojia_utils import areautils
|
||||
from goojia_utils import dateutils
|
||||
from sklearn.mixture import GaussianMixture
|
||||
from sklearn.neighbors import BallTree
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
from sqlalchemy import create_engine
|
||||
import multiprocessing
|
||||
|
||||
|
||||
community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate")
|
||||
region_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "region_rate")
|
||||
region_sell_stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "region_sell_stat")
|
||||
sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent")
|
||||
|
||||
|
||||
def process_city(args):
|
||||
city = args['city']
|
||||
calc_year_month = args['calc_year_month']
|
||||
community_rate = pd.DataFrame(list(community_rate_client.find({"city": city, "date": calc_year_month})))
|
||||
region_stat = community_rate.groupby(by=["city", "region", "date"]).agg({
|
||||
"total_score" : 'mean',
|
||||
"community_quality_score" :'mean',
|
||||
"community_price_score" : 'mean',
|
||||
"location_advantage_score" : 'mean',
|
||||
"flowability_score" : 'mean',
|
||||
"living_facilities_score" : 'mean',
|
||||
"education_score" : 'mean',
|
||||
"hospital_score" : 'mean',
|
||||
"rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
|
||||
"community_quality_rating" :lambda x: x.mode().iloc[0] if not x.mode().empty else None,
|
||||
"community_price_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
|
||||
"location_advantage_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
|
||||
"flowability_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
|
||||
"living_facilities_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
|
||||
"education_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
|
||||
"hospital_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
|
||||
"resistance_coef" : 'mean'
|
||||
})
|
||||
region_stat.reset_index(inplace=True)
|
||||
for col in region_stat.columns:
|
||||
if 'score' in str(col):
|
||||
region_stat[col] = region_stat[col].apply(lambda x: round(x, 2))
|
||||
region_stat['resistance_coef'] = region_stat['resistance_coef'].apply(lambda x: int(x))
|
||||
|
||||
region_stat_keys = { "city": 1, "region": 1, "avg_price": 1, "mom": 1, "yoy": 1, "_id": 0 }
|
||||
region_sell_stat = pd.DataFrame(region_sell_stat_client.find({"city": city, "date": calc_year_month}, region_stat_keys))
|
||||
region_sell_stat['yoy'] = region_sell_stat['yoy'] * 100
|
||||
region_sell_stat['mom'] = region_sell_stat['mom'] * 100
|
||||
region_sell_stat['yoy'] = region_sell_stat['yoy'].apply(lambda x: round(x, 2))
|
||||
region_sell_stat['mom'] = region_sell_stat['mom'].apply(lambda x: round(x, 2))
|
||||
|
||||
three_month_before = int(dateutils.format_date(dateutils.parse_date(str(calc_year_month), fm='%Y%m') - relativedelta(months=3), fm='%Y%m'))
|
||||
three_month_before_region_sell_stat = pd.DataFrame(region_sell_stat_client.find({"city": city, "date": three_month_before}, region_stat_keys))
|
||||
three_month_before_region_sell_stat.rename(columns={"avg_price": "three_month_before_avg_price"}, inplace=True)
|
||||
region_sell_stat = region_sell_stat.merge(three_month_before_region_sell_stat[["city", "region", "three_month_before_avg_price"]], on=["city", "region"])
|
||||
|
||||
|
||||
|
||||
region_sell_stat['qoq'] = (region_sell_stat['avg_price'] - region_sell_stat['three_month_before_avg_price']) * 100/region_sell_stat['three_month_before_avg_price']
|
||||
region_sell_stat['qoq'] = region_sell_stat['qoq'].apply(lambda x: round(x, 2))
|
||||
region_sell_stat = region_sell_stat[["city", "region",'avg_price','mom', 'yoy', 'qoq']]
|
||||
|
||||
sqmetre_rent = pd.DataFrame(list(sqmetre_rent_client.find({"s_date": calc_year_month, "cat": "region", "city": city},{"city": 1, "region": 1, "avg_price": 1, "_id": 0})))
|
||||
sqmetre_rent = sqmetre_rent.rename(columns={"avg_price": "sqmetre_rent"})
|
||||
region_sell_stat = region_sell_stat.merge(sqmetre_rent, on=['city', 'region'], how='left')
|
||||
region_sell_stat['roi'] = round(region_sell_stat['sqmetre_rent'] * 12 / region_sell_stat['avg_price'] * 100, 2)
|
||||
region_sell_stat['rent_to_price_ratio'] =region_sell_stat['sqmetre_rent'] /region_sell_stat['avg_price']
|
||||
region_sell_stat['rent_to_price_ratio'] = region_sell_stat['rent_to_price_ratio'].apply(lambda x: round(x, 4))
|
||||
|
||||
region_stat = region_stat.merge(region_sell_stat, on=["city", "region"])
|
||||
|
||||
|
||||
dtype = {"date": int, 'resistance_coef': int}
|
||||
updater = MongoDataFrameUpdater(region_rate_client)
|
||||
updater.bulk_insert_or_update_by_id(region_stat, id_fields=['city', 'region', 'date'], dtype=dtype)
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--city', type=str, default='all')
|
||||
parser.add_argument('--month', type=str, default='')
|
||||
args = parser.parse_args()
|
||||
if args.city == 'all':
|
||||
cities = areautils.get_all_cities()
|
||||
else:
|
||||
cities = [city.strip() for city in args.city.split(',') if city.strip()]
|
||||
if args.month == '':
|
||||
calc_year_months = [int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), "%Y%m"))]
|
||||
else:
|
||||
calc_year_months = [int(calc_year_month.strip()) for calc_year_month in args.month.split(',') if calc_year_month.strip()]
|
||||
|
||||
with multiprocessing.Pool(processes=min(8, len(calc_year_months))) as pool:
|
||||
# for calc_year_month in calc_year_months:
|
||||
# args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities]
|
||||
# pool.map(process_city, args_list)
|
||||
# for calc_year_month in calc_year_months:
|
||||
# args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities]
|
||||
# pool.map(process_city, args_list)
|
||||
for city in cities:
|
||||
args_list = [{"city": city, "calc_year_month": calc_year_month} for calc_year_month in calc_year_months]
|
||||
pool.map(process_city, args_list)
|
||||
@@ -14,4 +14,8 @@ with DAG(
|
||||
task_id='community_rating',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/stat/community/community_rate.py --city "{{ dag_run.conf.get("city", "all")}}" --month "{{ dag_run.conf.get("month", "")}}"'
|
||||
)
|
||||
t1
|
||||
t2 = BashOperator(
|
||||
task_id='region_rating',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/stat/community/region_rate.py --city "{{ dag_run.conf.get("city", "all")}}" --month "{{ dag_run.conf.get("month", "")}}"'
|
||||
)
|
||||
t1 >> t2
|
||||
Reference in New Issue
Block a user