import argparse import logging import os import re from datetime import datetime import numpy as np import pandas as pd from dateutil.relativedelta import relativedelta from goojia_utils import ElasticSearchClient from goojia_utils import MAIN_MONGO_CLIENT, MongoDataFrameUpdater from goojia_utils import areautils from goojia_utils import dateutils from sklearn.mixture import GaussianMixture from sklearn.neighbors import BallTree from sklearn.preprocessing import MinMaxScaler from sklearn.preprocessing import StandardScaler from sqlalchemy import create_engine import multiprocessing community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate") region_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "region_rate") region_sell_stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "region_sell_stat") sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent") def process_city(args): city = args['city'] calc_year_month = args['calc_year_month'] community_rate = pd.DataFrame(list(community_rate_client.find({"city": city, "date": calc_year_month}))) region_stat = community_rate.groupby(by=["city", "region", "date"]).agg({ "total_score" : 'mean', "community_quality_score" :'mean', "community_price_score" : 'mean', "location_advantage_score" : 'mean', "flowability_score" : 'mean', "living_facilities_score" : 'mean', "education_score" : 'mean', "hospital_score" : 'mean', "rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None, "community_quality_rating" :lambda x: x.mode().iloc[0] if not x.mode().empty else None, "community_price_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None, "location_advantage_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None, "flowability_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None, "living_facilities_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None, "education_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None, "hospital_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None, "resistance_coef" : 'mean' }) region_stat.reset_index(inplace=True) for col in region_stat.columns: if 'score' in str(col): region_stat[col] = region_stat[col].apply(lambda x: round(x, 2)) region_stat['resistance_coef'] = region_stat['resistance_coef'].apply(lambda x: int(x)) region_stat_keys = { "city": 1, "region": 1, "avg_price": 1, "mom": 1, "yoy": 1, "_id": 0 } region_sell_stat = pd.DataFrame(region_sell_stat_client.find({"city": city, "date": calc_year_month}, region_stat_keys)) region_sell_stat['yoy'] = region_sell_stat['yoy'] * 100 region_sell_stat['mom'] = region_sell_stat['mom'] * 100 region_sell_stat['yoy'] = region_sell_stat['yoy'].apply(lambda x: round(x, 2)) region_sell_stat['mom'] = region_sell_stat['mom'].apply(lambda x: round(x, 2)) three_month_before = int(dateutils.format_date(dateutils.parse_date(str(calc_year_month), fm='%Y%m') - relativedelta(months=3), fm='%Y%m')) three_month_before_region_sell_stat = pd.DataFrame(region_sell_stat_client.find({"city": city, "date": three_month_before}, region_stat_keys)) three_month_before_region_sell_stat.rename(columns={"avg_price": "three_month_before_avg_price"}, inplace=True) region_sell_stat = region_sell_stat.merge(three_month_before_region_sell_stat[["city", "region", "three_month_before_avg_price"]], on=["city", "region"]) region_sell_stat['qoq'] = (region_sell_stat['avg_price'] - region_sell_stat['three_month_before_avg_price']) * 100/region_sell_stat['three_month_before_avg_price'] region_sell_stat['qoq'] = region_sell_stat['qoq'].apply(lambda x: round(x, 2)) region_sell_stat = region_sell_stat[["city", "region",'avg_price','mom', 'yoy', 'qoq']] sqmetre_rent = pd.DataFrame(list(sqmetre_rent_client.find({"s_date": calc_year_month, "cat": "region", "city": city},{"city": 1, "region": 1, "avg_price": 1, "_id": 0}))) sqmetre_rent = sqmetre_rent.rename(columns={"avg_price": "sqmetre_rent"}) region_sell_stat = region_sell_stat.merge(sqmetre_rent, on=['city', 'region'], how='left') region_sell_stat['roi'] = round(region_sell_stat['sqmetre_rent'] * 12 / region_sell_stat['avg_price'] * 100, 2) region_sell_stat['rent_to_price_ratio'] =region_sell_stat['sqmetre_rent'] /region_sell_stat['avg_price'] region_sell_stat['rent_to_price_ratio'] = region_sell_stat['rent_to_price_ratio'].apply(lambda x: round(x, 4)) region_stat = region_stat.merge(region_sell_stat, on=["city", "region"]) dtype = {"date": int, 'resistance_coef': int} updater = MongoDataFrameUpdater(region_rate_client) updater.bulk_insert_or_update_by_id(region_stat, id_fields=['city', 'region', 'date'], dtype=dtype) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--city', type=str, default='all') parser.add_argument('--month', type=str, default='') args = parser.parse_args() if args.city == 'all': cities = areautils.get_all_cities() else: cities = [city.strip() for city in args.city.split(',') if city.strip()] if args.month == '': calc_year_months = [int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), "%Y%m"))] else: calc_year_months = [int(calc_year_month.strip()) for calc_year_month in args.month.split(',') if calc_year_month.strip()] with multiprocessing.Pool(processes=min(8, len(calc_year_months))) as pool: # for calc_year_month in calc_year_months: # args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities] # pool.map(process_city, args_list) # for calc_year_month in calc_year_months: # args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities] # pool.map(process_city, args_list) for city in cities: args_list = [{"city": city, "calc_year_month": calc_year_month} for calc_year_month in calc_year_months] pool.map(process_city, args_list)