121 lines
6.5 KiB
Python
121 lines
6.5 KiB
Python
import argparse
|
|
import logging
|
|
import os
|
|
import re
|
|
from datetime import datetime
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
from dateutil.relativedelta import relativedelta
|
|
from goojia_utils import ElasticSearchClient
|
|
from goojia_utils import MAIN_MONGO_CLIENT, MongoDataFrameUpdater
|
|
from goojia_utils import areautils
|
|
from goojia_utils import dateutils
|
|
from sklearn.mixture import GaussianMixture
|
|
from sklearn.neighbors import BallTree
|
|
from sklearn.preprocessing import MinMaxScaler
|
|
from sklearn.preprocessing import StandardScaler
|
|
from sqlalchemy import create_engine
|
|
import multiprocessing
|
|
|
|
|
|
community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate")
|
|
region_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "region_rate")
|
|
region_sell_stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "region_sell_stat")
|
|
sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent")
|
|
|
|
|
|
def process_city(args):
|
|
city = args['city']
|
|
calc_year_month = args['calc_year_month']
|
|
community_rate = pd.DataFrame(list(community_rate_client.find({"city": city, "date": calc_year_month})))
|
|
if community_rate.empty:
|
|
logging.warning(f"{city}楼盘评级数据为0")
|
|
return
|
|
region_stat = community_rate.groupby(by=["city", "region", "date"]).agg({
|
|
"total_score" : 'mean',
|
|
"community_quality_score" :'mean',
|
|
"community_price_score" : 'mean',
|
|
"location_advantage_score" : 'mean',
|
|
"flowability_score" : 'mean',
|
|
"living_facilities_score" : 'mean',
|
|
"education_score" : 'mean',
|
|
"hospital_score" : 'mean',
|
|
"rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
|
|
"community_quality_rating" :lambda x: x.mode().iloc[0] if not x.mode().empty else None,
|
|
"community_price_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
|
|
"location_advantage_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
|
|
"flowability_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
|
|
"living_facilities_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
|
|
"education_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
|
|
"hospital_rating" : lambda x: x.mode().iloc[0] if not x.mode().empty else None,
|
|
"resistance_coef" : 'mean'
|
|
})
|
|
region_stat.reset_index(inplace=True)
|
|
for col in region_stat.columns:
|
|
if 'score' in str(col):
|
|
region_stat[col] = region_stat[col].apply(lambda x: round(x, 2))
|
|
region_stat['resistance_coef'] = region_stat['resistance_coef'].apply(lambda x: int(x))
|
|
region_stat_keys = { "city": 1, "region": 1, "avg_price": 1, "mom": 1, "yoy": 1, "_id": 0 }
|
|
region_sell_stat = pd.DataFrame(region_sell_stat_client.find({"city": city, "date": calc_year_month}, region_stat_keys))
|
|
if region_sell_stat.empty:
|
|
return
|
|
region_sell_stat['yoy'] = region_sell_stat['yoy'] * 100
|
|
region_sell_stat['mom'] = region_sell_stat['mom'] * 100
|
|
region_sell_stat['yoy'] = region_sell_stat['yoy'].apply(lambda x: round(x, 2))
|
|
region_sell_stat['mom'] = region_sell_stat['mom'].apply(lambda x: round(x, 2))
|
|
|
|
three_month_before = int(dateutils.format_date(dateutils.parse_date(str(calc_year_month), fm='%Y%m') - relativedelta(months=3), fm='%Y%m'))
|
|
three_month_before_region_sell_stat = pd.DataFrame(region_sell_stat_client.find({"city": city, "date": three_month_before}, region_stat_keys))
|
|
three_month_before_region_sell_stat.rename(columns={"avg_price": "three_month_before_avg_price"}, inplace=True)
|
|
region_sell_stat = region_sell_stat.merge(three_month_before_region_sell_stat[["city", "region", "three_month_before_avg_price"]], on=["city", "region"])
|
|
|
|
|
|
|
|
region_sell_stat['qoq'] = (region_sell_stat['avg_price'] - region_sell_stat['three_month_before_avg_price']) * 100/region_sell_stat['three_month_before_avg_price']
|
|
region_sell_stat['qoq'] = region_sell_stat['qoq'].apply(lambda x: round(x, 2))
|
|
region_sell_stat = region_sell_stat[["city", "region",'avg_price','mom', 'yoy', 'qoq']]
|
|
|
|
sqmetre_rent = pd.DataFrame(list(sqmetre_rent_client.find({"s_date": calc_year_month, "cat": "region", "city": city},{"city": 1, "region": 1, "avg_price": 1, "_id": 0})))
|
|
sqmetre_rent = sqmetre_rent.rename(columns={"avg_price": "sqmetre_rent"})
|
|
region_sell_stat = region_sell_stat.merge(sqmetre_rent, on=['city', 'region'], how='left')
|
|
region_sell_stat['roi'] = round(region_sell_stat['sqmetre_rent'] * 12 / region_sell_stat['avg_price'] * 100, 2)
|
|
region_sell_stat['rent_to_price_ratio'] =region_sell_stat['sqmetre_rent'] /region_sell_stat['avg_price']
|
|
region_sell_stat['rent_to_price_ratio'] = region_sell_stat['rent_to_price_ratio'].apply(lambda x: round(x, 4))
|
|
|
|
region_stat = region_stat.merge(region_sell_stat, on=["city", "region"])
|
|
|
|
|
|
dtype = {"date": int, 'resistance_coef': int}
|
|
updater = MongoDataFrameUpdater(region_rate_client)
|
|
updater.bulk_insert_or_update_by_id(region_stat, id_fields=['city', 'region', 'date'], dtype=dtype)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--city', type=str, default='all')
|
|
parser.add_argument('--month', type=str, default='')
|
|
args = parser.parse_args()
|
|
if args.city == 'all':
|
|
cities = areautils.get_all_cities()
|
|
else:
|
|
cities = [city.strip() for city in args.city.split(',') if city.strip()]
|
|
if args.month == '':
|
|
calc_year_months = [int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), "%Y%m"))]
|
|
else:
|
|
calc_year_months = [int(calc_year_month.strip()) for calc_year_month in args.month.split(',') if calc_year_month.strip()]
|
|
# for city in cities:
|
|
# args_list = [{"city": city, "calc_year_month": calc_year_month} for calc_year_month in calc_year_months]
|
|
# for args in args_list:
|
|
# process_city(args)
|
|
with multiprocessing.Pool(processes=min(8, len(calc_year_months))) as pool:
|
|
# for calc_year_month in calc_year_months:
|
|
# args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities]
|
|
# pool.map(process_city, args_list)
|
|
# for calc_year_month in calc_year_months:
|
|
# args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities]
|
|
# pool.map(process_city, args_list)
|
|
for city in cities:
|
|
args_list = [{"city": city, "calc_year_month": calc_year_month} for calc_year_month in calc_year_months]
|
|
pool.map(process_city, args_list) |