diff --git a/.airflowignore b/.airflowignore index 851ab04..1df32a4 100644 --- a/.airflowignore +++ b/.airflowignore @@ -1 +1,2 @@ -poi/business/.* \ No newline at end of file +poi/business/.* +stat/evaluate/.* \ No newline at end of file diff --git a/dags/stat/__init__.py b/dags/stat/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/stat/evaluate/__init__.py b/dags/stat/evaluate/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/stat/evaluate/data/估值数据统计.xlsx b/dags/stat/evaluate/data/估值数据统计.xlsx new file mode 100644 index 0000000..275455d Binary files /dev/null and b/dags/stat/evaluate/data/估值数据统计.xlsx differ diff --git a/dags/stat/evaluate/district_count_thirdavg_deal.py b/dags/stat/evaluate/district_count_thirdavg_deal.py new file mode 100644 index 0000000..1f1e1f7 --- /dev/null +++ b/dags/stat/evaluate/district_count_thirdavg_deal.py @@ -0,0 +1,123 @@ +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils +import pandas as pd +import datetime +import logging + +""" +统计小区数量、近半年第三方均价、近半年成交信息 +""" +log = logging.getLogger(__name__) + +df = pd.read_excel('data/估值数据统计.xlsx') +year_month_day = dateutils.format_date(datetime.datetime.now()) +aim_file_name = 'data/估值数据统计结果' + year_month_day + '.xlsx' + +seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +es206_client = ElasticSearchClient(host='192.168.0.206') +district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat') # 成交 +average_statistics_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'average_statistics') # 第三方均价 +deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price') + +seaweed_df = pd.DataFrame( + list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'alias': 1}) + .batch_size(10000))) + +cities = df[['城市']].drop_duplicates(keep='first')['城市'].values.tolist() + + +def count_stat(): + """ + 统计小区数量 + """ + result_df = pd.DataFrame(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0}}, + {'$group': {'_id': '$city', 'community_count': {'$sum': 1}}} + ])) + result_rename_df = result_df.rename(columns={'_id': '城市', 'community_count': '小区量'}) + new_df = pd.merge(df, result_rename_df, how='left', on=['城市']) + new_df.head(3) + new_df.to_excel(aim_file_name, index=False) + + +def third_avg_stat(): + """ + 近半年有第三方均价小区数量及占比 + """ + new_df = pd.read_excel('data/估值数据统计结果' + year_month_day + '.xlsx') + s_date = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-180), fm='%Y%m') + average_statistics_df = pd.DataFrame( + list(average_statistics_conn.find({'s_date': {'$gte': int(s_date)}, 'category': 'district'}).batch_size(10000))) + average_statistics_new_df = average_statistics_df[ + ['city', 'name', 'region', '58tc_esf_price', 'cityhouse_esf_price', 'soufang_esf_price', 'lianjia_esf_price', + '5i5j_esf_price', 'maitian_esf_price', '365house_esf_price', 'beike_esf_price', 'qfang_esf_price', + 'fangtan_esf_price', + 'tuboshi_esf_price', 'zhugezhaofang_esf_price', 'zhongyuan_esf_price', 'jiwu_esf_price', 'anjuke_esf_price']] + average_statistics_new_df.fillna(0, inplace=True) + average_statistics_new_df.loc[:, 'has_esf_price'] = average_statistics_new_df['58tc_esf_price'] + \ + average_statistics_new_df['cityhouse_esf_price'] + \ + average_statistics_new_df['soufang_esf_price'] + \ + average_statistics_new_df['lianjia_esf_price'] + \ + average_statistics_new_df['5i5j_esf_price'] + \ + average_statistics_new_df['maitian_esf_price'] + \ + average_statistics_new_df['365house_esf_price'] + \ + average_statistics_new_df['beike_esf_price'] + \ + average_statistics_new_df['qfang_esf_price'] + \ + average_statistics_new_df['fangtan_esf_price'] + \ + average_statistics_new_df['tuboshi_esf_price'] + \ + average_statistics_new_df['zhugezhaofang_esf_price'] + \ + average_statistics_new_df['zhongyuan_esf_price'] + \ + average_statistics_new_df['jiwu_esf_price'] + \ + average_statistics_new_df['anjuke_esf_price'] + average_statistics_new_df.loc[:, ['flag']] = average_statistics_new_df['has_esf_price'].apply( + lambda x: True if x > 0 else False) + average_statistics_new_df = average_statistics_new_df[average_statistics_new_df['flag'] == True] + average_statistics_new_df = average_statistics_new_df[['city', 'region', 'name']] + average_statistics_new_df = average_statistics_new_df.drop_duplicates(keep='first') + average_statistics_new_df['近半年有第三方均价小区数量'] = average_statistics_new_df.groupby('city')['city'].transform('size') + average_statistics_new_df = average_statistics_new_df[['city', '近半年有第三方均价小区数量']].drop_duplicates(keep='first') + average_statistics_new_df = average_statistics_new_df.rename(columns={'city': '城市'}) + new_df = pd.merge(new_df, average_statistics_new_df, how='left', on=['城市']) + new_df['近半年有第三方均价小区占比'] = round(new_df['近半年有第三方均价小区数量'] * 100 / new_df['小区量'], 2) + new_df.to_excel(aim_file_name, index=False) + pass + + +def deal_stat(): + """ + 近半年成交量及小区数量统计 + """ + new_df = pd.read_excel('data/估值数据统计结果' + year_month_day + '.xlsx') + # 小区别名升级为单独df + actions = list() + for index, item in seaweed_df.iterrows(): + if type(item['alias']) == list: + for name in item['alias']: + actions.append({'_id': item['_id'], 'city': item['city'], 'name': name, 'region': item['region']}) + alias_df = pd.DataFrame(actions) + seaweed_alias_df = pd.concat([seaweed_df[['_id', 'city', 'region', 'name']], alias_df]) + deal_price_df = pd.DataFrame(list(deal_price_conn.find( + {'trade_date': {'$gte': dateutils.add_months(dateutils.datetime.datetime.now(), days=-180)}}, + {'city': 1, 'region': 1, 'district_name': 1, '_id': 0}).batch_size(10000))) + deal_price_df = deal_price_df.rename(columns={'district_name': 'name'}) + deal_price_seaweed_df = pd.merge(deal_price_df, seaweed_alias_df, how='left', on=['city', 'region', 'name']) + # 剔除没有配到的小区 + deal_price_seaweed_notnull_df = deal_price_seaweed_df.dropna(subset=['_id']) + deal_price_seaweed_notnull_df.loc[:, '近半年成交数据量'] = deal_price_seaweed_notnull_df.groupby(['city'])[ + 'city'].transform('size') + deal_price_half_year_df = deal_price_seaweed_notnull_df[['city', '近半年成交数据量']].drop_duplicates(keep='first') + deal_price_half_year_df = deal_price_half_year_df.rename(columns={'city': '城市'}) + new_df = pd.merge(new_df, deal_price_half_year_df, how='left', on=['城市']) + seaweed_half_year_df = deal_price_seaweed_notnull_df[['city', '_id']] + seaweed_half_year_df = seaweed_half_year_df.drop_duplicates(subset=['city', '_id'], keep='first') + seaweed_half_year_df['近半年有成交小区数量'] = seaweed_half_year_df.groupby(['city'])['city'].transform('size') + seaweed_half_year_rename_df = seaweed_half_year_df[['city', '近半年有成交小区数量']].rename(columns={'city': '城市'}) + seaweed_half_year_rename_df = seaweed_half_year_rename_df.drop_duplicates(keep='first') + new_df = pd.merge(new_df, seaweed_half_year_rename_df, how='left', on=['城市']) + new_df.to_excel(aim_file_name, index=False) + + +log.info("开始统计小区数量、近半年第三方均价、近半年成交信息") +count_stat() +third_avg_stat() +deal_stat() +log.info("完成统计小区数量、近半年第三方均价、近半年成交信息") \ No newline at end of file