diff --git a/dags/stat/district_sell_deal_dag.py b/dags/stat/district_sell_deal_dag.py new file mode 100644 index 0000000..3b5196f --- /dev/null +++ b/dags/stat/district_sell_deal_dag.py @@ -0,0 +1,20 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + "district_sell_deal_dag", + # 每周五运行 + schedule_interval="0 1 * * 5", + start_date=pendulum.datetime(2022, 6, 20, tz="Asia/Shanghai"), + catchup=True, +) as dag: + t1 = BashOperator( + task_id='district_count_thirdavg_deal', + bash_command='python /opt/airflow/dags/repo/dags/stat/evaluate/district_count_thirdavg_deal.py', + ) + t2 = BashOperator( + task_id='district_sell_property', + bash_command='python /opt/airflow/dags/repo/dags/stat/evaluate/district_sell_property.py', + ) + t1 >> t2 diff --git a/dags/stat/evaluate/district_count_thirdavg_deal.py b/dags/stat/evaluate/district_count_thirdavg_deal.py index 1f1e1f7..4129c78 100644 --- a/dags/stat/evaluate/district_count_thirdavg_deal.py +++ b/dags/stat/evaluate/district_count_thirdavg_deal.py @@ -8,9 +8,9 @@ import logging """ log = logging.getLogger(__name__) -df = pd.read_excel('data/估值数据统计.xlsx') +df = pd.read_excel('/opt/airflow/dags/repo/dags/stat/evaluate/data/估值数据统计.xlsx') year_month_day = dateutils.format_date(datetime.datetime.now()) -aim_file_name = 'data/估值数据统计结果' + year_month_day + '.xlsx' +aim_file_name = '/opt/airflow/dags/repo/dags/stat/evaluate/data/估值数据统计结果' + year_month_day + '.xlsx' seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') @@ -43,7 +43,7 @@ def third_avg_stat(): """ 近半年有第三方均价小区数量及占比 """ - new_df = pd.read_excel('data/估值数据统计结果' + year_month_day + '.xlsx') + new_df = pd.read_excel(aim_file_name) s_date = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-180), fm='%Y%m') average_statistics_df = pd.DataFrame( list(average_statistics_conn.find({'s_date': {'$gte': int(s_date)}, 'category': 'district'}).batch_size(10000))) @@ -86,7 +86,7 @@ def deal_stat(): """ 近半年成交量及小区数量统计 """ - new_df = pd.read_excel('data/估值数据统计结果' + year_month_day + '.xlsx') + new_df = pd.read_excel(aim_file_name) # 小区别名升级为单独df actions = list() for index, item in seaweed_df.iterrows(): diff --git a/dags/stat/evaluate/district_sell_property.py b/dags/stat/evaluate/district_sell_property.py new file mode 100644 index 0000000..f0ebe8e --- /dev/null +++ b/dags/stat/evaluate/district_sell_property.py @@ -0,0 +1,140 @@ +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils +import pandas as pd +import datetime +import logging + +""" +统计近三个月、近一年样本量及小区属性 +""" +log = logging.getLogger(__name__) + +year_month_day = dateutils.format_date(datetime.datetime.now()) +aim_file_name = '/opt/airflow/dags/repo/dags/stat/evaluate/data/估值数据统计结果' + year_month_day + '.xlsx' +start_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-181)) +one_year_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-366)) + +seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +es206_client = ElasticSearchClient(host='192.168.0.206') + +seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'alias': 1}) + .batch_size(10000))) + + +def sell_stat(): + """ + 最近一年有样本小区占比 最近3月有样本小区占比 + """ + df = pd.read_excel(aim_file_name) + sell_seaweed_df = es206_client.scan_by_sql_to_data_frame( + "select city, region, district_name, p_date from latest_seawater where p_date > " + start_time + " and cat='sell' ") + # 小区别名升级为单独df + actions = list() + for index, item in seaweed_df.iterrows(): + if type(item['alias']) == list: + for name in item['alias']: + actions.append({'_id': item['_id'], 'city': item['city'], 'name': name, 'region': item['region']}) + alias_df = pd.DataFrame(actions) + + sell_seaweed_rename_df = sell_seaweed_df[['city', 'region', 'district_name']] + sell_seaweed_rename_df = sell_seaweed_rename_df.rename(columns={'district_name': 'name'}) + sell_seaweed_merge_df = pd.merge(sell_seaweed_rename_df, alias_df, how='left', on=['city', 'region', 'name']) + sell_seaweed_merge_df['count'] = sell_seaweed_merge_df[sell_seaweed_merge_df['_id'].notna()].groupby('_id')[ + '_id'].transform('size') + sell_seaweed_merge_df = sell_seaweed_merge_df[['city', 'region', '_id', 'count']] + sell_seaweed_merge_df = sell_seaweed_merge_df.drop_duplicates(subset=['city', 'region', '_id']) + sell_seaweed_ok_df = sell_seaweed_merge_df[sell_seaweed_merge_df['count'] > 3] + sell_seaweed_ok_df['city_seaweed_count'] = sell_seaweed_ok_df.groupby('city')['city'].transform('size') + city_seaweed_count_df = sell_seaweed_ok_df[['city', 'city_seaweed_count']].drop_duplicates( + subset=['city', 'city_seaweed_count']) + city_seaweed_count_df = city_seaweed_count_df.rename(columns={'city': '城市', 'city_seaweed_count': '最近3月有样本小区占比'}) + df = pd.merge(df, city_seaweed_count_df, how='left', on=['城市']) + df['最近3月有样本小区占比'] = round(df['最近3月有样本小区占比'] * 100 / df['小区量'], 2) + df.to_excel(aim_file_name, index=False) + + cities = df[['城市']].drop_duplicates(keep='first')['城市'].values.tolist() + city_one_year_count = list() + for city in cities: + flag = True + for item in city_one_year_count: + if item['城市'] == city: + flag = False + break + if flag == False: + continue + seaweed_temp_df = seaweed_df[seaweed_df['city'] == city] + count = 0 + total_num = 0 + all_num = 0 + for index, item in seaweed_temp_df.iterrows(): + names = list() + region = item['region'] + names.append(item['name']) + if type(item['alias']) == list: + for alias_item in item['alias']: + names.append(alias_item) + names_str = "','".join(names) + names_str = "'" + names_str + "'" + try: + sql = "select count(*) from latest_seawater where city='" + city + "' and region='" + region + "' and cat='sell' and district_name in(" + names_str + ") and p_date > " + one_year_time + num = es206_client.search_by_sql(sql)['hits']['total']['value'] + if num > 3: + count += 1 + total_num += num + all_num += num + except Exception as e: + print(item) + print(e) + city_one_year_count.append({'城市': city, '近一年有样本小区数量': count, '近一年的样本总量': total_num, '总数量': all_num}) + print(city + "已处理完, 近一年有样本小区数量" + str(count) + " 近一年的样本总量" + str(total_num)) + + result_df = pd.DataFrame(city_one_year_count) + result_df = result_df[['城市', '近一年有样本小区数量', '近一年的样本总量']] + df = pd.merge(df, result_df, how='left', on=['城市']) + df['最近一年有样本小区占比'] = round(df['近一年有样本小区数量'] * 100 / df['小区量'], 2) + df.to_excel(aim_file_name, index=False) + + +def property_stat(): + """ + 建筑年代覆盖率 物业类型覆盖率 建筑类型覆盖率 + """ + df = pd.read_excel(aim_file_name) + df['建筑年代'] = None + df['物业类型'] = None + df['建筑类型'] = None + for index, item in df.iterrows(): + city = item['城市'] + print(city) + count = seaweed_conn.count( + {'cat': 'district', 'status': 0, 'city': city, 'finish_building_date': {'$ne': None}}) + df.loc[index, '建筑年代'] = round(count * 100 / item['小区量'], 2) + + estate_type2_count = seaweed_conn.count( + {'cat': 'district', 'status': 0, 'city': city, 'estate_type2': {'$ne': None}}) + df.loc[index, '物业类型'] = round(estate_type2_count * 100 / item['小区量'], 2) + + build_type_count = seaweed_conn.count( + {'cat': 'district', 'status': 0, 'city': city, 'build_type': {'$ne': None}}) + df.loc[index, '建筑类型'] = round(build_type_count * 100 / item['小区量'], 2) + df.to_excel(aim_file_name, index=False) + + +def up_to_nextcloud(): + """ + 上传统计信息至nextcloud + """ + from nextcloud import NextCloud + NEXTCLOUD_URL = "https://nextcloud.fangjia.com" + NEXTCLOUD_USERNAME = "shenjian@fangjia.com" + NEXTCLOUD_PASSWORD = "goojia123456" + aim_file_path = '共享/二期估值数据每周统计/估值数据统计结果' + year_month_day + '.xlsx' + nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True) + nxc.upload_file(aim_file_name, aim_file_path) + +log.info("开始统计统计近三个月、近一年样本量及小区属性") +sell_stat() +property_stat() +log.info("完成统计统计近三个月、近一年样本量及小区属性") +log.info("开始上传至云盘") +up_to_nextcloud() +log.info("云盘上传完成") diff --git a/requirements.txt b/requirements.txt index 2a8ad69..6ff14dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -goojia_utils \ No newline at end of file +goojia_utils +nextcloud-api-wrapper \ No newline at end of file