diff --git a/.airflowignore b/.airflowignore index cf4aec3..1e7a37b 100644 --- a/.airflowignore +++ b/.airflowignore @@ -1,2 +1,3 @@ poi/business/.* -stat/evaluate/* \ No newline at end of file +stat/evaluate/* +mt/supply/* \ No newline at end of file diff --git a/dags/mt/__init__.py b/dags/mt/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/mt/mt_dag.py b/dags/mt/mt_dag.py new file mode 100644 index 0000000..3c015ac --- /dev/null +++ b/dags/mt/mt_dag.py @@ -0,0 +1,28 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + dag_id="mt_dag", + # 每月5日运行 + schedule_interval="0 5 5 * *", + start_date=pendulum.datetime(2022, 6, 28, tz="Asia/Shanghai"), + catchup=True, +) as dag: + t1 = BashOperator( + task_id='mt_comm', + bash_command='python /opt/airflow/dags/repo/dags/mt/supply/comm.py', + ) + t2 = BashOperator( + task_id='mt_rent_avg_price', + bash_command='python /opt/airflow/dags/repo/dags/mt/supply/rent_avg_price.py', + ) + t3 = BashOperator( + task_id='mt_facilitys', + bash_command='python /opt/airflow/dags/repo/dags/mt/supply/facilitys.py', + ) + t4 = BashOperator( + task_id='mt_deal', + bash_command='python /opt/airflow/dags/repo/dags/mt/supply/deal.py', + ) + t1 >> t2 >> t3 >> t4 diff --git a/dags/mt/supply/__init__.py b/dags/mt/supply/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/mt/supply/comm.py b/dags/mt/supply/comm.py new file mode 100644 index 0000000..1530e54 --- /dev/null +++ b/dags/mt/supply/comm.py @@ -0,0 +1,68 @@ +""" +小区属性导出 +""" +from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils +import pandas as pd +import datetime +import os + +seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') + +path = "/home/airflow/data/mt" +if not os.path.exists(path): + os.makedirs(path) + +city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx') +cities = city_df['城市列表'].values.tolist() + +seaweed_df = pd.DataFrame() +for city in cities: + temp_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0, 'city': city}).batch_size(10000))) + seaweed_df = pd.concat([seaweed_df, temp_df]) + +seaweed_result_df = seaweed_df[ + ['_id', 'city', 'region', 'name', 'alias', 'address', 'addr_alias', 'lng', 'lat', 'block', 'loop_line_info', + 'business_quarter', + 'household_count', 'estate_charge', 'car_site_count', 'property_life', 'house_property', 'all_area', 'build_area', + 'cubage_percent', 'virescence_percent', 'estate_type2', 'build_type', 'build_structure', 'finish_building_date', + 'developer', + 'main_type', 'main_type_area']] + + +def format_seaweed(x): + """ + 格式化输出小区属性 + :param x: + :return: + """ + names = [x.name] + try: + if x.alias is not None and str(x.alias) != 'nan': + names.extend(x.alias) + x.alias = ','.join(x.alias) + if x.addr_alias is not None and str(x.addr_alias) != 'nan': + x.addr_alias = ','.join(x.addr_alias) + x.landscaping_ratio = commutils.format_greening_rate(x.landscaping_ratio) + x.all_area = commutils.format_all_area(x.all_area) + x.build_area = commutils.format_build_area(x.build_area) + x.car_site_count = commutils.format_car_site_count(x.car_site_count) + x.households = commutils.format_household_count(x.households) + x.lng = commutils.format_lat_lng(x.lng) + x.lat = commutils.format_lat_lng(x.lat) + x.plot_ratio = commutils.format_plot_ratio(x.plot_ratio) + if type(x.alias) == str and '@' in x.alias: + x.alias = None + + return x + except Exception as e: + print(e) + print(x) + pass + + +seaweed_new_result_df = seaweed_result_df.apply(lambda x: format_seaweed(x), axis=1) +seaweed_new_result_df = seaweed_new_result_df.rename(columns={'_id': 'id', 'household_count': 'households', 'estate_charge': 'property_fee', + 'cubage_percent' : 'plot_ratio', 'virescence_percent': 'landscaping_ratio', 'estate_type2': 'property_type', + 'main_type': 'house_type', 'main_type_area': 'house_area'}) +seaweed_new_result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', + sep='@', encoding='utf-8', index=False) diff --git a/dags/mt/supply/data/租金和房价设置阀值(19个城市).xlsx b/dags/mt/supply/data/租金和房价设置阀值(19个城市).xlsx new file mode 100644 index 0000000..84ff1d0 Binary files /dev/null and b/dags/mt/supply/data/租金和房价设置阀值(19个城市).xlsx differ diff --git a/dags/mt/supply/data/美团19城市列表.xlsx b/dags/mt/supply/data/美团19城市列表.xlsx new file mode 100644 index 0000000..f90c24a Binary files /dev/null and b/dags/mt/supply/data/美团19城市列表.xlsx differ diff --git a/dags/mt/supply/deal.py b/dags/mt/supply/deal.py new file mode 100644 index 0000000..4c61dad --- /dev/null +++ b/dags/mt/supply/deal.py @@ -0,0 +1,90 @@ +""" +每个小区近一年成交统计 +""" +from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ElasticSearchClient +import pandas as pd +import datetime +import logging + + +es45_client = ElasticSearchClient(host='192.168.0.45') +deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price') +district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') + +city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx') +cities = city_df['城市列表'].values.tolist() + +# 1. 获取「今天」 +today = datetime.date.today() +# 2. 获取当前月的第一天 +first = today.replace(day=1) +# 3. 减一天,得到上个月的最后一天 +last_month = first - datetime.timedelta(days=1) + +month_start = last_month.strftime("%Y%m") + '01' +curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01' +logging.info(month_start + ' ' + curr_month_start) + +month_start_date = dateutils.parse_date(month_start, '%Y%m%d') +curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d') +path = "/home/airflow/data/mt" + +result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8') + +""" +MONGO城市成交数据 +""" +deal_num_df = pd.DataFrame(list(deal_price_conn.aggregate([ + {'$match': {'match_id': {'$exists': True}, 'area': {'$gte': 20}, 'avg_price': {'$gt': 0}, + 'trade_date': {'$gte': dateutils.add_months(curr_month_start_date, months=-12), + '$lt': curr_month_start_date}}}, + {'$group': {'_id': '$match_id', 'deal_num': {'$sum': 1}}} +]))) + +deal_num_df = deal_num_df.rename(columns={'_id': 'id'}) +result_df = pd.merge(result_df, deal_num_df, how='left', on='id') + +""" +ES上海成交数据 +""" +deal_es_num_df = es45_client.scan_to_data_frame('esf_deal_price_lansi', { + "_source": ["fjCity", "fjRegion", "fjName"], + "query": { + "bool": { + "must": [ + { + "exists": {"field": "fjName"} + }, + { + "range": { + "contractTime": { + "gte": dateutils.add_months(curr_month_start_date, months=-12), + "lt": curr_month_start_date + } + } + } + ] + } + } +}) +deal_es_num_df = deal_es_num_df.rename(columns={'fjCity': 'city', 'fjRegion': 'region', 'fjName': 'name'}) +deal_es_num_df = deal_es_num_df.drop(columns={'_id'}) +deal_es_num_df['es_num'] = 0 +deal_es_num_df['es_num'] = deal_es_num_df.groupby(by=['city', 'region', 'name'])['es_num'].transform('count') +deal_es_num_df = deal_es_num_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first') + +result_df['deal_num'].fillna(0, inplace=True) +result_df = pd.merge(result_df, deal_es_num_df, how='left', on=['city', 'region', 'name']) +result_df['deal_num'] += result_df['es_num'] +result_df = result_df.drop(columns={'es_num'}) + + +columns = ['id', 'city', 'region', 'name', 'alias', 'address', 'addr_alias', 'avg_price', 'rent_price', 'lng', 'lat', + 'block', 'loop_line_info', 'business_quarter', 'households', 'property_fee', 'car_site_count', + 'property_life', 'house_property', + 'all_area', 'build_area', 'plot_ratio', 'landscaping_ratio', 'property_type', 'build_type', + 'build_structure', 'finish_building_date', + 'developer', 'house_type', 'house_area', 'deal_num', 'district_facilitys', 'school', 'market', 'subway', + 'bank', 'hospital'] +result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', + encoding='utf-8', index=False, columns=columns) \ No newline at end of file diff --git a/dags/mt/supply/facilitys.py b/dags/mt/supply/facilitys.py new file mode 100644 index 0000000..da283f2 --- /dev/null +++ b/dags/mt/supply/facilitys.py @@ -0,0 +1,92 @@ +""" +周边配套数量统计 +""" +from goojia_utils import MAIN_MONGO_CLIENT, dateutils +import pandas as pd +import datetime +import logging + +district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') + +city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx') +cities = city_df['城市列表'].values.tolist() + +# 1. 获取「今天」 +today = datetime.date.today() +# 2. 获取当前月的第一天 +first = today.replace(day=1) +# 3. 减一天,得到上个月的最后一天 +last_month = first - datetime.timedelta(days=1) + +month_start = last_month.strftime("%Y%m") + '01' +curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01' +logging.info(month_start + ' ' + curr_month_start) + +month_start_date = dateutils.parse_date(month_start, '%Y%m%d') +curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d') + +path = "/home/airflow/data/mt" + +district_facilitys_df = pd.DataFrame() +for city in cities: + temp_df = pd.DataFrame( + list(district_facilitys_conn.find({'city': city}, {'city': 1, 'region': 1, 'district_name': 1, 'school': 1, + 'supermarket': 1, 'subway': 1, 'bank': 1, 'hospital': 1}).batch_size(10000))) + district_facilitys_df = pd.concat([district_facilitys_df, temp_df]) + + +def count_facilitys(x): + ret = x + try: + if 'school' in x and x['school'] is not None and str(x['school']) != 'nan': + for item in x['school']: + if item is not None: + ret.school = len(item) + if 'supermarket' in x and x['supermarket'] is not None and str(x['supermarket']) != 'nan': + for item in x['supermarket']: + if item is not None: + ret.supermarket = len(item) + if 'subway' in x and x['subway'] is not None and str(x['subway']) != 'nan': + for item in x['subway']: + if item is not None: + ret.subway = len(item) + if 'bank' in x and x['bank'] is not None and str(x['bank']) != 'nan': + for item in x['bank']: + if item is not None: + ret.bank = len(item) + if 'hospital' in x and x['hospital'] is not None and str(x['hospital']) != 'nan': + for item in x['hospital']: + if item is not None: + ret.hospital = len(item) + return ret + except Exception as e: + print(e) + print(x) + return None + + +district_facilitys_new_df = district_facilitys_df.apply(lambda x: count_facilitys(x), axis=1) + +district_facilitys_new_df = district_facilitys_new_df[['city', 'region', 'district_name', 'school', 'bank', 'hospital', 'supermarket', 'subway']] +district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_name': 'name', 'supermarket': 'market'}) + +result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8') +result_df = pd.merge(result_df, district_facilitys_new_df, how='left', on=['city', 'region', 'name']) +result_df.loc[:, 'district_facilitys'] = '学校、商场、地铁、银行、医院' + + +def format_facilitys(x): + if type(x['school']) is not int: + x.school = None + if type(x['bank']) is not int: + x.bank = None + if type(x['hospital']) is not int: + x.hospital = None + if type(x['market']) is not int: + x.market = None + if type(x['subway']) is not int: + x.subway = None + return x + + +result_df = result_df.apply(lambda x: format_facilitys(x), axis=1) diff --git a/dags/mt/supply/rent_avg_price.py b/dags/mt/supply/rent_avg_price.py new file mode 100644 index 0000000..e87f3d1 --- /dev/null +++ b/dags/mt/supply/rent_avg_price.py @@ -0,0 +1,94 @@ +""" +租金、均价导出 +""" + +from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils +import pandas as pd +import datetime +import logging + +district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat') +sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent') + + +city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx') +cities = city_df['城市列表'].values.tolist() + +path = "/home/airflow/data/mt" + +# 1. 获取「今天」 +today = datetime.date.today() +# 2. 获取当前月的第一天 +first = today.replace(day=1) +# 3. 减一天,得到上个月的最后一天 +last_month = first - datetime.timedelta(days=1) + +month_start = last_month.strftime("%Y%m") + '01' +curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01' +logging.info("日期信息:" + month_start + ' ' + curr_month_start) + +month_start_date = dateutils.parse_date(month_start, '%Y%m%d') +curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d') + +threshold_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/租金和房价设置阀值(19个城市).xlsx') +min_rent = dict() +max_rent = dict() +min_price = dict() +max_price = dict() +for index, item in threshold_df.iterrows(): + if item['租金最小值'] != '缺租金': + min_rent[item['城市']] = item['租金最小值'] + if item['租金最大值'] != '缺租金': + max_rent[item['城市']] = item['租金最大值'] + min_price[item['城市']] = item['均价最小值'] + +avg_df = pd.DataFrame() +for city in cities: + # 求当月均价均值 + temp_df = pd.DataFrame(list(district_stat_conn.find( + {'weekend': {'$gt': month_start_date, '$lte': curr_month_start_date}, 'city': city}, + {'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1, 'weekend': 1}).batch_size(10000))) + temp_df['avg_price_mean'] = temp_df.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform('mean').astype(int) + temp_df = temp_df[['city', 'region', 'district_name', 'avg_price_mean']] + temp_df = temp_df.drop_duplicates(subset=['city', 'region', 'district_name'], keep='first') + # 过滤阈值 + if city in min_price: + temp_df = temp_df[(temp_df['avg_price_mean'] >= min_price[city]) & (temp_df['avg_price_mean'] <= max_price[city])] + avg_df = pd.concat([avg_df, temp_df]) + +avg_df = avg_df.rename(columns={'district_name': 'name', 'avg_price_mean': 'avg_price'}) + +rent_df = pd.DataFrame() +for city in cities: + # 求租金 + temp_df = pd.DataFrame(list(sqmetre_rent_conn.find( + {'weekend': {'$gt': month_start_date, '$lte': curr_month_start_date}, 'city': city}, + {'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1}).batch_size(10000))) + # 根据阈值过滤 + if city in min_rent: + temp_df = temp_df[(temp_df['avg_price'] >= min_rent[city]) & (temp_df['avg_price'] <= max_rent[city])] + rent_df = pd.concat([rent_df, temp_df]) + +rent_df = rent_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_price'}) + +avg_df = avg_df[['city', 'region', 'name', 'avg_price']] +rent_df = rent_df[['city', 'region', 'name', 'rent_price']] + + +seaweed_new_result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8') +result_df = pd.merge(seaweed_new_result_df, avg_df, how='left', on=['city', 'region', 'name']) +result_df = pd.merge(result_df, rent_df, how='left', on=['city', 'region', 'name']) + + +def format_rent(x): + """ + 过滤租售比,大于2000租金为空 + :param x: + :return: + """ + x.rent_price = commutils.format_rent_price(x.avg_price, x.rent_price) + return x + + +result_df = result_df.apply(lambda x: format_rent(x), axis=1) +result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8', index=False)