diff --git a/dags/mt/mt_dag.py b/dags/mt/mt_dag.py index 62f1541..16de2df 100644 --- a/dags/mt/mt_dag.py +++ b/dags/mt/mt_dag.py @@ -11,22 +11,22 @@ with DAG( ) as dag: t1 = BashOperator( task_id='mt_comm', - bash_command='python /opt/airflow/dags/repo/dags/mt/supply/comm.py', + bash_command='python /opt/airflow/dags/repo/dags/mt/supply/comm.py {{ dag_run.conf["output_date"] }}', ) t2 = BashOperator( task_id='mt_rent_avg_price', - bash_command='python /opt/airflow/dags/repo/dags/mt/supply/rent_avg_price.py', + bash_command='python /opt/airflow/dags/repo/dags/mt/supply/rent_avg_price.py {{ dag_run.conf["output_date"] }}', ) t3 = BashOperator( task_id='mt_facilitys', - bash_command='python /opt/airflow/dags/repo/dags/mt/supply/facilitys.py', + bash_command='python /opt/airflow/dags/repo/dags/mt/supply/facilitys.py {{ dag_run.conf["output_date"] }}', ) t4 = BashOperator( task_id='mt_deal', - bash_command='python /opt/airflow/dags/repo/dags/mt/supply/deal.py', + bash_command='python /opt/airflow/dags/repo/dags/mt/supply/deal.py {{ dag_run.conf["output_date"] }}', ) t5 = BashOperator( task_id='mt_stat', - bash_command='python /opt/airflow/dags/repo/dags/mt/supply/data_stat.py', + bash_command='python /opt/airflow/dags/repo/dags/mt/supply/data_stat.py {{ dag_run.conf["output_date"] }}', ) t1 >> t2 >> t3 >> t4 >> t5 diff --git a/dags/mt/supply/comm.py b/dags/mt/supply/comm.py index 2e38f30..85f8b92 100644 --- a/dags/mt/supply/comm.py +++ b/dags/mt/supply/comm.py @@ -5,6 +5,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils import pandas as pd import datetime import os +import sys seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -15,6 +16,9 @@ if not os.path.exists(path): city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx') cities = city_df['城市列表'].values.tolist() +output_date = sys.argv[1] if sys.argv[1] else dateutils.format_date(datetime.datetime.now(), '%Y%m') +print("输出日期" + output_date) + seaweed_df = pd.DataFrame() for city in cities: temp_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0, 'city': city}).batch_size(10000))) @@ -69,5 +73,6 @@ seaweed_new_result_df = seaweed_result_df.apply(lambda x: format_seaweed(x), axi seaweed_new_result_df = seaweed_new_result_df.rename(columns={'_id': 'id', 'mapping_alias': 'alias', 'household_count': 'households', 'estate_charge': 'property_fee', 'cubage_percent' : 'plot_ratio', 'virescence_percent': 'landscaping_ratio', 'estate_type2': 'property_type', 'main_type': 'house_type', 'main_type_area': 'house_area'}) -seaweed_new_result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', + +seaweed_new_result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False) diff --git a/dags/mt/supply/data_stat.py b/dags/mt/supply/data_stat.py index ce224e3..24f4f03 100644 --- a/dags/mt/supply/data_stat.py +++ b/dags/mt/supply/data_stat.py @@ -4,13 +4,16 @@ import pandas as pd from goojia_utils import dateutils, ClickHouseClient import datetime +import sys path = "/home/airflow/data/mt" clickhouse_client = ClickHouseClient(database='fangjia_stat') -result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', - encoding='utf-8') +output_date = sys.argv[1] if sys.argv[1] else dateutils.format_date(datetime.datetime.now(), '%Y%m') +print("输出日期" + output_date) + +result_df = pd.read_csv(path + '/FJW_COMM_' + output_date, sep='@', encoding='utf-8') city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx') cities = city_df['城市列表'].values.tolist() @@ -31,37 +34,51 @@ for city in cities: actions.append(action) df = pd.DataFrame(actions) -aim_file_name = path + "/" + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '覆盖率统计.xlsx' +aim_file_name = path + "/" + output_date + '覆盖率统计.xlsx' df.to_excel(aim_file_name, index=False) def save2_clickhouse(): - curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01' - result_df['avg_price'].fillna(-99.0, inplace=True) - result_df['rent_price'].fillna(-99.0, inplace=True) - result_df['lng'].fillna(-99.0, inplace=True) - result_df['lat'].fillna(-99.0, inplace=True) - result_df['car_site_count'].fillna(-99.0, inplace=True) - result_df['households'].fillna(-99.0, inplace=True) - result_df['all_area'].fillna(-99.0, inplace=True) - result_df['build_area'].fillna(-99.0, inplace=True) - result_df['house_area'].fillna(-99.0, inplace=True) - result_df['deal_num'].fillna(-99.0, inplace=True) - result_df['school'].fillna(-99.0, inplace=True) - result_df['market'].fillna(-99.0, inplace=True) - result_df['subway'].fillna(-99.0, inplace=True) - result_df['bank'].fillna(-99.0, inplace=True) - result_df['hospital'].fillna(-99.0, inplace=True) - result_df['property_fee'].fillna(-99.0, inplace=True) + curr_month_start = output_date + '01' + # result_df['avg_price'].fillna(-99.0, inplace=True) + # result_df['rent_price'].fillna(-99.0, inplace=True) + # result_df['lng'].fillna(-99.0, inplace=True) + # result_df['lat'].fillna(-99.0, inplace=True) + # result_df['car_site_count'].fillna(-99.0, inplace=True) + # result_df['households'].fillna(-99.0, inplace=True) + # result_df['all_area'].fillna(-99.0, inplace=True) + # result_df['build_area'].fillna(-99.0, inplace=True) + # result_df['house_area'].fillna(-99.0, inplace=True) + # result_df['deal_num'].fillna(-99.0, inplace=True) + # result_df['school'].fillna(-99.0, inplace=True) + # result_df['market'].fillna(-99.0, inplace=True) + # result_df['subway'].fillna(-99.0, inplace=True) + # result_df['bank'].fillna(-99.0, inplace=True) + # result_df['hospital'].fillna(-99.0, inplace=True) + # result_df['property_fee'].fillna(-99.0, inplace=True) result_df['plot_ratio'] = result_df['plot_ratio'].astype(str) result_df['landscaping_ratio'] = result_df['landscaping_ratio'].astype(str) result_df['property_life'] = result_df['property_life'].astype(str) result_df['finish_building_date'] = result_df['finish_building_date'].astype(str) result_df['house_type'] = result_df['house_type'].astype(str) - result_df.fillna('null', inplace=True) - # 小区临时设置为0 - result_df['status'] = 0 + #result_df.fillna('', inplace=True) + result_df['avg_price'].astype(int) + result_df['rent_price'].astype(int) + result_df['lng'].astype(int) + result_df['lat'].astype(int) + result_df['car_site_count'].astype(int) + result_df['households'].astype(int) + result_df['all_area'].astype(int) + result_df['build_area'].astype(int) + result_df['house_area'].astype(int) + result_df['deal_num'].astype(int) + result_df['school'].astype(int) + result_df['market'].astype(int) + result_df['subway'].astype(int) + result_df['bank'].astype(int) + result_df['hospital'].astype(int) + result_df['property_fee'].astype(int) for city in cities: temp_df = result_df[result_df['city'] == city] @@ -69,7 +86,7 @@ def save2_clickhouse(): # 删除旧数据 clickhouse_client.execute( "ALTER TABLE fangjia_stat.mt_community_stat DELETE WHERE city='" + city + "' and year_month = toDate('" + curr_month_start + "')") - clickhouse_client.to_sql(data=temp_df, table='mt_community_stat', engine='MergeTree', if_exists=False, order_by={'year_month'}) + clickhouse_client.to_sql(data=temp_df.head(1), table='mt_community_stat', engine='MergeTree', if_exists=False, order_by={'year_month'}) def up_to_nextcloud(): @@ -80,7 +97,7 @@ def up_to_nextcloud(): NEXTCLOUD_URL = "https://nextcloud.fangjia.com" NEXTCLOUD_USERNAME = "shenjian@fangjia.com" NEXTCLOUD_PASSWORD = "goojia123456" - aim_file_path = '共享/美团/' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '覆盖率统计.xlsx' + aim_file_path = '共享/美团/' + output_date + '覆盖率统计.xlsx' nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True) nxc.upload_file(aim_file_name, aim_file_path) diff --git a/dags/mt/supply/deal.py b/dags/mt/supply/deal.py index 695dd93..77b558a 100644 --- a/dags/mt/supply/deal.py +++ b/dags/mt/supply/deal.py @@ -5,6 +5,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ElasticSearchClient import pandas as pd import datetime import logging +import sys es45_client = ElasticSearchClient(host='192.168.0.45') @@ -14,20 +15,18 @@ district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_ city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx') cities = city_df['城市列表'].values.tolist() -# 1. 获取「今天」 -today = datetime.date.today() -# 2. 获取当前月的第一天 -first = today.replace(day=1) -# 3. 减一天,得到上个月的最后一天 -last_month = first - datetime.timedelta(days=1) +path = "/home/airflow/data/mt" -month_start = last_month.strftime("%Y%m") + '01' -curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01' -logging.info(month_start + ' ' + curr_month_start) +output_date = sys.argv[1] if sys.argv[1] else dateutils.format_date(datetime.datetime.now(), '%Y%m') + +seaweed_new_result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8') + +month_start = dateutils.format_date(dateutils.add_months(dateutils.parse_date(output_date, '%Y%m'), months=-1), '%Y%m') + '01' +curr_month_start = output_date + '01' +logging.info("日期信息:" + month_start + ' ' + curr_month_start) month_start_date = dateutils.parse_date(month_start, '%Y%m%d') curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d') -path = "/home/airflow/data/mt" result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8') @@ -97,5 +96,4 @@ columns = ['id', 'city', 'region', 'name', 'alias', 'address', 'addr_alias', 'av 'build_structure', 'finish_building_date', 'developer', 'house_type', 'house_area', 'deal_num', 'district_facilitys', 'school', 'market', 'subway', 'bank', 'hospital', 'status'] -result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', - encoding='utf-8', index=False, columns=columns) \ No newline at end of file +result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False, columns=columns) \ No newline at end of file diff --git a/dags/mt/supply/facilitys.py b/dags/mt/supply/facilitys.py index 86198be..7a708d6 100644 --- a/dags/mt/supply/facilitys.py +++ b/dags/mt/supply/facilitys.py @@ -5,28 +5,26 @@ from goojia_utils import MAIN_MONGO_CLIENT, dateutils import pandas as pd import datetime import logging +import sys district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx') cities = city_df['城市列表'].values.tolist() -# 1. 获取「今天」 -today = datetime.date.today() -# 2. 获取当前月的第一天 -first = today.replace(day=1) -# 3. 减一天,得到上个月的最后一天 -last_month = first - datetime.timedelta(days=1) +path = "/home/airflow/data/mt" -month_start = last_month.strftime("%Y%m") + '01' -curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01' -logging.info(month_start + ' ' + curr_month_start) +output_date = sys.argv[1] if sys.argv[1] else dateutils.format_date(datetime.datetime.now(), '%Y%m') + +seaweed_new_result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8') + +month_start = dateutils.format_date(dateutils.add_months(dateutils.parse_date(output_date, '%Y%m'), months=-1), '%Y%m') + '01' +curr_month_start = output_date + '01' +logging.info("日期信息:" + month_start + ' ' + curr_month_start) month_start_date = dateutils.parse_date(month_start, '%Y%m%d') curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d') -path = "/home/airflow/data/mt" - district_facilitys_df = pd.DataFrame() district_facilitys_school_df = pd.DataFrame(list(district_facilitys_conn.aggregate([ {'$match': {'school': {'$ne': None}}}, @@ -77,8 +75,7 @@ district_facilitys_new_df = district_facilitys_df[ district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_name': 'name', 'supermarket': 'market'}) district_facilitys_new_df = district_facilitys_new_df.drop_duplicates(subsets=['city', 'region', 'name'], keep='first') -result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', - encoding='utf-8') +result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8') result_df = pd.merge(result_df, district_facilitys_new_df, how='left', on=['city', 'region', 'name']) result_df.loc[:, 'district_facilitys'] = None @@ -129,5 +126,4 @@ result_df['bank'].fillna(0, inplace=True) result_df['hospital'].fillna(0, inplace=True) result_df['subway'].fillna(0, inplace=True) -result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', - sep='@', encoding='utf-8', index=False) +result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False) diff --git a/dags/mt/supply/rent_avg_price.py b/dags/mt/supply/rent_avg_price.py index 6b1a20f..49b6dde 100644 --- a/dags/mt/supply/rent_avg_price.py +++ b/dags/mt/supply/rent_avg_price.py @@ -6,6 +6,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils import pandas as pd import datetime import logging +import sys district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat') sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent') @@ -16,15 +17,12 @@ cities = city_df['城市列表'].values.tolist() path = "/home/airflow/data/mt" -# 1. 获取「今天」 -today = datetime.date.today() -# 2. 获取当前月的第一天 -first = today.replace(day=1) -# 3. 减一天,得到上个月的最后一天 -last_month = first - datetime.timedelta(days=1) +output_date = sys.argv[1] if sys.argv[1] else dateutils.format_date(datetime.datetime.now(), '%Y%m') -month_start = last_month.strftime("%Y%m") + '01' -curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01' +seaweed_new_result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8') + +month_start = dateutils.format_date(dateutils.add_months(dateutils.parse_date(output_date, '%Y%m'), months=-1), '%Y%m') + '01' +curr_month_start = output_date + '01' logging.info("日期信息:" + month_start + ' ' + curr_month_start) month_start_date = dateutils.parse_date(month_start, '%Y%m%d') @@ -75,8 +73,6 @@ rent_df = rent_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_pr avg_df = avg_df[['city', 'region', 'name', 'avg_price']] rent_df = rent_df[['city', 'region', 'name', 'rent_price']] - -seaweed_new_result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8') result_df = pd.merge(seaweed_new_result_df, avg_df, how='left', on=['city', 'region', 'name']) result_df = pd.merge(result_df, rent_df, how='left', on=['city', 'region', 'name']) @@ -92,4 +88,4 @@ def format_rent(x): result_df = result_df.apply(lambda x: format_rent(x), axis=1) -result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8', index=False) +result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False) diff --git a/install.bat b/install.bat new file mode 100644 index 0000000..525a105 --- /dev/null +++ b/install.bat @@ -0,0 +1 @@ +pip install -r requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/ \ No newline at end of file