diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index a3497a2..f08d67f 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -59,6 +59,27 @@ def build_doc(item): return doc_dict +def deal_comm(community_summary_df, seaweed_df): + """ + 删除额外小区信息 + """ + actions = list() + diff_ids = set(community_summary_df['_id']).difference(set(seaweed_df['_id'])) + for _id in diff_ids: + action = { + "_op_type": "delete", + "_index": index_name, + "_id": _id, + } + actions.append(action) + if len(actions) == 5000: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() + if len(actions) > 0: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() + + def sync(city=None): log.info(f"开始同步小区基本属性至206 {index_name}...") seaweed_query = {'status': 0, 'cat': 'district'} @@ -128,6 +149,8 @@ def sync(city=None): # 写入csv文件,防止小区变动导致的后续过程更新报错 seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False) + + deal_comm(community_summary_df, seaweed_df) log.info(f"完成同步小区基本属性至206 {index_name}") diff --git a/dags/business/community_summary_dag.py b/dags/business/community_summary_dag.py index 182259c..09b84ac 100644 --- a/dags/business/community_summary_dag.py +++ b/dags/business/community_summary_dag.py @@ -11,75 +11,75 @@ with DAG( description="地址输出信息分类数据定时更新" ) as dag: t1 = BashOperator( - task_id='comm_dag', + task_id='comm', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t2 = BashOperator( - task_id='price_dag', + task_id='price', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t3 = BashOperator( - task_id='sell_dag', + task_id='sell', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t4 = BashOperator( - task_id='avg_cost_dag', + task_id='avg_cost', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t5 = BashOperator( - task_id='trade_dag', + task_id='trade', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t6 = BashOperator( - task_id='total_price_dag', + task_id='total_price', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t6_1 = BashOperator( - task_id='total_price_complete_dag', + task_id='total_price_complete', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t7 = BashOperator( - task_id='traffic_dag', + task_id='traffic', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t8 = BashOperator( - task_id='medical_dag', + task_id='medical', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t9 = BashOperator( - task_id='life_dag', + task_id='life', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t10 = BashOperator( - task_id='relative_price_ranking_dag', + task_id='relative_price_ranking', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t11 = BashOperator( - task_id='consumption_dag', + task_id='consumption', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t12 = BashOperator( - task_id='price_rating_dag', + task_id='price_rating', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t13 = BashOperator( - task_id='education_dag', + task_id='education', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t14 = BashOperator( - task_id='education2_dag', + task_id='education2', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t15 = BashOperator( - task_id='extra_dag', + task_id='extra', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t16 = BashOperator( - task_id='other_dag', + task_id='other', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t17 = BashOperator( - task_id='business_district_dag', + task_id='business_district', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t8 >> t9 >> t10 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16 >> t17