新增无效小区删除

This commit is contained in:
shenjian@fangjia.com
2022-11-18 11:04:05 +08:00
parent 5e3f9d6001
commit 2c17552b84
2 changed files with 41 additions and 18 deletions
+23
View File
@@ -59,6 +59,27 @@ def build_doc(item):
return doc_dict
def deal_comm(community_summary_df, seaweed_df):
"""
删除额外小区信息
"""
actions = list()
diff_ids = set(community_summary_df['_id']).difference(set(seaweed_df['_id']))
for _id in diff_ids:
action = {
"_op_type": "delete",
"_index": index_name,
"_id": _id,
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
if len(actions) > 0:
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
def sync(city=None):
log.info(f"开始同步小区基本属性至206 {index_name}...")
seaweed_query = {'status': 0, 'cat': 'district'}
@@ -128,6 +149,8 @@ def sync(city=None):
# 写入csv文件,防止小区变动导致的后续过程更新报错
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
deal_comm(community_summary_df, seaweed_df)
log.info(f"完成同步小区基本属性至206 {index_name}")
+18 -18
View File
@@ -11,75 +11,75 @@ with DAG(
description="地址输出信息分类数据定时更新"
) as dag:
t1 = BashOperator(
task_id='comm_dag',
task_id='comm',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t2 = BashOperator(
task_id='price_dag',
task_id='price',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t3 = BashOperator(
task_id='sell_dag',
task_id='sell',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t4 = BashOperator(
task_id='avg_cost_dag',
task_id='avg_cost',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t5 = BashOperator(
task_id='trade_dag',
task_id='trade',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t6 = BashOperator(
task_id='total_price_dag',
task_id='total_price',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t6_1 = BashOperator(
task_id='total_price_complete_dag',
task_id='total_price_complete',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t7 = BashOperator(
task_id='traffic_dag',
task_id='traffic',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t8 = BashOperator(
task_id='medical_dag',
task_id='medical',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t9 = BashOperator(
task_id='life_dag',
task_id='life',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t10 = BashOperator(
task_id='relative_price_ranking_dag',
task_id='relative_price_ranking',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t11 = BashOperator(
task_id='consumption_dag',
task_id='consumption',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t12 = BashOperator(
task_id='price_rating_dag',
task_id='price_rating',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t13 = BashOperator(
task_id='education_dag',
task_id='education',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t14 = BashOperator(
task_id='education2_dag',
task_id='education2',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t15 = BashOperator(
task_id='extra_dag',
task_id='extra',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t16 = BashOperator(
task_id='other_dag',
task_id='other',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t17 = BashOperator(
task_id='business_district_dag',
task_id='business_district',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t8 >> t9 >> t10 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16 >> t17