新增无效小区删除
This commit is contained in:
@@ -59,6 +59,27 @@ def build_doc(item):
|
||||
return doc_dict
|
||||
|
||||
|
||||
def deal_comm(community_summary_df, seaweed_df):
|
||||
"""
|
||||
删除额外小区信息
|
||||
"""
|
||||
actions = list()
|
||||
diff_ids = set(community_summary_df['_id']).difference(set(seaweed_df['_id']))
|
||||
for _id in diff_ids:
|
||||
action = {
|
||||
"_op_type": "delete",
|
||||
"_index": index_name,
|
||||
"_id": _id,
|
||||
}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步小区基本属性至206 {index_name}...")
|
||||
seaweed_query = {'status': 0, 'cat': 'district'}
|
||||
@@ -128,6 +149,8 @@ def sync(city=None):
|
||||
|
||||
# 写入csv文件,防止小区变动导致的后续过程更新报错
|
||||
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||
|
||||
deal_comm(community_summary_df, seaweed_df)
|
||||
log.info(f"完成同步小区基本属性至206 {index_name}")
|
||||
|
||||
|
||||
|
||||
@@ -11,75 +11,75 @@ with DAG(
|
||||
description="地址输出信息分类数据定时更新"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='comm_dag',
|
||||
task_id='comm',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t2 = BashOperator(
|
||||
task_id='price_dag',
|
||||
task_id='price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t3 = BashOperator(
|
||||
task_id='sell_dag',
|
||||
task_id='sell',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t4 = BashOperator(
|
||||
task_id='avg_cost_dag',
|
||||
task_id='avg_cost',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t5 = BashOperator(
|
||||
task_id='trade_dag',
|
||||
task_id='trade',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t6 = BashOperator(
|
||||
task_id='total_price_dag',
|
||||
task_id='total_price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t6_1 = BashOperator(
|
||||
task_id='total_price_complete_dag',
|
||||
task_id='total_price_complete',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t7 = BashOperator(
|
||||
task_id='traffic_dag',
|
||||
task_id='traffic',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t8 = BashOperator(
|
||||
task_id='medical_dag',
|
||||
task_id='medical',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t9 = BashOperator(
|
||||
task_id='life_dag',
|
||||
task_id='life',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t10 = BashOperator(
|
||||
task_id='relative_price_ranking_dag',
|
||||
task_id='relative_price_ranking',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t11 = BashOperator(
|
||||
task_id='consumption_dag',
|
||||
task_id='consumption',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t12 = BashOperator(
|
||||
task_id='price_rating_dag',
|
||||
task_id='price_rating',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t13 = BashOperator(
|
||||
task_id='education_dag',
|
||||
task_id='education',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t14 = BashOperator(
|
||||
task_id='education2_dag',
|
||||
task_id='education2',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t15 = BashOperator(
|
||||
task_id='extra_dag',
|
||||
task_id='extra',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t16 = BashOperator(
|
||||
task_id='other_dag',
|
||||
task_id='other',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t17 = BashOperator(
|
||||
task_id='business_district_dag',
|
||||
task_id='business_district',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t8 >> t9 >> t10 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16 >> t17
|
||||
|
||||
Reference in New Issue
Block a user