diff --git a/dags/company/company/fill_company_comm.py b/dags/company/company/fill_company_comm.py index 1c33890..52e61f4 100644 --- a/dags/company/company/fill_company_comm.py +++ b/dags/company/company/fill_company_comm.py @@ -22,8 +22,8 @@ def sync(province): index="didata_company", body={"index": {"refresh_interval": "-1"}} ) - #ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name, committee from didata_company where province='{province}'") - ids = pd.DataFrame({}) + ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name, committee from didata_company where province='{province}'") + #ids = pd.DataFrame({}) if len(ids) > 0: company_ids = ids['_id'].to_list() else: @@ -105,7 +105,4 @@ if __name__ == '__main__': # for province in areautils.get_all_province(): # print(province) for province in areautils.get_all_province(): - if '甘肃省' == province: - flag = True - if flag: - sync(province) \ No newline at end of file + sync(province) \ No newline at end of file diff --git a/dags/company/company/fill_company_dev.py b/dags/company/company/fill_company_dev.py index ff8ad08..a8ddd02 100644 --- a/dags/company/company/fill_company_dev.py +++ b/dags/company/company/fill_company_dev.py @@ -1,16 +1,18 @@ -from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, threads, geohashutils, areautils +from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, areautils import pandas as pd from elasticsearch import helpers import numpy as np import re import logging -log = logging.getLogger(__name__) +# 配置日志 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) """ 通过开发区围栏更新企业开发区 TODO """ def sync(province): - log.info(f"开始同步{province}企业数据的开发区数据") + logger.info(f"开始同步{province}企业数据的开发区数据") # 先临时禁用索引刷新 DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( index="didata_company", @@ -18,14 +20,18 @@ def sync(province): ) shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select name, shape from development_zone where province='{province}'") + if len(shape_df) == 0 or 'shape' not in shape_df.columns or len(shape_df[shape_df['shape'].notna()]) == 0: + logger.info(f"{province}无围栏数据") + return actions = list() count = 0 - for index, item in shape_df.iterrows(): - if pd.isna(item['shape']): - continue + for index, item in shape_df[shape_df['shape'].notna()].iterrows(): + logger.info(f"开始查找{item['name']}下企业数据") ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_company', item['shape']['coordinates'][0], field='location') + com_count = 0 for info in ret: + com_count += 1 doc_info = {'dev': item['name']} action = { "_op_type": "update", @@ -36,31 +42,32 @@ def sync(province): } actions.append(action) - if len(actions) == 10000: - for success, info in helpers.parallel_bulk( - DIDATA_DEV_ELASTIC_CLIENT.elastic_client, - actions, - thread_count=5, # 并行线程数 - chunk_size=1000 # 批量大小 - ): - if not success: - print(f"索引失败: {info}") - count += len(actions) - print("更新:" + str(count)) - actions.clear() + logger.info(f"已查找到该围栏下{com_count}个企业数据") + if len(actions) > 0: + for success, info in helpers.parallel_bulk( + DIDATA_DEV_ELASTIC_CLIENT.elastic_client, + actions, + thread_count=5, # 并行线程数 + chunk_size=10000 # 批量大小 + ): + if not success: + print(f"索引失败: {info}") + count += len(actions) + print("更新:" + str(count)) + actions.clear() if len(actions) > 0: for success, info in helpers.parallel_bulk( DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions, thread_count=5, # 并行线程数 - chunk_size=1000 # 批量大小 + chunk_size=10000 # 批量大小 ): if not success: print(f"索引失败: {info}") count += len(actions) print("更新:" + str(count)) actions.clear() - log.info(f"完成同步{province}企业数据的开发区数据") + logger.info(f"完成同步{province}企业数据的开发区数据") # 恢复索引刷新 DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( index="didata_company", diff --git a/dags/company/company/mark_right_dev.py b/dags/company/company/mark_right_dev.py index 550cd8d..51a4fe6 100644 --- a/dags/company/company/mark_right_dev.py +++ b/dags/company/company/mark_right_dev.py @@ -1,7 +1,6 @@ -from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, threads, geohashutils, areautils +from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, areautils import pandas as pd from elasticsearch import helpers -import numpy as np import re import logging log = logging.getLogger(__name__) @@ -18,7 +17,7 @@ def sync(): '绥安工业开发区': '旧镇工业园', '绥安工业开发区': '大南坂开发区', '绥安工业开发区': '大南坂工业开发区', '福建龙州工业园区': '龙州工业园', '长汀经济开发区':'腾飞开发区', '福建福州金山工业园区': '金山投资区', '厦门象屿综合保税区': '厦门现代物流园', '福清融侨经济技术开发区': '融侨经济', '福建洛江经济开发区': '万安开发区'} - dev_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame("select province, city, district, name, alias from development_zone where province='福建省'") + dev_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame("select province, city, district, name, alias from development_zone") # 不要去重,有可能存在别名, 如福建云霄常山经济开发区 #dev_df = dev_df[dev_df['name'] == '福建政和经济开发区'] actions = list() @@ -55,19 +54,20 @@ def sync(): # 福建诏安工业园区 福建省漳州市诏安工业园区丹诏大道15号 if pd.notna(dev.city): province_short = areautils.get_province(dev.city) - new_name = re.sub(province_short, '', dev.name) - if pd.notna(dev.city) and pd.notna(dev.district): - city_short = areautils.get_city_short(dev.city) - district_short = areautils.get_district_short(dev.city, dev.district) - if pd.notna(city_short) and city_short in new_name: - names.append(new_name) - if pd.notna(district_short) and district_short in new_name: - names.append(new_name) - # 福建省漳州市长泰县经济开发区林墩办事处帝君路 福建长泰经济开发区 - new_new_name = re.sub(district_short, district_short + '县', new_name) - names.append(new_new_name) - new_new_name = re.sub(district_short, district_short + '区', new_name) - names.append(new_new_name) + if pd.notna(province_short): + new_name = re.sub(province_short, '', dev.name) + if pd.notna(dev.city) and pd.notna(dev.district): + city_short = areautils.get_city_short(dev.city) + district_short = areautils.get_district_short(dev.city, dev.district) + if pd.notna(city_short) and city_short in new_name: + names.append(new_name) + if pd.notna(district_short) and district_short in new_name: + names.append(new_name) + # 福建省漳州市长泰县经济开发区林墩办事处帝君路 福建长泰经济开发区 + new_new_name = re.sub(district_short, district_short + '县', new_name) + names.append(new_new_name) + new_new_name = re.sub(district_short, district_short + '区', new_name) + names.append(new_new_name) for key in dev_alias: if dev.name == key: diff --git a/dags/company/dag_company.py b/dags/company/dag_company.py index 773a51a..107569f 100644 --- a/dags/company/dag_company.py +++ b/dags/company/dag_company.py @@ -11,16 +11,16 @@ with DAG( catchup=True, description="企业涉农处理" ) as dag: - t1 = BashOperator( - task_id='fill_commpany_comm', - bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_company_comm.py', - ) - t2 = BashOperator( - task_id='fill_commpany_dev', - bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_commpany_dev.py', - ) + # t1 = BashOperator( + # task_id='fill_commpany_comm', + # bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_company_comm.py', + # ) + # t2 = BashOperator( + # task_id='fill_commpany_dev', + # bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_company_dev.py', + # ) t3 = BashOperator( task_id='mark_right_dev', bash_command='python /opt/airflow/dags/repo/dags/company/company/mark_right_dev.py', ) - t1 >> t2 >> t3 \ No newline at end of file + t3 \ No newline at end of file