This commit is contained in:
2025-07-03 16:03:53 +08:00
4 changed files with 55 additions and 51 deletions
+3 -6
View File
@@ -22,8 +22,8 @@ def sync(province):
index="didata_company",
body={"index": {"refresh_interval": "-1"}}
)
#ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name, committee from didata_company where province='{province}'")
ids = pd.DataFrame({})
ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name, committee from didata_company where province='{province}'")
#ids = pd.DataFrame({})
if len(ids) > 0:
company_ids = ids['_id'].to_list()
else:
@@ -105,7 +105,4 @@ if __name__ == '__main__':
# for province in areautils.get_all_province():
# print(province)
for province in areautils.get_all_province():
if '甘肃省' == province:
flag = True
if flag:
sync(province)
sync(province)
+27 -20
View File
@@ -1,16 +1,18 @@
from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, threads, geohashutils, areautils
from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, areautils
import pandas as pd
from elasticsearch import helpers
import numpy as np
import re
import logging
log = logging.getLogger(__name__)
# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
"""
通过开发区围栏更新企业开发区 TODO
"""
def sync(province):
log.info(f"开始同步{province}企业数据的开发区数据")
logger.info(f"开始同步{province}企业数据的开发区数据")
# 先临时禁用索引刷新
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
index="didata_company",
@@ -18,14 +20,18 @@ def sync(province):
)
shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select name, shape from development_zone where province='{province}'")
if len(shape_df) == 0 or 'shape' not in shape_df.columns or len(shape_df[shape_df['shape'].notna()]) == 0:
logger.info(f"{province}无围栏数据")
return
actions = list()
count = 0
for index, item in shape_df.iterrows():
if pd.isna(item['shape']):
continue
for index, item in shape_df[shape_df['shape'].notna()].iterrows():
logger.info(f"开始查找{item['name']}下企业数据")
ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_company', item['shape']['coordinates'][0],
field='location')
com_count = 0
for info in ret:
com_count += 1
doc_info = {'dev': item['name']}
action = {
"_op_type": "update",
@@ -36,31 +42,32 @@ def sync(province):
}
actions.append(action)
if len(actions) == 10000:
for success, info in helpers.parallel_bulk(
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
actions,
thread_count=5, # 并行线程数
chunk_size=1000 # 批量大小
):
if not success:
print(f"索引失败: {info}")
count += len(actions)
print("更新:" + str(count))
actions.clear()
logger.info(f"已查找到该围栏下{com_count}个企业数据")
if len(actions) > 0:
for success, info in helpers.parallel_bulk(
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
actions,
thread_count=5, # 并行线程数
chunk_size=10000 # 批量大小
):
if not success:
print(f"索引失败: {info}")
count += len(actions)
print("更新:" + str(count))
actions.clear()
if len(actions) > 0:
for success, info in helpers.parallel_bulk(
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
actions,
thread_count=5, # 并行线程数
chunk_size=1000 # 批量大小
chunk_size=10000 # 批量大小
):
if not success:
print(f"索引失败: {info}")
count += len(actions)
print("更新:" + str(count))
actions.clear()
log.info(f"完成同步{province}企业数据的开发区数据")
logger.info(f"完成同步{province}企业数据的开发区数据")
# 恢复索引刷新
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
index="didata_company",
+16 -16
View File
@@ -1,7 +1,6 @@
from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, threads, geohashutils, areautils
from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, areautils
import pandas as pd
from elasticsearch import helpers
import numpy as np
import re
import logging
log = logging.getLogger(__name__)
@@ -18,7 +17,7 @@ def sync():
'绥安工业开发区': '旧镇工业园', '绥安工业开发区': '大南坂开发区', '绥安工业开发区': '大南坂工业开发区', '福建龙州工业园区': '龙州工业园',
'长汀经济开发区':'腾飞开发区', '福建福州金山工业园区': '金山投资区', '厦门象屿综合保税区': '厦门现代物流园', '福清融侨经济技术开发区': '融侨经济',
'福建洛江经济开发区': '万安开发区'}
dev_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame("select province, city, district, name, alias from development_zone where province='福建省'")
dev_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame("select province, city, district, name, alias from development_zone")
# 不要去重,有可能存在别名, 如福建云霄常山经济开发区
#dev_df = dev_df[dev_df['name'] == '福建政和经济开发区']
actions = list()
@@ -55,19 +54,20 @@ def sync():
# 福建诏安工业园区 福建省漳州市诏安工业园区丹诏大道15号
if pd.notna(dev.city):
province_short = areautils.get_province(dev.city)
new_name = re.sub(province_short, '', dev.name)
if pd.notna(dev.city) and pd.notna(dev.district):
city_short = areautils.get_city_short(dev.city)
district_short = areautils.get_district_short(dev.city, dev.district)
if pd.notna(city_short) and city_short in new_name:
names.append(new_name)
if pd.notna(district_short) and district_short in new_name:
names.append(new_name)
# 福建省漳州市长泰县经济开发区林墩办事处帝君路 福建长泰经济开发区
new_new_name = re.sub(district_short, district_short + '县', new_name)
names.append(new_new_name)
new_new_name = re.sub(district_short, district_short + '区', new_name)
names.append(new_new_name)
if pd.notna(province_short):
new_name = re.sub(province_short, '', dev.name)
if pd.notna(dev.city) and pd.notna(dev.district):
city_short = areautils.get_city_short(dev.city)
district_short = areautils.get_district_short(dev.city, dev.district)
if pd.notna(city_short) and city_short in new_name:
names.append(new_name)
if pd.notna(district_short) and district_short in new_name:
names.append(new_name)
# 福建省漳州市长泰县经济开发区林墩办事处帝君路 福建长泰经济开发区
new_new_name = re.sub(district_short, district_short + '县', new_name)
names.append(new_new_name)
new_new_name = re.sub(district_short, district_short + '区', new_name)
names.append(new_new_name)
for key in dev_alias:
if dev.name == key:
+9 -9
View File
@@ -11,16 +11,16 @@ with DAG(
catchup=True,
description="企业涉农处理"
) as dag:
t1 = BashOperator(
task_id='fill_commpany_comm',
bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_company_comm.py',
)
t2 = BashOperator(
task_id='fill_commpany_dev',
bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_commpany_dev.py',
)
# t1 = BashOperator(
# task_id='fill_commpany_comm',
# bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_company_comm.py',
# )
# t2 = BashOperator(
# task_id='fill_commpany_dev',
# bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_company_dev.py',
# )
t3 = BashOperator(
task_id='mark_right_dev',
bash_command='python /opt/airflow/dags/repo/dags/company/company/mark_right_dev.py',
)
t1 >> t2 >> t3
t3