Merge remote-tracking branch 'origin/master'

This commit is contained in:
2025-06-18 11:52:34 +08:00
+27 -20
View File
@@ -5,21 +5,25 @@ from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, areautils
import pandas as pd
from elasticsearch import helpers
import logging
log = logging.getLogger(__name__)
# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
def sync(province):
log.info(f"开始同步{province}企业数据的居委村委数据")
logger.info(f"开始同步{province}企业数据的居委村委数据")
shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(
f"select city_name, area_name, street_name, name, shape from community where province_name='{province}'")
if len(shape_df) == 0 or 'shape' not in shape_df.columns or len(shape_df[shape_df['shape'].notna()]) == 0:
log.info(f"{province}无围栏数据")
logger.info(f"{province}无围栏数据")
return
logger.info(f"已加载{len(shape_df)}条围栏")
# 先临时禁用索引刷新
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
index="didata_company",
body={"index": {"refresh_interval": "-1"}}
)
ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name, committee from didata_company where province='{province}'")
#ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name, committee from didata_company where province='{province}'")
ids = pd.DataFrame({})
if len(ids) > 0:
company_ids = ids['_id'].to_list()
else:
@@ -28,8 +32,11 @@ def sync(province):
count = 0
errors = list()
for index, item in shape_df[shape_df['shape'].notna()].iterrows():
logger.info(f"开始查找{item['city_name'], item['area_name'], item['name']}下企业数据")
ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_enterprise', item['shape']['coordinates'][0], field='gaode_location')
com_count = 0
for info in ret:
com_count += 1
source = info['_source']
if 'district' in source and (source['district'] != item['area_name']):
errors.append(source)
@@ -58,20 +65,20 @@ def sync(province):
}
actions.append(action)
if len(actions) == 10000:
for success, info in helpers.parallel_bulk(
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
actions,
thread_count=5, # 并行线程数
refresh=False,
chunk_size=1000 # 批量大小
):
if not success:
print(f"索引失败: {info}")
count += len(actions)
print("更新:" + str(count))
actions.clear()
#pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False)
logger.info(f"已查找到该围栏下{com_count}个企业数据")
for success, info in helpers.parallel_bulk(
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
actions,
thread_count=5, # 并行线程数
refresh=False,
chunk_size=1000 # 批量大小
):
if not success:
logger.info(f"索引失败: {info}")
count += len(actions)
logger.info("更新:" + str(count))
actions.clear()
if len(actions) > 0:
for success, info in helpers.parallel_bulk(
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
@@ -81,12 +88,12 @@ def sync(province):
chunk_size=1000 # 批量大小
):
if not success:
print(f"索引失败: {info}")
logger.error(f"索引失败: {info}")
count += len(actions)
print("更新:" + str(count))
logger.info("更新:" + str(count))
actions.clear()
#pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False)
log.info(f"完成同步{province}企业数据的居委村委数据")
logger.info(f"完成同步{province}企业数据的居委村委数据")
# 恢复索引刷新
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
index="didata_company",