优化
This commit is contained in:
@@ -5,14 +5,16 @@ from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, areautils
|
||||
import pandas as pd
|
||||
from elasticsearch import helpers
|
||||
import logging
|
||||
log = logging.getLogger(__name__)
|
||||
# 配置日志
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def sync(province):
|
||||
log.info(f"开始同步{province}企业数据的居委村委数据")
|
||||
logger.info(f"开始同步{province}企业数据的居委村委数据")
|
||||
shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(
|
||||
f"select city_name, area_name, street_name, name, shape from community where province_name='{province}'")
|
||||
if len(shape_df) == 0 or 'shape' not in shape_df.columns or len(shape_df[shape_df['shape'].notna()]) == 0:
|
||||
log.info(f"{province}无围栏数据")
|
||||
logger.info(f"{province}无围栏数据")
|
||||
return
|
||||
# 先临时禁用索引刷新
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
|
||||
@@ -28,7 +30,7 @@ def sync(province):
|
||||
count = 0
|
||||
errors = list()
|
||||
for index, item in shape_df[shape_df['shape'].notna()].iterrows():
|
||||
print(index)
|
||||
logger.info(index)
|
||||
ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_enterprise', item['shape']['coordinates'][0], field='gaode_location')
|
||||
for info in ret:
|
||||
source = info['_source']
|
||||
@@ -68,9 +70,9 @@ def sync(province):
|
||||
chunk_size=1000 # 批量大小
|
||||
):
|
||||
if not success:
|
||||
print(f"索引失败: {info}")
|
||||
logger.info(f"索引失败: {info}")
|
||||
count += len(actions)
|
||||
print("更新:" + str(count))
|
||||
logger.info("更新:" + str(count))
|
||||
actions.clear()
|
||||
#pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False)
|
||||
if len(actions) > 0:
|
||||
@@ -82,12 +84,12 @@ def sync(province):
|
||||
chunk_size=1000 # 批量大小
|
||||
):
|
||||
if not success:
|
||||
print(f"索引失败: {info}")
|
||||
logger.error(f"索引失败: {info}")
|
||||
count += len(actions)
|
||||
print("更新:" + str(count))
|
||||
logger.info("更新:" + str(count))
|
||||
actions.clear()
|
||||
#pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False)
|
||||
log.info(f"完成同步{province}企业数据的居委村委数据")
|
||||
logger.info(f"完成同步{province}企业数据的居委村委数据")
|
||||
# 恢复索引刷新
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
|
||||
index="didata_company",
|
||||
|
||||
Reference in New Issue
Block a user