修復sys报错
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
"""
|
||||
第一步: 将百度经纬度转为高德经纬度(只需跑批一次即可)
|
||||
"""
|
||||
@@ -0,0 +1,80 @@
|
||||
"""
|
||||
通过居委围栏更新隶属关系
|
||||
"""
|
||||
from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, areautils
|
||||
import pandas as pd
|
||||
from elasticsearch import helpers
|
||||
import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
def sync(province):
|
||||
log.info(f"开始同步{province}企业数据的居委村委数据")
|
||||
ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name from didata_company where province='{province}'")
|
||||
company_ids = ids['_id'].to_list()
|
||||
|
||||
shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select city_name, area_name, street_name, name, shape from community where province_name='{province}' ")
|
||||
if len(shape_df) == 0:
|
||||
log.info(f"{province}无围栏数据")
|
||||
actions = list()
|
||||
count = 0
|
||||
errors = list()
|
||||
for index, item in shape_df[shape_df['shape'].notna()].iterrows():
|
||||
ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_enterprise', item['shape']['coordinates'][0], field='gaode_location')
|
||||
for info in ret:
|
||||
source = info['_source']
|
||||
if 'district' in source and (source['district'] != item['area_name']):
|
||||
errors.append(source)
|
||||
continue
|
||||
if ('registered_address' not in source) or ('company_name' not in source):
|
||||
continue
|
||||
doc_info = {'province': source['province'], 'city': source['city'], 'district': source['district'],
|
||||
'company_name': source['company_name'], 'registered_address': source['registered_address'],
|
||||
'location': source['gaode_location'], 'township': item['street_name'],
|
||||
'committee': item['name']}
|
||||
if info['_id'] not in company_ids:
|
||||
action = {
|
||||
"_op_type": "index",
|
||||
"_index": "didata_company",
|
||||
"_id": info['_id'],
|
||||
"_source": doc_info,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
else:
|
||||
action = {
|
||||
"_op_type": "update",
|
||||
"_index": "didata_company",
|
||||
"_id": info['_id'],
|
||||
"doc": doc_info,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
|
||||
actions.append(action)
|
||||
if len(actions) == 10000:
|
||||
for success, info in helpers.parallel_bulk(
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||
actions,
|
||||
thread_count=5, # 并行线程数
|
||||
chunk_size=1000 # 批量大小
|
||||
):
|
||||
if not success:
|
||||
print(f"索引失败: {info}")
|
||||
count += len(actions)
|
||||
print("更新:" + str(count))
|
||||
actions.clear()
|
||||
#pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False)
|
||||
if len(actions) > 0:
|
||||
for success, info in helpers.parallel_bulk(
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||
actions,
|
||||
thread_count=5, # 并行线程数
|
||||
chunk_size=1000 # 批量大小
|
||||
):
|
||||
if not success:
|
||||
print(f"索引失败: {info}")
|
||||
count += len(actions)
|
||||
print("更新:" + str(count))
|
||||
actions.clear()
|
||||
#pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False)
|
||||
log.info(f"完成同步{province}企业数据的居委村委数据")
|
||||
for province in areautils.get_all_province():
|
||||
sync(province)
|
||||
@@ -0,0 +1,18 @@
|
||||
from datetime import timedelta
|
||||
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="didata_company",
|
||||
schedule_interval=None,
|
||||
start_date=pendulum.datetime(2025, 6, 15, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="企业涉农处理"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='comm',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/company/company/comm.py',
|
||||
)
|
||||
t1
|
||||
Reference in New Issue
Block a user