diff --git a/dags/company/company/fill_company_comm.py b/dags/company/company/fill_company_comm.py index 122d472..65acd1d 100644 --- a/dags/company/company/fill_company_comm.py +++ b/dags/company/company/fill_company_comm.py @@ -16,6 +16,7 @@ def sync(province): if len(shape_df) == 0 or 'shape' not in shape_df.columns or len(shape_df[shape_df['shape'].notna()]) == 0: logger.info(f"{province}无围栏数据") return + logger.info(f"已加载{len(shape_df)}条围栏") # 先临时禁用索引刷新 DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( index="didata_company", @@ -30,9 +31,11 @@ def sync(province): count = 0 errors = list() for index, item in shape_df[shape_df['shape'].notna()].iterrows(): - logger.info(index) + logger.info(f"开始查找{item['city_name'], item['area_name'], item['name']}下企业数据") ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_enterprise', item['shape']['coordinates'][0], field='gaode_location') + com_count = 0 for info in ret: + com_count += 1 source = info['_source'] if 'district' in source and (source['district'] != item['area_name']): errors.append(source) @@ -61,20 +64,20 @@ def sync(province): } actions.append(action) - if len(actions) == 10000: - for success, info in helpers.parallel_bulk( - DIDATA_DEV_ELASTIC_CLIENT.elastic_client, - actions, - thread_count=5, # 并行线程数 - refresh=False, - chunk_size=1000 # 批量大小 - ): - if not success: - logger.info(f"索引失败: {info}") - count += len(actions) - logger.info("更新:" + str(count)) - actions.clear() #pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False) + logger.info(f"已查找到该围栏下{com_count}个企业数据") + for success, info in helpers.parallel_bulk( + DIDATA_DEV_ELASTIC_CLIENT.elastic_client, + actions, + thread_count=5, # 并行线程数 + refresh=False, + chunk_size=1000 # 批量大小 + ): + if not success: + logger.info(f"索引失败: {info}") + count += len(actions) + logger.info("更新:" + str(count)) + actions.clear() if len(actions) > 0: for success, info in helpers.parallel_bulk( DIDATA_DEV_ELASTIC_CLIENT.elastic_client,