优化交通
This commit is contained in:
@@ -71,22 +71,17 @@ def deal(actions, item):
|
|||||||
|
|
||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
log.info(f"开始同步交通至206 {index_name}")
|
log.info(f"开始同步交通至206 {index_name}")
|
||||||
seaweed_query = {'status': 0, 'cat': 'district'}
|
|
||||||
if pd.notna(city):
|
|
||||||
seaweed_query['city'] = city
|
|
||||||
|
|
||||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||||
|
if pd.notna(city):
|
||||||
|
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||||
|
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||||
|
|
||||||
actions = list()
|
actions = list()
|
||||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||||
for index, item in seaweed_df.iterrows():
|
for index, item in seaweed_df.iterrows():
|
||||||
pool.submit(deal, actions, item)
|
pool.submit(deal, actions, item)
|
||||||
|
|
||||||
if len(actions) > 0:
|
if len(actions) > 0:
|
||||||
try:
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
helpers.bulk(es206_client.elastic_client, actions)
|
|
||||||
except Exception as e:
|
|
||||||
print(e)
|
|
||||||
actions.clear()
|
actions.clear()
|
||||||
|
|
||||||
log.info(f"完成同步交通至206 {index_name}")
|
log.info(f"完成同步交通至206 {index_name}")
|
||||||
|
|||||||
Reference in New Issue
Block a user