多线程问题排查

This commit is contained in:
shenjian@fangjia.com
2023-03-09 16:40:07 +08:00
parent 32798ccefc
commit f31502935d
+25 -23
View File
@@ -68,6 +68,29 @@ def build_doc(item):
return doc_dict
def find_around_price_item(seaweed_df, index, item):
if index % 1000 == 0:
log.info("已查找" + str(index))
if 'lng' not in item:
return
lng = commutils.format_lat_lng(item['lng'])
lat = commutils.format_lat_lng(item['lat'])
city_now = areautils.get_city_display(item['city'])
price = find_around_price(city_now, lng, lat)
if item['_id'] == '50bd172ae4b0fbe60a6fe628':
print(city_now)
print(lng)
print(lat)
print(price)
print(index)
if pd.notna(price):
seaweed_df.loc[index, 'avg_cost'] = int(price)
else:
seaweed_df.loc[index, 'avg_cost'] = None
if item['_id'] == '50bd172ae4b0fbe60a6fe628':
print(seaweed_df.loc[index])
def sync(city=None):
log.info("开始同步消费数据...")
# 【过期,无需状态】
@@ -94,30 +117,9 @@ def sync(city=None):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
def find_around_price_item(index, item):
if index % 1000 == 0:
log.info("已查找" + str(index))
if 'lng' not in item:
return
lng = commutils.format_lat_lng(item['lng'])
lat = commutils.format_lat_lng(item['lat'])
city_now = areautils.get_city_display(item['city'])
price = find_around_price(city_now, lng, lat)
if item['_id'] == '50bd172ae4b0fbe60a6fe628':
print(city_now)
print(lng)
print(lat)
print(price)
print(index)
if pd.notna(price):
seaweed_df.loc[index, 'avg_cost'] = int(price)
else:
seaweed_df.loc[index, 'avg_cost'] = None
if item['_id'] == '50bd172ae4b0fbe60a6fe628':
print(seaweed_df.loc[index])
with ThreadPoolExecutor(max_workers=30) as pool:
for index, item in seaweed_df.iterrows():
pool.submit(find_around_price_item, index, item)
pool.submit(find_around_price_item, seaweed_df, index, item)
# for index, item in seaweed_df[seaweed_df['_id'] == '50bd172ae4b0fbe60a6fe628'].iterrows():
# find_around_price_item(index, item)
actions = list()
@@ -149,4 +151,4 @@ def sync(city=None):
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
sync(city)