多线程问题排查
This commit is contained in:
@@ -68,6 +68,29 @@ def build_doc(item):
|
|||||||
return doc_dict
|
return doc_dict
|
||||||
|
|
||||||
|
|
||||||
|
def find_around_price_item(seaweed_df, index, item):
|
||||||
|
if index % 1000 == 0:
|
||||||
|
log.info("已查找" + str(index))
|
||||||
|
if 'lng' not in item:
|
||||||
|
return
|
||||||
|
lng = commutils.format_lat_lng(item['lng'])
|
||||||
|
lat = commutils.format_lat_lng(item['lat'])
|
||||||
|
city_now = areautils.get_city_display(item['city'])
|
||||||
|
price = find_around_price(city_now, lng, lat)
|
||||||
|
if item['_id'] == '50bd172ae4b0fbe60a6fe628':
|
||||||
|
print(city_now)
|
||||||
|
print(lng)
|
||||||
|
print(lat)
|
||||||
|
print(price)
|
||||||
|
print(index)
|
||||||
|
if pd.notna(price):
|
||||||
|
seaweed_df.loc[index, 'avg_cost'] = int(price)
|
||||||
|
else:
|
||||||
|
seaweed_df.loc[index, 'avg_cost'] = None
|
||||||
|
if item['_id'] == '50bd172ae4b0fbe60a6fe628':
|
||||||
|
print(seaweed_df.loc[index])
|
||||||
|
|
||||||
|
|
||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
log.info("开始同步消费数据...")
|
log.info("开始同步消费数据...")
|
||||||
# 【过期,无需状态】
|
# 【过期,无需状态】
|
||||||
@@ -94,30 +117,9 @@ def sync(city=None):
|
|||||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||||
|
|
||||||
def find_around_price_item(index, item):
|
|
||||||
if index % 1000 == 0:
|
|
||||||
log.info("已查找" + str(index))
|
|
||||||
if 'lng' not in item:
|
|
||||||
return
|
|
||||||
lng = commutils.format_lat_lng(item['lng'])
|
|
||||||
lat = commutils.format_lat_lng(item['lat'])
|
|
||||||
city_now = areautils.get_city_display(item['city'])
|
|
||||||
price = find_around_price(city_now, lng, lat)
|
|
||||||
if item['_id'] == '50bd172ae4b0fbe60a6fe628':
|
|
||||||
print(city_now)
|
|
||||||
print(lng)
|
|
||||||
print(lat)
|
|
||||||
print(price)
|
|
||||||
print(index)
|
|
||||||
if pd.notna(price):
|
|
||||||
seaweed_df.loc[index, 'avg_cost'] = int(price)
|
|
||||||
else:
|
|
||||||
seaweed_df.loc[index, 'avg_cost'] = None
|
|
||||||
if item['_id'] == '50bd172ae4b0fbe60a6fe628':
|
|
||||||
print(seaweed_df.loc[index])
|
|
||||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||||
for index, item in seaweed_df.iterrows():
|
for index, item in seaweed_df.iterrows():
|
||||||
pool.submit(find_around_price_item, index, item)
|
pool.submit(find_around_price_item, seaweed_df, index, item)
|
||||||
# for index, item in seaweed_df[seaweed_df['_id'] == '50bd172ae4b0fbe60a6fe628'].iterrows():
|
# for index, item in seaweed_df[seaweed_df['_id'] == '50bd172ae4b0fbe60a6fe628'].iterrows():
|
||||||
# find_around_price_item(index, item)
|
# find_around_price_item(index, item)
|
||||||
actions = list()
|
actions = list()
|
||||||
@@ -149,4 +151,4 @@ def sync(city=None):
|
|||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
sync(city)
|
sync(city)
|
||||||
|
|||||||
Reference in New Issue
Block a user