解决airflow苏州城市price同步bug
This commit is contained in:
@@ -79,16 +79,22 @@ def sync_by_city(city):
|
||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
|
||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
|
||||
columns={'district_name': 'name', 'up_and_downs': 'year_on_year'})
|
||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first').reset_index(
|
||||
drop=True).copy()
|
||||
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
|
||||
|
||||
|
||||
# 近半年均价涨跌幅
|
||||
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-6)
|
||||
if len(avg_price_up_and_downs_df) > 0:
|
||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
|
||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
|
||||
columns={'district_name': 'name', 'up_and_downs': 'half_on_year'})
|
||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first').reset_index(
|
||||
drop=True).copy()
|
||||
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
|
||||
|
||||
|
||||
# 平米租金
|
||||
rent_price_df = commutils.get_rent_price(city=city)
|
||||
if len(rent_price_df) > 0:
|
||||
@@ -102,7 +108,6 @@ def sync_by_city(city):
|
||||
gov_price_df = gov_price_df[['city', 'region', 'name', 'avg_price']]
|
||||
gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'})
|
||||
seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name'])
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in seaweed_df.iterrows():
|
||||
@@ -114,9 +119,13 @@ def sync_by_city(city):
|
||||
"_id": item['_id'],
|
||||
"doc": doc_info
|
||||
}
|
||||
|
||||
if index < 6500:
|
||||
continue
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
|
||||
if len(actions) == 1:
|
||||
print(index)
|
||||
print(item)
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
log.info("更新:" + str(count))
|
||||
|
||||
Reference in New Issue
Block a user