解决airflow苏州城市price同步bug

This commit is contained in:
shenjian@fangjia.com
2023-04-25 13:24:21 +08:00
parent 2f7478f017
commit 29f827e2cc
+12 -3
View File
@@ -79,16 +79,22 @@ def sync_by_city(city):
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
columns={'district_name': 'name', 'up_and_downs': 'year_on_year'})
avg_price_up_and_downs_df = avg_price_up_and_downs_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first').reset_index(
drop=True).copy()
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
# 近半年均价涨跌幅
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-6)
if len(avg_price_up_and_downs_df) > 0:
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
columns={'district_name': 'name', 'up_and_downs': 'half_on_year'})
avg_price_up_and_downs_df = avg_price_up_and_downs_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first').reset_index(
drop=True).copy()
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
# 平米租金
rent_price_df = commutils.get_rent_price(city=city)
if len(rent_price_df) > 0:
@@ -102,7 +108,6 @@ def sync_by_city(city):
gov_price_df = gov_price_df[['city', 'region', 'name', 'avg_price']]
gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'})
seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name'])
actions = list()
count = 0
for index, item in seaweed_df.iterrows():
@@ -114,9 +119,13 @@ def sync_by_city(city):
"_id": item['_id'],
"doc": doc_info
}
if index < 6500:
continue
actions.append(action)
if len(actions) == 5000:
if len(actions) == 1:
print(index)
print(item)
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
log.info("更新:" + str(count))