添加API启动支持
This commit is contained in:
@@ -164,64 +164,66 @@ def update_office(city, office, sample, year_month):
|
||||
office['estate_type2'] = None
|
||||
if 'alias' not in office.columns:
|
||||
office['alias'] = None
|
||||
if sample.empty:
|
||||
return
|
||||
sample_new = sample
|
||||
if 'alias' not in sample_new.columns:
|
||||
sample_new['alias'] = None
|
||||
sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform(
|
||||
'median')
|
||||
sample_new['sample_count'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform(
|
||||
'count')
|
||||
sample_new = sample_new[sample_new['sample_count'] >= 5]
|
||||
sample_new = sample_new[['city', 'region', 'district_name', 'sample_avg_price']].dropna()
|
||||
sample_new['day_avg_price'] = sample_new['sample_avg_price']
|
||||
sample_new = sample_new[['city', 'region', 'district_name', 'day_avg_price']]
|
||||
sample_new = sample_new.rename(columns={'district_name': 'name'})
|
||||
sample_new['alias'] = sample_new['name']
|
||||
# 使用主名与别名查找样本
|
||||
merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left',
|
||||
on=['city', 'region', 'name'])
|
||||
if len(sample_new['alias'].unique()) > 0:
|
||||
merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left',
|
||||
on=['city', 'region', 'alias'])
|
||||
merge_df = pd.concat([merge_df1, merge_df2])
|
||||
else:
|
||||
merge_df = merge_df1.copy()
|
||||
merge_df = merge_df[merge_df['day_avg_price'].notna()]
|
||||
merge_df = merge_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first')
|
||||
# 第一步:计算有样本写字楼的基准价
|
||||
print("有样本数量:" + str(len(merge_df)))
|
||||
actions = list()
|
||||
for index, item in merge_df.iterrows():
|
||||
city = item['city']
|
||||
region = item['region']
|
||||
name = item['name']
|
||||
avg_price = int(item['day_avg_price'])
|
||||
location = {
|
||||
"type": "Point",
|
||||
"coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)]
|
||||
}
|
||||
estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None
|
||||
bed_kind = item['bed_kind'] if pd.notna(item['bed_kind']) else None
|
||||
action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind,
|
||||
'location': location,
|
||||
'calc_type': 1, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
if len(actions) == 1000:
|
||||
merge_df = None
|
||||
if not sample.empty:
|
||||
sample_new = sample
|
||||
if 'alias' not in sample_new.columns:
|
||||
sample_new['alias'] = None
|
||||
sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform(
|
||||
'median')
|
||||
sample_new['sample_count'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform(
|
||||
'count')
|
||||
sample_new = sample_new[sample_new['sample_count'] >= 5]
|
||||
sample_new = sample_new[['city', 'region', 'district_name', 'sample_avg_price']].dropna()
|
||||
sample_new['day_avg_price'] = sample_new['sample_avg_price']
|
||||
sample_new = sample_new[['city', 'region', 'district_name', 'day_avg_price']]
|
||||
sample_new = sample_new.rename(columns={'district_name': 'name'})
|
||||
sample_new['alias'] = sample_new['name']
|
||||
# 使用主名与别名查找样本
|
||||
merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left',
|
||||
on=['city', 'region', 'name'])
|
||||
if len(sample_new['alias'].unique()) > 0:
|
||||
merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left',
|
||||
on=['city', 'region', 'alias'])
|
||||
merge_df = pd.concat([merge_df1, merge_df2])
|
||||
else:
|
||||
merge_df = merge_df1.copy()
|
||||
merge_df = merge_df[merge_df['day_avg_price'].notna()]
|
||||
merge_df = merge_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first')
|
||||
# 第一步:计算有样本写字楼的基准价
|
||||
print("有样本数量:" + str(len(merge_df)))
|
||||
actions = list()
|
||||
for index, item in merge_df.iterrows():
|
||||
city = item['city']
|
||||
region = item['region']
|
||||
name = item['name']
|
||||
avg_price = int(item['day_avg_price'])
|
||||
location = {
|
||||
"type": "Point",
|
||||
"coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)]
|
||||
}
|
||||
estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None
|
||||
bed_kind = item['bed_kind'] if pd.notna(item['bed_kind']) else None
|
||||
action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind,
|
||||
'location': location,
|
||||
'calc_type': 1, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
if len(actions) == 1000:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
# 第二步 取周边3公里内同等级、同类型的最近10个有样本写字楼的基准价的平均值
|
||||
office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates(
|
||||
keep='first')
|
||||
combined = pd.concat([office_info, merge_df])
|
||||
combined = office_info
|
||||
if merge_df is not None and not merge_df.empty:
|
||||
combined = pd.concat([office_info, merge_df])
|
||||
diff_df = combined.drop_duplicates(subset=['city', 'region', 'name'], keep=False)
|
||||
actions = list()
|
||||
for index, item in diff_df[
|
||||
@@ -464,7 +466,7 @@ def getOffice(city, district=None, name=None):
|
||||
query['city'] = city
|
||||
if pd.notna(district):
|
||||
query['region'] = district
|
||||
if pd.notna(city):
|
||||
if pd.notna(name):
|
||||
query['name'] = name
|
||||
office = pd.DataFrame(list(seaweed_df.find(query).batch_size(10000)))
|
||||
for col in ['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']:
|
||||
@@ -477,4 +479,4 @@ def getOffice(city, district=None, name=None):
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 测试脚本:直接运行时模拟 Airflow 的上下文
|
||||
calc(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}}))
|
||||
calc(dag_run=type('DagRun', (), {'conf': {'city': '西安'}}))
|
||||
Reference in New Issue
Block a user