diff --git a/dags/office/sell/base_price.py b/dags/office/sell/base_price.py index 902c57f..c0c10f6 100644 --- a/dags/office/sell/base_price.py +++ b/dags/office/sell/base_price.py @@ -164,64 +164,66 @@ def update_office(city, office, sample, year_month): office['estate_type2'] = None if 'alias' not in office.columns: office['alias'] = None - if sample.empty: - return - sample_new = sample - if 'alias' not in sample_new.columns: - sample_new['alias'] = None - sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( - 'median') - sample_new['sample_count'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( - 'count') - sample_new = sample_new[sample_new['sample_count'] >= 5] - sample_new = sample_new[['city', 'region', 'district_name', 'sample_avg_price']].dropna() - sample_new['day_avg_price'] = sample_new['sample_avg_price'] - sample_new = sample_new[['city', 'region', 'district_name', 'day_avg_price']] - sample_new = sample_new.rename(columns={'district_name': 'name'}) - sample_new['alias'] = sample_new['name'] - # 使用主名与别名查找样本 - merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left', - on=['city', 'region', 'name']) - if len(sample_new['alias'].unique()) > 0: - merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left', - on=['city', 'region', 'alias']) - merge_df = pd.concat([merge_df1, merge_df2]) - else: - merge_df = merge_df1.copy() - merge_df = merge_df[merge_df['day_avg_price'].notna()] - merge_df = merge_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first') - # 第一步:计算有样本写字楼的基准价 - print("有样本数量:" + str(len(merge_df))) - actions = list() - for index, item in merge_df.iterrows(): - city = item['city'] - region = item['region'] - name = item['name'] - avg_price = int(item['day_avg_price']) - location = { - "type": "Point", - "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] - } - estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None - bed_kind = item['bed_kind'] if pd.notna(item['bed_kind']) else None - action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, - {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, - 'name': name, - 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind, - 'location': location, - 'calc_type': 1, 'updated_date': datetime.now()}}, - upsert=True) - actions.append(action) - if len(actions) == 1000: + merge_df = None + if not sample.empty: + sample_new = sample + if 'alias' not in sample_new.columns: + sample_new['alias'] = None + sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( + 'median') + sample_new['sample_count'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( + 'count') + sample_new = sample_new[sample_new['sample_count'] >= 5] + sample_new = sample_new[['city', 'region', 'district_name', 'sample_avg_price']].dropna() + sample_new['day_avg_price'] = sample_new['sample_avg_price'] + sample_new = sample_new[['city', 'region', 'district_name', 'day_avg_price']] + sample_new = sample_new.rename(columns={'district_name': 'name'}) + sample_new['alias'] = sample_new['name'] + # 使用主名与别名查找样本 + merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left', + on=['city', 'region', 'name']) + if len(sample_new['alias'].unique()) > 0: + merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left', + on=['city', 'region', 'alias']) + merge_df = pd.concat([merge_df1, merge_df2]) + else: + merge_df = merge_df1.copy() + merge_df = merge_df[merge_df['day_avg_price'].notna()] + merge_df = merge_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first') + # 第一步:计算有样本写字楼的基准价 + print("有样本数量:" + str(len(merge_df))) + actions = list() + for index, item in merge_df.iterrows(): + city = item['city'] + region = item['region'] + name = item['name'] + avg_price = int(item['day_avg_price']) + location = { + "type": "Point", + "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] + } + estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None + bed_kind = item['bed_kind'] if pd.notna(item['bed_kind']) else None + action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, + 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind, + 'location': location, + 'calc_type': 1, 'updated_date': datetime.now()}}, + upsert=True) + actions.append(action) + if len(actions) == 1000: + office_sell_stat_conn.bulk_write(actions) + actions.clear() + if len(actions) > 0: office_sell_stat_conn.bulk_write(actions) actions.clear() - if len(actions) > 0: - office_sell_stat_conn.bulk_write(actions) - actions.clear() # 第二步 取周边3公里内同等级、同类型的最近10个有样本写字楼的基准价的平均值 office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates( keep='first') - combined = pd.concat([office_info, merge_df]) + combined = office_info + if merge_df is not None and not merge_df.empty: + combined = pd.concat([office_info, merge_df]) diff_df = combined.drop_duplicates(subset=['city', 'region', 'name'], keep=False) actions = list() for index, item in diff_df[ @@ -464,7 +466,7 @@ def getOffice(city, district=None, name=None): query['city'] = city if pd.notna(district): query['region'] = district - if pd.notna(city): + if pd.notna(name): query['name'] = name office = pd.DataFrame(list(seaweed_df.find(query).batch_size(10000))) for col in ['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']: @@ -477,4 +479,4 @@ def getOffice(city, district=None, name=None): if __name__ == "__main__": # 测试脚本:直接运行时模拟 Airflow 的上下文 - calc(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}})) \ No newline at end of file + calc(dag_run=type('DagRun', (), {'conf': {'city': '西安'}})) \ No newline at end of file