diff --git a/dags/office/rent/base_price.py b/dags/office/rent/base_price.py new file mode 100644 index 0000000..76dcfbb --- /dev/null +++ b/dags/office/rent/base_price.py @@ -0,0 +1,302 @@ +""" +写字楼基准价 +""" +# 第一步:计算有样本写字楼的基准价 +# 取近一年,最近100个样本的中位值 +# 第二步:计算无样本写字楼的基准价 +# 取周边3公里内同等级、同类型的最近10个有样本写字楼的基准价的平均值 +# 第三步:根据第一步和第二步有写字楼等级和写字楼类型的写字楼基准化计算出写字楼等级和类型对基准价影响的系数 +# 第四步:根据写字楼等级系数和类型系数计算出没有相同特征写字楼的基准化 +# +# 第五步:本身没有等级和类型的写字楼 +# 最近10个有样本写字楼的基准价的平均值 +from goojia_utils import ElasticSearchClient, dateutils, MAIN_MONGO_CLIENT +import pandas as pd +from datetime import datetime +from pymongo import UpdateOne +import math +import numpy as np + +es = ElasticSearchClient(host='192.168.0.206') +seaweed_df = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +one_year_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-365)) +office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat') + +# 计算有样本写字楼的基准价 1. 取近一年,最近100个样本的中位值 +def calc(city, year_month): + """ + 写字楼基准价 + """ + sql = "select * from latest_seawater where cat='office_rent' " + if pd.notna(city): + sql += "and city='" + city + "' " + if pd.notna(year_month): + last_year_month = dateutils.format_date( + dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-13), '%Y%m') + sql += "and p_date >= '" + last_year_month + "01' " + else: + last_year_month = dateutils.format_date(dateutils.add_months(datetime.now(), months=-13), '%Y%m') + sql += "and p_date >= '" + last_year_month + "01' " + date = dateutils.add_months(datetime.today(), months=-1) + year_month = int(dateutils.format_date(date, '%Y%m')) + + sample = es.scan_by_sql_to_data_frame(sql) + office = pd.DataFrame(list(seaweed_df.find({'cat': 'office', 'status': 0}).batch_size(10000))) + office = office[['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']] + office = office.explode('alias') + + sample_new = sample + sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( + 'median') + sample_new['sample_count'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( + 'count') + sample_new = sample_new[sample_new['sample_count'] >= 5] + sample_new = sample_new[['city', 'region', 'district_name', 'sample_avg_price']].dropna() + sample_new['day_avg_price'] = sample_new['sample_avg_price'] / 30 + sample_new = sample_new[['city', 'region', 'district_name', 'day_avg_price']] + sample_new = sample_new.rename(columns={'district_name': 'name'}) + sample_new['alias'] = sample_new['name'] + + # 使用主名与别名查找样本 + merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left', + on=['city', 'region', 'name']) + merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left', + on=['city', 'region', 'alias']) + merge_df = pd.concat([merge_df1, merge_df2]) + merge_df = merge_df[merge_df['day_avg_price'].notna()] + merge_df = merge_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first') + # 第一步:计算有样本写字楼的基准价 + print("有样本数量:" + str(len(merge_df))) + actions = list() + for index, item in merge_df.iterrows(): + city = item['city'] + region = item['region'] + name = item['name'] + avg_price = round(item['day_avg_price'], 2) + location = { + "type": "Point", + "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] + } + estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None + bed_kind = item['bed_kind'] if pd.notna(item['bed_kind']) else None + action = UpdateOne({'city': city, 'cat': 'office', 'date': year_month, 'district': region, 'name': name}, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, 'name': name, + 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind, + 'location': location, + 'calc_type': 1, 'updated_date': datetime.now()}}, + upsert=True) + actions.append(action) + if len(actions) == 1000: + office_rent_stat_conn.bulk_write(actions) + actions.clear() + office_rent_stat_conn.bulk_write(actions) + actions.clear() + + # 第二步 取周边3公里内同等级、同类型的最近10个有样本写字楼的基准价的平均值 + office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates(keep='first') + combined = pd.concat([office_info, merge_df]) + diff_df = combined.drop_duplicates(subset=['city', 'region', 'name'], keep=False) + actions = list() + for index, item in diff_df[(diff_df['estate_type2'].notna()) & (diff_df['bed_kind'].notna())].iterrows(): + location = { + "type": "Point", + "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] + } + city = item['city'] + region = item['region'] + name = item['name'] + estate_type2 = item['estate_type2'] + bed_kind = item['bed_kind'] + if estate_type2 == '' or bed_kind == '': + continue + ret = office_rent_stat_conn.aggregate([ + { + '$geoNear': { + 'near': location, + 'distanceField': "distance", + 'maxDistance': 3000, + 'spherical': True + } + }, + { + '$match': { + 'estate_type2': estate_type2, + 'bed_kind': bed_kind, + 'calc_type': 1 + } + }, + { + '$limit': 10 + } + ]); + # 提取 avg_price 的值 + avg_prices = [doc["avg_price"] for doc in ret] + # 计算平均值 + average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0 + if average_price > 0: + avg_price = round(average_price, 2) + action = UpdateOne({'city': city, 'cat': 'office', 'date': year_month, 'district': region, 'name': name}, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, + 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind, + 'location': location, + 'calc_type': 2, 'updated_date': datetime.now()}}, + upsert=True) + actions.append(action) + if len(actions) == 1000: + office_rent_stat_conn.bulk_write(actions) + actions.clear() + office_rent_stat_conn.bulk_write(actions) + actions.clear() + + # 系数计算在Jupyter中 + bed_kind_dict = {'甲级': 1.111888111888112, '丙级': 0.9143356643356645, '乙级': 1} + radio_dict = {'创业园区': 0.7799999999999999, '商住两用': 1.0466666666666666, '标准写字楼': 1} + # 根据写字楼等级系数和类型系数计算出没有相同特征写字楼的基准化 + actions.clear() + for index, item in diff_df[(diff_df['estate_type2'].notna()) | (diff_df['bed_kind'].notna())].iterrows(): + location = { + "type": "Point", + "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] + } + city = item['city'] + region = item['region'] + name = item['name'] + estate_type2 = item['estate_type2'] + bed_kind = item['bed_kind'] + if estate_type2 == '' or bed_kind == '': + continue + if pd.notna(estate_type2) and pd.notna(bed_kind): + continue + if pd.notna(estate_type2): + ret = office_rent_stat_conn.aggregate([ + { + '$geoNear': { + 'near': location, + 'distanceField': "distance", + 'maxDistance': 3000, + 'spherical': True + } + }, + { + '$match': { + 'estate_type2': estate_type2 + } + }, + { + '$limit': 10 + } + ]); + # 提取 avg_price 的值 + count = 0 + sum_price = 0 + if estate_type2 in radio_dict.keys(): + for doc in ret: + if doc['estate_type2'] in radio_dict.keys(): + new_price = doc['avg_price'] / radio_dict[doc['estate_type2']] * radio_dict[estate_type2] + sum_price += new_price + count += 1 + if count > 0: + avg_price = round(sum_price / count, 2) + action = UpdateOne({'city': city, 'cat': 'office', 'date': year_month, 'district': region, 'name': name}, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, + 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None, + 'location': location, + 'calc_type': 3, 'updated_date': datetime.now()}}, + upsert=True) + actions.append(action) + elif pd.notna(bed_kind): + ret = office_rent_stat_conn.aggregate([ + { + '$geoNear': { + 'near': location, + 'distanceField': "distance", + 'maxDistance': 3000, + 'spherical': True + } + }, + { + '$match': { + 'bed_kind': bed_kind + } + }, + { + '$limit': 10 + } + ]); + # 提取 avg_price 的值 + count = 0 + sum_price = 0 + if bed_kind in bed_kind_dict.keys(): + for doc in ret: + if doc['bed_kind'] in bed_kind_dict.keys(): + new_price = doc['avg_price'] / bed_kind_dict[doc['bed_kind']] * bed_kind_dict[bed_kind] + sum_price += new_price + count += 1 + if count > 0: + avg_price = round(sum_price / count, 2) + action = UpdateOne({'city': city, 'cat': 'office', 'date': year_month, 'district': region, 'name': name}, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, + 'avg_price': avg_price, 'estate_type2': None, 'bed_kind': bed_kind, + 'location': location, + 'calc_type': 3, 'updated_date': datetime.now()}}, + upsert=True) + actions.append(action) + if len(actions) == 5000: + office_rent_stat_conn.bulk_write(actions) + actions.clear() + + # 第五步 + db_df = pd.DataFrame(list(office_rent_stat_conn.find({'cat': 'office', 'date': year_month, 'calc_type': {'$in': [1, 2, 3]}}))) + office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates( + keep='first') + # 计算价格的写字楼 + db_df = db_df.rename(columns={'district': 'region'}) + combined = pd.concat([office_info, db_df[['city', 'region', 'name']]]) + diff_df = combined.drop_duplicates(subset=['city', 'region', 'name'], keep=False) + actions.clear() + for index, item in diff_df.iterrows(): + location = { + "type": "Point", + "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] + } + city = item['city'] + region = item['region'] + name = item['name'] + estate_type2 = item['estate_type2'] + bed_kind = item['bed_kind'] + ret = office_rent_stat_conn.aggregate([ + { + '$geoNear': { + 'near': location, + 'distanceField': "distance", + 'maxDistance': 3000, + 'spherical': True + } + }, + { + '$limit': 10 + } + ]); + # 提取 avg_price 的值 + avg_prices = [doc["avg_price"] for doc in ret] + # 计算平均值 + average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0 + if average_price > 0: + avg_price = round(average_price, 2) + action = UpdateOne({'city': city, 'cat': 'office', 'date': year_month, 'district': region, 'name': name}, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, + 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None, + 'location': location, + 'calc_type': 4, 'updated_date': datetime.now()}}, + upsert=True) + actions.append(action) + if len(actions) == 5000: + office_rent_stat_conn.bulk_write(actions) + actions.clear() + office_rent_stat_conn.bulk_write(actions) + +calc('上海', 202412) \ No newline at end of file