From 43e9b5af1472ec0da701539a2fe9615760eec765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 10 Jun 2025 14:04:11 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/dag_office_rent_base_price.py | 9 +- dags/office/rent/base_price.py | 301 +++++++++++++++------- dags/office/sell/base_price.py | 19 +- dags/office/sell/fill_base_price.py | 22 +- 4 files changed, 234 insertions(+), 117 deletions(-) diff --git a/dags/office/dag_office_rent_base_price.py b/dags/office/dag_office_rent_base_price.py index dfef620..d78e609 100644 --- a/dags/office/dag_office_rent_base_price.py +++ b/dags/office/dag_office_rent_base_price.py @@ -1,9 +1,10 @@ """ 写字楼基准价计算 """ -from airflow.operators.bash import BashOperator +from airflow.operators.bash import BashOperator, PythonOperator import pendulum +from sell.base_price import calc from airflow import DAG @@ -14,9 +15,9 @@ with DAG( catchup=True, description="写字楼基准价计算" ) as dag: - t1 = BashOperator( + t1 = PythonOperator( task_id='base_price', - bash_command='python /opt/airflow/dags/repo/dags/office/rent/base_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}', + python_callable=calc, + provide_context=True ) t1 - diff --git a/dags/office/rent/base_price.py b/dags/office/rent/base_price.py index 47c95c3..6037757 100644 --- a/dags/office/rent/base_price.py +++ b/dags/office/rent/base_price.py @@ -10,7 +10,7 @@ # # 第五步:本身没有等级和类型的写字楼 # 最近10个有样本写字楼的基准价的平均值 -from goojia_utils import ElasticSearchClient, dateutils, MAIN_MONGO_CLIENT +from goojia_utils import ElasticSearchClient, dateutils, MAIN_MONGO_CLIENT, commutils import pandas as pd from datetime import datetime from pymongo import UpdateOne @@ -23,7 +23,103 @@ seaweed_df = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat') # 计算有样本写字楼的基准价 1. 取近一年,最近100个样本的中位值 -def calc(city, year_month): + +def calc(**kwargs): + """ + 写字楼基准价 + """ + # 从 Airflow 的 dag_run.conf 获取参数 + conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {} + city = conf.get('city', '') + year_month = conf.get('year_month', '') + district = conf.get('district', '') + office_name = conf.get('name', '') + + if pd.isna(office_name): + calcByCity(city, year_month) + else: + # 直接重算出最近一年的数据,保证一年有数据 + calcByOffice(city, district, office_name) + +def calcByOffice(city, district, office_name): + """ + 写字楼价格重算 + """ + query = {'cat': 'office', 'status': 0} + if pd.notna(city): + query['city'] = city + if pd.notna(district): + query['region'] = district + if pd.notna(city): + query['name'] = office_name + office = seaweed_df.find_one(query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'lng': 1, 'lat': 1, 'estate_type2': 1, 'bed_kind': 1}) + names = list() + names.append(office['name']) + if type(office['alias']) == list: + for alias_item in office['alias']: + names.append(alias_item) + names_str = "','".join(names) + names_str = "'" + names_str + "'" + + # 只查询该小区的样本 + year_time = dateutils.format_date(dateutils.datetime.datetime.now()) + one_year_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-365)) + sql = f"select * from latest_seawater where cat='office_rent' and city='{city}' and region='{district}' and district_name in ({names_str}) and p_date > {one_year_time}" + sample = es.scan_by_sql_to_data_frame(sql) + + location = { + "type": "Point", + "coordinates": [office['lng'] / math.pow(10, 10), office['lat'] / math.pow(10, 10)] + } + + date_range = pd.date_range(start=one_year_time, end=year_time, freq='M') + for date in date_range: + curr_date = int(dateutils.format_date(date, fm='%Y%m')) + # 筛选出当前月的样本 + curr_sample = sample[(sample['p_date']>=int(str(curr_date)+'01')) & (sample['p_date']<=int(str(curr_date)+'31'))] + update_office(city, pd.DataFrame(office), curr_sample, curr_date) + # 核查当月写字楼是否存在,不存在取最后6个月的平均值 + office_db = office_rent_stat_conn.find_one({'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date}) + if pd.isna(office_db): + avg_price_ret = office_rent_stat_conn.aggregate([ + { + "$match": { + "cat": "office", + "city": city, + "district": district, + "name": office_name, + "date": { "$gte": curr_date - 3, "$lte": curr_date + 3 } + } + }, + { + "$group": { + "_id": "$name", + "avg_price": { "$avg": "$avg_price" } + } + } + ]) + avg_prices = [doc["avg_price"] for doc in avg_price_ret] + # 计算平均值 + average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0 + if average_price > 0: + new_avg_price = average_price + else: + # 取样本的平均值 + new_avg_price = sample['avg_price'].mean() + if pd.notna(new_avg_price): + action = UpdateOne( + {'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date}, + {'$set': {'city': city, 'cat': 'office', 'date': curr_date, 'district': district, + 'name': office_name, + 'avg_price': new_avg_price, 'estate_type2': office['estate_type2'], + 'bed_kind': office['bed_kind'], + 'location': location, + 'calc_type': 5, 'updated_date': datetime.now()}}, + upsert=True) + office_rent_stat_conn.bulk_write([action]) + + +def calcByCity(city, year_month): """ 写字楼基准价 """ @@ -46,13 +142,12 @@ def calc(city, year_month): print("开始计算年" + str(year_month) + "写字楼基准价") sample = es.scan_by_sql_to_data_frame(sql) - query = {'cat': 'office', 'status': 0} - if pd.notna(city): - query['city'] = city - office = pd.DataFrame(list(seaweed_df.find(query).batch_size(10000))) - office = office[['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']] - office = office.explode('alias') + office = getOffice(city) + update_office(city, office, sample, year_month) + + +def update_office(city, office, sample, year_month): sample_new = sample sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( 'median') @@ -64,7 +159,6 @@ def calc(city, year_month): sample_new = sample_new[['city', 'region', 'district_name', 'day_avg_price']] sample_new = sample_new.rename(columns={'district_name': 'name'}) sample_new['alias'] = sample_new['name'] - # 使用主名与别名查找样本 merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left', on=['city', 'region', 'name']) @@ -88,7 +182,8 @@ def calc(city, year_month): estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None bed_kind = item['bed_kind'] if pd.notna(item['bed_kind']) else None action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, - {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, 'name': name, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind, 'location': location, 'calc_type': 1, 'updated_date': datetime.now()}}, @@ -100,13 +195,14 @@ def calc(city, year_month): if len(actions) > 0: office_rent_stat_conn.bulk_write(actions) actions.clear() - # 第二步 取周边3公里内同等级、同类型的最近10个有样本写字楼的基准价的平均值 - office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates(keep='first') + office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates( + keep='first') combined = pd.concat([office_info, merge_df]) diff_df = combined.drop_duplicates(subset=['city', 'region', 'name'], keep=False) actions = list() - for index, item in diff_df[(diff_df['estate_type2'].notna()) & (diff_df['bed_kind'].notna()) & (diff_df['lng'].notna()) ].iterrows(): + for index, item in diff_df[ + (diff_df['estate_type2'].notna()) & (diff_df['bed_kind'].notna()) & (diff_df['lng'].notna())].iterrows(): location = { "type": "Point", "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] @@ -159,7 +255,6 @@ def calc(city, year_month): if len(actions) > 0: office_rent_stat_conn.bulk_write(actions) actions.clear() - # 系数计算在Jupyter中 bed_kind_dict = {'甲级': 1.111888111888112, '丙级': 0.9143356643356645, '乙级': 1} radio_dict = {'创业园区': 0.7799999999999999, '商住两用': 1.0466666666666666, '标准写字楼': 1} @@ -210,13 +305,14 @@ def calc(city, year_month): count += 1 if count > 0: avg_price = round(sum_price / count, 2) - action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, - {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, - 'name': name, - 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None, - 'location': location, - 'calc_type': 3, 'updated_date': datetime.now()}}, - upsert=True) + action = UpdateOne( + {'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, + 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None, + 'location': location, + 'calc_type': 3, 'updated_date': datetime.now()}}, + upsert=True) actions.append(action) elif pd.notna(bed_kind): ret = office_rent_stat_conn.aggregate([ @@ -249,13 +345,14 @@ def calc(city, year_month): count += 1 if count > 0: avg_price = round(sum_price / count, 2) - action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, - {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, - 'name': name, - 'avg_price': avg_price, 'estate_type2': None, 'bed_kind': bed_kind, - 'location': location, - 'calc_type': 3, 'updated_date': datetime.now()}}, - upsert=True) + action = UpdateOne( + {'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, + 'avg_price': avg_price, 'estate_type2': None, 'bed_kind': bed_kind, + 'location': location, + 'calc_type': 3, 'updated_date': datetime.now()}}, + upsert=True) actions.append(action) if len(actions) == 1000: office_rent_stat_conn.bulk_write(actions) @@ -273,79 +370,85 @@ def calc(city, year_month): # 计算价格的写字楼 db_df = db_df.rename(columns={'district': 'region'}) # 使用 merge 操作并结合 indicator 参数 - merged = office_info.merge(db_df[['city', 'region', 'name']], on=['city', 'region', 'name'], how='left', - indicator=True) - - # 过滤出仅在 office_info 中存在的记录 - diff_df = merged[merged['_merge'] == 'left_only'].drop(columns=['_merge']) - - # 如果需要保持原始列顺序,可以重新排列列 - diff_df = diff_df[office_info.columns] - - actions.clear() - for index, item in diff_df.iterrows(): - try: - location = { - "type": "Point", - "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] - } - city = item['city'] - region = item['region'] - name = item['name'] - estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None - bed_kind = item['bed_kind'] - ret = office_rent_stat_conn.aggregate([ - { - '$geoNear': { - 'near': location, - 'distanceField': "distance", - 'maxDistance': 3000, - 'spherical': True - } - }, - { - '$match': { - 'date': year_month - } - }, - { - '$limit': 5 + if len(db_df) > 0: + merged = office_info.merge(db_df[['city', 'region', 'name']], on=['city', 'region', 'name'], how='left', + indicator=True) + # 过滤出仅在 office_info 中存在的记录 + diff_df = merged[merged['_merge'] == 'left_only'].drop(columns=['_merge']) + # 如果需要保持原始列顺序,可以重新排列列 + diff_df = diff_df[office_info.columns] + actions.clear() + for index, item in diff_df.iterrows(): + try: + location = { + "type": "Point", + "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] } - ]); - except Exception as e: - print("Error occurred:", str(e)) - print(item) - # 提取 avg_price 的值 - avg_prices = [doc["avg_price"] for doc in ret] - # 计算平均值 - average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0 - if average_price > 0: - avg_price = round(average_price, 2) - action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, - {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, - 'name': name, - 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None, - 'location': location, - 'calc_type': 4, 'updated_date': datetime.now()}}, - upsert=True) - actions.append(action) - if len(actions) == 1000: - office_rent_stat_conn.bulk_write(actions) - actions.clear() + city = item['city'] + region = item['region'] + name = item['name'] + estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None + bed_kind = item['bed_kind'] + ret = office_rent_stat_conn.aggregate([ + { + '$geoNear': { + 'near': location, + 'distanceField': "distance", + 'maxDistance': 3000, + 'spherical': True + } + }, + { + '$match': { + 'date': year_month + } + }, + { + '$limit': 5 + } + ]); + except Exception as e: + print("Error occurred:", str(e)) + print(item) + # 提取 avg_price 的值 + avg_prices = [doc["avg_price"] for doc in ret] + # 计算平均值 + average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0 + if average_price > 0: + avg_price = round(average_price, 2) + action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, + 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None, + 'location': location, + 'calc_type': 4, 'updated_date': datetime.now()}}, + upsert=True) + actions.append(action) + if len(actions) == 1000: + office_rent_stat_conn.bulk_write(actions) + actions.clear() if len(actions) > 0: office_rent_stat_conn.bulk_write(actions) actions.clear() -yearmonth = sys.argv[1] if len(sys.argv) > 1 else None -# for yearmonth in [ -# "202406", -# "202407", -# "202408", -# "202409", -# "202410", -# "202411", -# "202412", -# "202501" -# ]: -# calc(None, yearmonth) -calc(None, yearmonth) \ No newline at end of file + +def getOffice(city, district, name): + """ + 获取写字楼 + """ + query = {'cat': 'office', 'status': 0} + if pd.notna(city): + query['city'] = city + if pd.notna(district): + query['region'] = district + if pd.notna(city): + query['name'] = name + office = pd.DataFrame(list(seaweed_df.find(query).batch_size(10000))) + office = office[['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']] + office = office.explode('alias') + return office + + +if __name__ == "__main__": + # 测试脚本:直接运行时模拟 Airflow 的上下文 + calc(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}})) \ No newline at end of file diff --git a/dags/office/sell/base_price.py b/dags/office/sell/base_price.py index cf8c0bd..b455270 100644 --- a/dags/office/sell/base_price.py +++ b/dags/office/sell/base_price.py @@ -23,10 +23,18 @@ seaweed_df = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat') # 计算有样本写字楼的基准价 1. 取近一年,最近100个样本的中位值 -def calc(city, year_month): +def calc(**kwargs): """ 写字楼基准价 """ + # 从 Airflow 的 dag_run.conf 获取参数 + conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {} + city = conf.get('city', '') + year_month = conf.get('year_month', '') + district = conf.get('district', '') + office_name = conf.get('name', '') + + sql = "select * from latest_seawater where cat='office_sell' " if pd.notna(city): sql += "and city='" + city + "' " @@ -49,6 +57,10 @@ def calc(city, year_month): query = {'cat': 'office', 'status': 0} if pd.notna(city): query['city'] = city + if pd.notna(district): + query['region'] = district + if pd.notna(office_name): + query['name'] = office_name office = pd.DataFrame(list(seaweed_df.find(query).batch_size(10000))) office = office[['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']] office = office.explode('alias') @@ -353,5 +365,6 @@ def calc(city, year_month): print("已计算" + str(count) + "条数据") actions.clear() -yearmonth = sys.argv[1] if len(sys.argv) > 1 else None -calc(None, yearmonth) \ No newline at end of file +if __name__ == "__main__": + # 测试脚本:直接运行时模拟 Airflow 的上下文 + calc(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}})) \ No newline at end of file diff --git a/dags/office/sell/fill_base_price.py b/dags/office/sell/fill_base_price.py index d1c6b01..7245f8a 100644 --- a/dags/office/sell/fill_base_price.py +++ b/dags/office/sell/fill_base_price.py @@ -83,16 +83,16 @@ def calc(city, year_month): print("已处理{}条数据".format(count)) print("处理完毕") +if __name__ == '__main__': + yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' -yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' - -if yearmonth == '': - date = dateutils.add_months(datetime.today(), months=-1) - yearmonth = int(dateutils.format_date(date, '%Y%m')) -try: - if type(yearmonth) is str: - yearmonth = int(yearmonth) - calc(None, yearmonth) -except Exception as e: - print(e) + if yearmonth == '': + date = dateutils.add_months(datetime.today(), months=-1) + yearmonth = int(dateutils.format_date(date, '%Y%m')) + try: + if type(yearmonth) is str: + yearmonth = int(yearmonth) + calc(None, yearmonth) + except Exception as e: + print(e)