diff --git a/dags/office/sell/base_price.py b/dags/office/sell/base_price.py index 7939d73..966a0fa 100644 --- a/dags/office/sell/base_price.py +++ b/dags/office/sell/base_price.py @@ -35,6 +35,8 @@ def calc(city, year_month): dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-13), '%Y%m') last_month = dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-1), '%Y%m') sql += "and p_date >= '" + last_year_month + "01' " + if isinstance(year_month, str): + year_month = int(year_month) else: last_year_month = dateutils.format_date(dateutils.add_months(datetime.now(), months=-13), '%Y%m') last_month = dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), '%Y%m') @@ -95,15 +97,16 @@ def calc(city, year_month): if len(actions) == 1000: office_sell_stat_conn.bulk_write(actions) actions.clear() - office_sell_stat_conn.bulk_write(actions) - actions.clear() + if len(actions) > 0: + office_sell_stat_conn.bulk_write(actions) + actions.clear() # 第二步 取周边3公里内同等级、同类型的最近10个有样本写字楼的基准价的平均值 office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates(keep='first') combined = pd.concat([office_info, merge_df]) diff_df = combined.drop_duplicates(subset=['city', 'region', 'name'], keep=False) actions = list() - for index, item in diff_df[(diff_df['estate_type2'].notna()) & (diff_df['bed_kind'].notna())].iterrows(): + for index, item in diff_df[(diff_df['estate_type2'].notna()) & (diff_df['bed_kind'].notna()) & (diff_df['lng'].notna()) ].iterrows(): location = { "type": "Point", "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] @@ -153,8 +156,9 @@ def calc(city, year_month): if len(actions) == 1000: office_sell_stat_conn.bulk_write(actions) actions.clear() - office_sell_stat_conn.bulk_write(actions) - actions.clear() + if len(actions) > 0: + office_sell_stat_conn.bulk_write(actions) + actions.clear() # 系数计算在Jupyter中 bed_kind_dict = {'甲级': 1.111888111888112, '丙级': 0.9143356643356645, '乙级': 1} @@ -256,7 +260,9 @@ def calc(city, year_month): if len(actions) == 1000: office_sell_stat_conn.bulk_write(actions) actions.clear() - + if len(actions) > 0: + office_sell_stat_conn.bulk_write(actions) + actions.clear() # 第五步 query = {'cat': 'office', 'date': year_month, 'calc_type': {'$in': [1, 2, 3]}} if pd.notna(city): @@ -266,37 +272,49 @@ def calc(city, year_month): keep='first') # 计算价格的写字楼 db_df = db_df.rename(columns={'district': 'region'}) - combined = pd.concat([office_info, db_df[['city', 'region', 'name']]]) - diff_df = combined.drop_duplicates(subset=['city', 'region', 'name'], keep=False) + # 使用 merge 操作并结合 indicator 参数 + merged = office_info.merge(db_df[['city', 'region', 'name']], on=['city', 'region', 'name'], how='left', + indicator=True) + + # 过滤出仅在 office_info 中存在的记录 + diff_df = merged[merged['_merge'] == 'left_only'].drop(columns=['_merge']) + + # 如果需要保持原始列顺序,可以重新排列列 + diff_df = diff_df[office_info.columns] + actions.clear() for index, item in diff_df.iterrows(): - location = { - "type": "Point", - "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] - } - city = item['city'] - region = item['region'] - name = item['name'] - estate_type2 = item['estate_type2'] - bed_kind = item['bed_kind'] - ret = office_sell_stat_conn.aggregate([ - { - '$geoNear': { - 'near': location, - 'distanceField': "distance", - 'maxDistance': 3000, - 'spherical': True - } - }, - { - '$match': { - 'date': year_month - } - }, - { - '$limit': 5 + try: + location = { + "type": "Point", + "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] } - ]); + city = item['city'] + region = item['region'] + name = item['name'] + estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None + bed_kind = item['bed_kind'] + ret = office_sell_stat_conn.aggregate([ + { + '$geoNear': { + 'near': location, + 'distanceField': "distance", + 'maxDistance': 3000, + 'spherical': True + } + }, + { + '$match': { + 'date': year_month + } + }, + { + '$limit': 5 + } + ]); + except Exception as e: + print("Error occurred:", str(e)) + print(item) # 提取 avg_price 的值 avg_prices = [doc["avg_price"] for doc in ret] # 计算平均值 @@ -314,8 +332,22 @@ def calc(city, year_month): if len(actions) == 1000: office_sell_stat_conn.bulk_write(actions) actions.clear() - office_sell_stat_conn.bulk_write(actions) + if len(actions) > 0: + office_sell_stat_conn.bulk_write(actions) + actions.clear() yearmonth = sys.argv[1] if len(sys.argv) > 1 else None - -calc('郑州', yearmonth) \ No newline at end of file +# for yearmonth in [ +# "202406", +# "202407", +# "202408", +# "202409", +# "202410", +# "202411", +# "202412", +# "202501" +# ]: +# calc(None, yearmonth) +calc("广州", yearmonth) +calc("武汉", yearmonth) +calc("深圳", yearmonth) \ No newline at end of file diff --git a/dags/train/office/sell/README.md b/dags/train/office/sell/README.md new file mode 100644 index 0000000..8fd6bf0 --- /dev/null +++ b/dags/train/office/sell/README.md @@ -0,0 +1 @@ +样本量太少,直接复用出租计算的结果 \ No newline at end of file