diff --git a/dags/office/rent/base_price.py b/dags/office/rent/base_price.py index 1af5161..a2f4e6d 100644 --- a/dags/office/rent/base_price.py +++ b/dags/office/rent/base_price.py @@ -35,7 +35,7 @@ def calc(**kwargs): district = conf.get('district', '') office_name = conf.get('name', '') - if pd.isna(office_name): + if office_name == '': calcByCity(city, year_month) else: # 直接重算出最近一年的数据,保证一年有数据 diff --git a/dags/office/sell/fill_base_price.py b/dags/office/sell/fill_base_price.py index 7245f8a..bfd334d 100644 --- a/dags/office/sell/fill_base_price.py +++ b/dags/office/sell/fill_base_price.py @@ -11,18 +11,40 @@ office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat') -def calc(city, year_month): - query = {'date': year_month, 'cat': 'office'} - if pd.notna(city): +def calc(**kwargs): + """ + 写字楼基准价 + """ + # 从 Airflow 的 dag_run.conf 获取参数 + conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {} + city = conf.get('city', '') + year_month = conf.get('year_month', '') + district = conf.get('district', '') + office_name = conf.get('name', '') + + query = {'cat': 'office'} + if year_month != '': + query['date'] = year_month + if city != '': query['city'] = city + if district != '' and office_name != '': + query['district'] = district + query['name'] = office_name office_sell = pd.DataFrame(list(office_sell_stat_conn.find(query))) office_rent = pd.DataFrame(list(office_rent_stat_conn.find(query))) if len(office_rent) <= 0: return + if year_month != '': + calc_by_yearmonth(office_sell, office_rent, year_month) + else: + # 对于单个重算的不进行补充,已经自己补充过了 + pass + +def calc_by_yearmonth(office_sell, office_rent, year_month): office_sell_df = office_sell[['city', 'district', 'name', 'avg_price']] office_sell_df = office_sell_df.rename(columns={'avg_price': 'sell_price'}) - office_rent_df = office_rent[['city', 'district', 'name', 'avg_price', 'estate_type2', 'bed_kind','location']] + office_rent_df = office_rent[['city', 'district', 'name', 'avg_price', 'estate_type2', 'bed_kind', 'location']] office_rent_df = office_rent_df.rename(columns={'avg_price': 'rent_price'}) # 每月动态计算系数,因为不同月份会有差异 merge_df = pd.merge(office_sell_df, office_rent_df, how='left', on=['city', 'district', 'name']) @@ -43,13 +65,13 @@ def calc(city, year_month): 填充存在租金,不存在售价的数据 """ if item['city'] + "_" + item['district'] in district_radio.keys(): - item['sell_price'] = round(district_radio[item['city'] + "_" + item['district']] * item['rent_price'] * 30 * 12, 0) + item['sell_price'] = round( + district_radio[item['city'] + "_" + item['district']] * item['rent_price'] * 30 * 12, 0) elif item['city'] in city_radio.keys(): item['sell_price'] = round(city_radio[item['city']] * item['rent_price'] * 30 * 12, 0) return item if len(merge2_df) == 0: - print(city + "无数据补充") return merge2_df = threads.process_dataframe_multithread(merge2_df, deal, 10) merge2_df = merge2_df[merge2_df['sell_price'].notna()] @@ -83,16 +105,7 @@ def calc(city, year_month): print("已处理{}条数据".format(count)) print("处理完毕") -if __name__ == '__main__': - yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' - - if yearmonth == '': - date = dateutils.add_months(datetime.today(), months=-1) - yearmonth = int(dateutils.format_date(date, '%Y%m')) - try: - if type(yearmonth) is str: - yearmonth = int(yearmonth) - calc(None, yearmonth) - except Exception as e: - print(e) +if __name__ == "__main__": + # 测试脚本:直接运行时模拟 Airflow 的上下文 + calc(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}}))