diff --git a/dags/stat/community/community_rate.py b/dags/stat/community/community_rate.py index a455e14..a2d5f28 100644 --- a/dags/stat/community/community_rate.py +++ b/dags/stat/community/community_rate.py @@ -280,7 +280,7 @@ def load_comms_summary(city): source = doc['_source'] community_summary = {"comm_id": source["id"]} for key in source.keys(): - if key.endswith("num") or key.endswith("_distance"): + if (key.endswith("num") or key.endswith("_distance")) and key not in ["half_on_year_sell_num"]: community_summary[key] = source[key] community_summary[f"org_{key}"] = source[key] all_community_summary.append(community_summary) @@ -452,15 +452,17 @@ def comms_rating(data): city_code_map = {"深圳": '440300', "临沂" : "371300"} -def load_house_data(city, calc_year_month): +def load_house_stat(city, calc_year_month): end_year_month = int(dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(calc_year_month), "%Y%m"), months=1), "%Y%m")) half_on_year = dateutils.add_months(dateutils.parse_date(str(calc_year_month), "%Y%m"), months=-6) half_on_year_year_month = int(dateutils.format_date(half_on_year, "%Y%m")) sql = f""" - select gujia_comm_id as comm_id, count(*) as half_on_year from base.bigdata_forsale where urbn_id = '{city_code_map[city]}' + select gujia_comm_id as comm_id, count(*) as half_on_year_sell_num from base.bigdata_forsale where urbn_id = '{city_code_map[city]}' and data_crt_tm >= to_date('{half_on_year_year_month}', 'YYYYMM') and data_crt_tm < to_date('{end_year_month}', 'YYYYMM') group by gujia_comm_id ; """ + house_stat = pd.read_sql(sql, con=engine236) + house_stat['org_half_on_year_sell_num'] = house_stat['half_on_year_sell_num'] return pd.read_sql(sql, con=engine236) # ## 按照城市加载楼盘评级数据宽表 @@ -468,7 +470,8 @@ def load_data_by_city(city, calc_year_month): comms = load_comms(city=city) stat = load_comm_stat(city, calc_year_month) comms_summary = load_comms_summary(city=city) - house_stat = load_house_data(city, calc_year_month) + house_stat = load_house_stat(city, calc_year_month) + comms_summary = comms_summary.merge(house_stat, on='comm_id', how='left') data = comms.merge(stat, how="left", on=['city', 'region', 'name']) data = data.merge(comms_summary, how="left", on=['comm_id']) num_fields = ["supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num", @@ -575,9 +578,11 @@ if __name__ == '__main__': else: cities = [city.strip() for city in args.city.split(',') if city.strip()] if args.month == '': - calc_year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), "%Y%m")) + calc_year_months = [int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), "%Y%m"))] else: - calc_year_month = int(args.month) - args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities] + calc_year_months = [int(calc_year_month.strip()) for calc_year_month in args.month.split(',') if calc_year_month.strip()] + with multiprocessing.Pool(processes=min(8, len(cities))) as pool: - pool.map(process_city, args_list) \ No newline at end of file + for calc_year_month in calc_year_months: + args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities] + pool.map(process_city, args_list) \ No newline at end of file