恢复加工数据到原始数据

This commit is contained in:
2025-07-03 18:10:11 +08:00
parent 5b8fe6947c
commit 5ef8c21a3b
+13 -8
View File
@@ -280,7 +280,7 @@ def load_comms_summary(city):
source = doc['_source']
community_summary = {"comm_id": source["id"]}
for key in source.keys():
if key.endswith("num") or key.endswith("_distance"):
if (key.endswith("num") or key.endswith("_distance")) and key not in ["half_on_year_sell_num"]:
community_summary[key] = source[key]
community_summary[f"org_{key}"] = source[key]
all_community_summary.append(community_summary)
@@ -452,15 +452,17 @@ def comms_rating(data):
city_code_map = {"深圳": '440300', "临沂" : "371300"}
def load_house_data(city, calc_year_month):
def load_house_stat(city, calc_year_month):
end_year_month = int(dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(calc_year_month), "%Y%m"), months=1), "%Y%m"))
half_on_year = dateutils.add_months(dateutils.parse_date(str(calc_year_month), "%Y%m"), months=-6)
half_on_year_year_month = int(dateutils.format_date(half_on_year, "%Y%m"))
sql = f"""
select gujia_comm_id as comm_id, count(*) as half_on_year from base.bigdata_forsale where urbn_id = '{city_code_map[city]}'
select gujia_comm_id as comm_id, count(*) as half_on_year_sell_num from base.bigdata_forsale where urbn_id = '{city_code_map[city]}'
and data_crt_tm >= to_date('{half_on_year_year_month}', 'YYYYMM') and data_crt_tm < to_date('{end_year_month}', 'YYYYMM')
group by gujia_comm_id ;
"""
house_stat = pd.read_sql(sql, con=engine236)
house_stat['org_half_on_year_sell_num'] = house_stat['half_on_year_sell_num']
return pd.read_sql(sql, con=engine236)
# ## 按照城市加载楼盘评级数据宽表
@@ -468,7 +470,8 @@ def load_data_by_city(city, calc_year_month):
comms = load_comms(city=city)
stat = load_comm_stat(city, calc_year_month)
comms_summary = load_comms_summary(city=city)
house_stat = load_house_data(city, calc_year_month)
house_stat = load_house_stat(city, calc_year_month)
comms_summary = comms_summary.merge(house_stat, on='comm_id', how='left')
data = comms.merge(stat, how="left", on=['city', 'region', 'name'])
data = data.merge(comms_summary, how="left", on=['comm_id'])
num_fields = ["supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num",
@@ -575,9 +578,11 @@ if __name__ == '__main__':
else:
cities = [city.strip() for city in args.city.split(',') if city.strip()]
if args.month == '':
calc_year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), "%Y%m"))
calc_year_months = [int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), "%Y%m"))]
else:
calc_year_month = int(args.month)
args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities]
calc_year_months = [int(calc_year_month.strip()) for calc_year_month in args.month.split(',') if calc_year_month.strip()]
with multiprocessing.Pool(processes=min(8, len(cities))) as pool:
pool.map(process_city, args_list)
for calc_year_month in calc_year_months:
args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities]
pool.map(process_city, args_list)