恢复加工数据到原始数据
This commit is contained in:
@@ -280,7 +280,7 @@ def load_comms_summary(city):
|
||||
source = doc['_source']
|
||||
community_summary = {"comm_id": source["id"]}
|
||||
for key in source.keys():
|
||||
if key.endswith("num") or key.endswith("_distance"):
|
||||
if (key.endswith("num") or key.endswith("_distance")) and key not in ["half_on_year_sell_num"]:
|
||||
community_summary[key] = source[key]
|
||||
community_summary[f"org_{key}"] = source[key]
|
||||
all_community_summary.append(community_summary)
|
||||
@@ -452,15 +452,17 @@ def comms_rating(data):
|
||||
|
||||
city_code_map = {"深圳": '440300', "临沂" : "371300"}
|
||||
|
||||
def load_house_data(city, calc_year_month):
|
||||
def load_house_stat(city, calc_year_month):
|
||||
end_year_month = int(dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(calc_year_month), "%Y%m"), months=1), "%Y%m"))
|
||||
half_on_year = dateutils.add_months(dateutils.parse_date(str(calc_year_month), "%Y%m"), months=-6)
|
||||
half_on_year_year_month = int(dateutils.format_date(half_on_year, "%Y%m"))
|
||||
sql = f"""
|
||||
select gujia_comm_id as comm_id, count(*) as half_on_year from base.bigdata_forsale where urbn_id = '{city_code_map[city]}'
|
||||
select gujia_comm_id as comm_id, count(*) as half_on_year_sell_num from base.bigdata_forsale where urbn_id = '{city_code_map[city]}'
|
||||
and data_crt_tm >= to_date('{half_on_year_year_month}', 'YYYYMM') and data_crt_tm < to_date('{end_year_month}', 'YYYYMM')
|
||||
group by gujia_comm_id ;
|
||||
"""
|
||||
house_stat = pd.read_sql(sql, con=engine236)
|
||||
house_stat['org_half_on_year_sell_num'] = house_stat['half_on_year_sell_num']
|
||||
return pd.read_sql(sql, con=engine236)
|
||||
# ## 按照城市加载楼盘评级数据宽表
|
||||
|
||||
@@ -468,7 +470,8 @@ def load_data_by_city(city, calc_year_month):
|
||||
comms = load_comms(city=city)
|
||||
stat = load_comm_stat(city, calc_year_month)
|
||||
comms_summary = load_comms_summary(city=city)
|
||||
house_stat = load_house_data(city, calc_year_month)
|
||||
house_stat = load_house_stat(city, calc_year_month)
|
||||
comms_summary = comms_summary.merge(house_stat, on='comm_id', how='left')
|
||||
data = comms.merge(stat, how="left", on=['city', 'region', 'name'])
|
||||
data = data.merge(comms_summary, how="left", on=['comm_id'])
|
||||
num_fields = ["supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num",
|
||||
@@ -575,9 +578,11 @@ if __name__ == '__main__':
|
||||
else:
|
||||
cities = [city.strip() for city in args.city.split(',') if city.strip()]
|
||||
if args.month == '':
|
||||
calc_year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), "%Y%m"))
|
||||
calc_year_months = [int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), "%Y%m"))]
|
||||
else:
|
||||
calc_year_month = int(args.month)
|
||||
args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities]
|
||||
calc_year_months = [int(calc_year_month.strip()) for calc_year_month in args.month.split(',') if calc_year_month.strip()]
|
||||
|
||||
with multiprocessing.Pool(processes=min(8, len(cities))) as pool:
|
||||
pool.map(process_city, args_list)
|
||||
for calc_year_month in calc_year_months:
|
||||
args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities]
|
||||
pool.map(process_city, args_list)
|
||||
Reference in New Issue
Block a user