优化物业费入模

This commit is contained in:
shenjian@fangjia.com
2023-03-14 16:14:00 +08:00
parent c2455331a6
commit 6c4385b4b4
2 changed files with 42 additions and 47 deletions
@@ -34,7 +34,7 @@ def calc_consumption_potential(city):
"""
计算消费潜力
"""
sql = "select id, city, region, name, total_price, estate_charge, relative_price_ranking, avg_cost from community_summary where city='" + city + "'"
sql = "select id, city, region, name, total_price, estate_charge, estate_charge_fix, relative_price_ranking, avg_cost from community_summary where city='" + city + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
columns = community_summary_df.columns
if 'total_price' not in columns or 'estate_charge' not in columns or 'relative_price_ranking' not in columns or 'avg_cost' not in columns:
@@ -55,11 +55,10 @@ def calc_consumption_potential(city):
community_summary_df['avg_cost_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform(
community_summary_df[['avg_cost']])
# 结合全国物业费情况,对于7以上的占比1%
community_summary_df.loc[community_summary_df['estate_charge'] > 7, 'estate_charge'] = 7
# 修正的物业费入模
community_summary_df['estate_charge_fix'].fillna(community_summary_df['estate_charge'], inplace=True)
community_summary_df['estate_charge_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform(
community_summary_df[['estate_charge']])
community_summary_df[['estate_charge_fix']])
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge_fixed'].transform(
np.mean)
@@ -15,7 +15,7 @@ es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
file_path = "/opt/airflow/dags/repo/dags/business/business/community/data"
mutex = Lock()
estate_charge_min_max_df = pd.read_excel(file_path + '/全国各区域人均消费中位值.xlsx')
estate_charge_min_max_df = pd.read_excel(file_path + '/全国各区域物业费中位值.xlsx')
estate_charge_city_region_dict = dict()
@@ -32,7 +32,7 @@ estate_charge_min_max_df.apply(lambda x: deal_estate_charge(x), axis=1)
def save_fix(community_summary_df):
if 'estate_charge_fix' not in community_summary_df.columns:
log.info("该城市无补充物业费")
log.info("该城市无调整物业费")
return
df = community_summary_df[['_id', 'city', 'estate_charge_fix']]
df = df.dropna(subset=['estate_charge_fix'])
@@ -159,38 +159,36 @@ def get_estate_charge(finish_building_date, docs):
return item['estate_charge']
return None
find_count = 0
def deal_adjust(community):
global find_count
find_count += 1
if find_count % 1000 == 0:
log.info("已查找" + str(find_count))
max_now = 7
city_short = community['city']
region_short = community['region']
if (city_short + region_short) in estate_charge_city_region_dict:
max_now = estate_charge_city_region_dict[city_short + region_short]['max']
elif city_short in estate_charge_city_region_dict:
max_now = estate_charge_city_region_dict[city_short]['max']
if community['estate_charge'] > max_now:
estate_charge = find_estate_charge(community)
# 如果查找的还大于分位值,不调整
if estate_charge > max_now:
estate_charge = None
if pd.notna(estate_charge):
mutex.acquire(timeout=60)
community['estate_charge_fix'] = estate_charge
mutex.release()
def adjust_estate_charge(city_now):
log.info(f"{city_now}开始调整物业费....")
sql = f"select id, city, region, name, estate_charge, location from community_summary where city='{city_now}' and estate_charge > 0 and estate_charge_flag != 1"
sql = f"select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='{city_now}' and estate_charge > 0 and estate_charge_flag != 1"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
community_summary_list = list()
community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1)
find_count = 0
def deal(community):
global find_count
find_count += 1
if find_count % 1000 == 0:
log.info("已查找" + str(find_count))
max_now = 7
city_short = community['city']
region_short = community['region']
if (city_short + region_short) in estate_charge_city_region_dict:
max_now = estate_charge_city_region_dict[city_short + region_short]['max']
elif city_short in estate_charge_city_region_dict:
max_now = estate_charge_city_region_dict[city_short]['max']
if community['estate_charge'] > max_now:
estate_charge = find_estate_charge(community)
# 如果查找的还大于分位值,不调整
if estate_charge > max_now:
estate_charge = None
if pd.notna(estate_charge):
mutex.acquire(timeout=60)
community['estate_charge_fix'] = estate_charge
mutex.release()
with ThreadPoolExecutor(max_workers=30) as pool:
for community_summary in community_summary_list:
@@ -221,6 +219,17 @@ def find_estate_charge(community):
estate_charge = docs['estate_charge'].mean()
return estate_charge
find_count = 0
def deal_fill(community):
global find_count
find_count += 1
if find_count % 1000 == 0:
log.info("已查找" + str(find_count))
estate_charge = find_estate_charge(community)
if pd.notna(estate_charge):
mutex.acquire(timeout=60)
community["estate_charge"] = estate_charge
mutex.release()
def fill_estate_charge(city_now):
log.info(f"{city_now}开始补充物业费....")
@@ -230,22 +239,9 @@ def fill_estate_charge(city_now):
community_summary_list = list()
community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1)
find_count = 0
def deal(community):
global find_count
find_count += 1
if find_count % 1000 == 0:
log.info("已查找" + str(find_count))
estate_charge = find_estate_charge(community)
if pd.notna(estate_charge):
mutex.acquire(timeout=60)
community["estate_charge"] = estate_charge
mutex.release()
with ThreadPoolExecutor(max_workers=30) as pool:
for community_summary in community_summary_list:
pool.submit(deal, community_summary)
pool.submit(deal_fill, community_summary)
community_summary_df = pd.DataFrame(community_summary_list)
save_fill(community_summary_df)