From 6c4385b4b44e6daea6245e470048169833f9eddb Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 14 Mar 2023 16:14:00 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=89=A9=E4=B8=9A=E8=B4=B9?= =?UTF-8?q?=E5=85=A5=E6=A8=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/community/consumption.py | 9 +-- .../community/estate_charge_complete.py | 80 +++++++++---------- 2 files changed, 42 insertions(+), 47 deletions(-) diff --git a/dags/business/business/community/consumption.py b/dags/business/business/community/consumption.py index 1d186c9..e811ac9 100644 --- a/dags/business/business/community/consumption.py +++ b/dags/business/business/community/consumption.py @@ -34,7 +34,7 @@ def calc_consumption_potential(city): """ 计算消费潜力 """ - sql = "select id, city, region, name, total_price, estate_charge, relative_price_ranking, avg_cost from community_summary where city='" + city + "'" + sql = "select id, city, region, name, total_price, estate_charge, estate_charge_fix, relative_price_ranking, avg_cost from community_summary where city='" + city + "'" community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) columns = community_summary_df.columns if 'total_price' not in columns or 'estate_charge' not in columns or 'relative_price_ranking' not in columns or 'avg_cost' not in columns: @@ -55,11 +55,10 @@ def calc_consumption_potential(city): community_summary_df['avg_cost_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform( community_summary_df[['avg_cost']]) - # 结合全国物业费情况,对于7以上的占比1% - community_summary_df.loc[community_summary_df['estate_charge'] > 7, 'estate_charge'] = 7 - + # 修正的物业费入模 + community_summary_df['estate_charge_fix'].fillna(community_summary_df['estate_charge'], inplace=True) community_summary_df['estate_charge_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform( - community_summary_df[['estate_charge']]) + community_summary_df[['estate_charge_fix']]) community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge_fixed'].transform( np.mean) diff --git a/dags/business/business/community/estate_charge_complete.py b/dags/business/business/community/estate_charge_complete.py index 57bce79..4e527c3 100644 --- a/dags/business/business/community/estate_charge_complete.py +++ b/dags/business/business/community/estate_charge_complete.py @@ -15,7 +15,7 @@ es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" file_path = "/opt/airflow/dags/repo/dags/business/business/community/data" mutex = Lock() -estate_charge_min_max_df = pd.read_excel(file_path + '/全国各区域人均消费中位值.xlsx') +estate_charge_min_max_df = pd.read_excel(file_path + '/全国各区域物业费中位值.xlsx') estate_charge_city_region_dict = dict() @@ -32,7 +32,7 @@ estate_charge_min_max_df.apply(lambda x: deal_estate_charge(x), axis=1) def save_fix(community_summary_df): if 'estate_charge_fix' not in community_summary_df.columns: - log.info("该城市无补充物业费") + log.info("该城市无调整物业费") return df = community_summary_df[['_id', 'city', 'estate_charge_fix']] df = df.dropna(subset=['estate_charge_fix']) @@ -159,38 +159,36 @@ def get_estate_charge(finish_building_date, docs): return item['estate_charge'] return None +find_count = 0 +def deal_adjust(community): + global find_count + find_count += 1 + if find_count % 1000 == 0: + log.info("已查找" + str(find_count)) + max_now = 7 + city_short = community['city'] + region_short = community['region'] + if (city_short + region_short) in estate_charge_city_region_dict: + max_now = estate_charge_city_region_dict[city_short + region_short]['max'] + elif city_short in estate_charge_city_region_dict: + max_now = estate_charge_city_region_dict[city_short]['max'] + + if community['estate_charge'] > max_now: + estate_charge = find_estate_charge(community) + # 如果查找的还大于分位值,不调整 + if estate_charge > max_now: + estate_charge = None + if pd.notna(estate_charge): + mutex.acquire(timeout=60) + community['estate_charge_fix'] = estate_charge + mutex.release() def adjust_estate_charge(city_now): log.info(f"{city_now}开始调整物业费....") - sql = f"select id, city, region, name, estate_charge, location from community_summary where city='{city_now}' and estate_charge > 0 and estate_charge_flag != 1" + sql = f"select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='{city_now}' and estate_charge > 0 and estate_charge_flag != 1" community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) community_summary_list = list() community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1) - find_count = 0 - - def deal(community): - global find_count - find_count += 1 - if find_count % 1000 == 0: - log.info("已查找" + str(find_count)) - max_now = 7 - city_short = community['city'] - region_short = community['region'] - if (city_short + region_short) in estate_charge_city_region_dict: - max_now = estate_charge_city_region_dict[city_short + region_short]['max'] - elif city_short in estate_charge_city_region_dict: - max_now = estate_charge_city_region_dict[city_short]['max'] - - if community['estate_charge'] > max_now: - estate_charge = find_estate_charge(community) - # 如果查找的还大于分位值,不调整 - if estate_charge > max_now: - estate_charge = None - if pd.notna(estate_charge): - mutex.acquire(timeout=60) - community['estate_charge_fix'] = estate_charge - mutex.release() - with ThreadPoolExecutor(max_workers=30) as pool: for community_summary in community_summary_list: @@ -221,6 +219,17 @@ def find_estate_charge(community): estate_charge = docs['estate_charge'].mean() return estate_charge +find_count = 0 +def deal_fill(community): + global find_count + find_count += 1 + if find_count % 1000 == 0: + log.info("已查找" + str(find_count)) + estate_charge = find_estate_charge(community) + if pd.notna(estate_charge): + mutex.acquire(timeout=60) + community["estate_charge"] = estate_charge + mutex.release() def fill_estate_charge(city_now): log.info(f"{city_now}开始补充物业费....") @@ -230,22 +239,9 @@ def fill_estate_charge(city_now): community_summary_list = list() community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1) - find_count = 0 - - def deal(community): - global find_count - find_count += 1 - if find_count % 1000 == 0: - log.info("已查找" + str(find_count)) - estate_charge = find_estate_charge(community) - if pd.notna(estate_charge): - mutex.acquire(timeout=60) - community["estate_charge"] = estate_charge - mutex.release() - with ThreadPoolExecutor(max_workers=30) as pool: for community_summary in community_summary_list: - pool.submit(deal, community_summary) + pool.submit(deal_fill, community_summary) community_summary_df = pd.DataFrame(community_summary_list) save_fill(community_summary_df)