diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index 818b708..f713d2a 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -48,6 +48,8 @@ def build_doc(item): estate_charge = commutils.format_property_fee(item['estate_charge']) if pd.notna(estate_charge): doc_dict['estate_charge'] = float(estate_charge) + # 表示该物业费已经从库中补充 + doc_dict['estate_charge_flag'] = 0 finish_building_date = commutils.format_finish_building_date(item['finish_building_date']) if pd.notna(finish_building_date): diff --git a/dags/business/business/community/estate_charge_complete.py b/dags/business/business/community/estate_charge_complete.py index 3cdaa90..b07fb58 100644 --- a/dags/business/business/community/estate_charge_complete.py +++ b/dags/business/business/community/estate_charge_complete.py @@ -111,8 +111,8 @@ def find_nearest_by_location(city, location): docs = list() try: scan = es206_client.scan(index_name, { - "size": 3, - "_source": ["finish_building_date", "estate_charge"], + "size": 4, + "_source": ["finish_building_date", "estate_charge", "estate_charge_fix"], "query": { "bool": { "filter": [ @@ -142,7 +142,12 @@ def find_nearest_by_location(city, location): } ] }, preserve_order=True) + ignored_first = False for doc in scan: + # 忽略自己 + if ignored_first is False: + ignored_first = True + continue docs.append(doc['_source']) if len(docs) == 3: return pd.DataFrame(docs) @@ -152,13 +157,22 @@ def find_nearest_by_location(city, location): return None -def get_estate_charge(finish_building_date, docs): +def get_estate_charge_for_adjust(finish_building_date, docs): aim_year = int(finish_building_date / 10) for index, item in docs.iterrows(): if pd.notna(item['finish_building_date']) and pd.notna(item['estate_charge']) and aim_year == int(item['finish_building_date'] / 10): return item['estate_charge'] return None + +def get_estate_charge(finish_building_date, docs): + aim_year = int(finish_building_date / 10) + for index, item in docs.iterrows(): + if pd.notna(item['finish_building_date']) and pd.notna(item['estate_charge_fix']) and aim_year == int(item['finish_building_date'] / 10): + return item['estate_charge_fix'] + return None + + find_count = 0 def deal_adjust(community): global find_count @@ -174,10 +188,7 @@ def deal_adjust(community): max_now = estate_charge_city_region_dict[city_short]['max'] if community['estate_charge'] > max_now: - estate_charge = find_estate_charge(community) - # 如果查找的还大于分位值,不调整 - if estate_charge > max_now: - estate_charge = None + estate_charge = find_estate_charge_for_adjust(community, max_now) if pd.notna(estate_charge): mutex.acquire(timeout=60) community['estate_charge_fix'] = estate_charge @@ -185,7 +196,7 @@ def deal_adjust(community): def adjust_estate_charge(city_now): log.info(f"{city_now}开始调整物业费....") - sql = f"select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='{city_now}' and estate_charge > 0 and estate_charge_flag != 1" + sql = f"select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='{city_now}' and estate_charge_flag != 1" community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) community_summary_list = list() community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1) @@ -193,14 +204,15 @@ def adjust_estate_charge(city_now): with ThreadPoolExecutor(max_workers=30) as pool: for community_summary in community_summary_list: pool.submit(deal_adjust, community_summary) - + # for community_summary in community_summary_list: + # deal_adjust(community_summary) community_summary_df = pd.DataFrame(community_summary_list) save_fix(community_summary_df) log.info(f"{city_now}完成调整物业费....") -def find_estate_charge(community): +def find_estate_charge_for_adjust(community, max_now): """ 查找小区周边物业费 """ @@ -213,10 +225,38 @@ def find_estate_charge(community): return estate_charge if pd.notna(community['finish_building_date']) and 'finish_building_date' in docs.columns: # 根据相近建筑年代赋值 - estate_charge = get_estate_charge(community['finish_building_date'], docs) - if pd.isna(estate_charge): + estate_charge = get_estate_charge_for_adjust(community['finish_building_date'], docs) + if pd.isna(estate_charge) or estate_charge > max_now: # 取周边均值 estate_charge = docs['estate_charge'].mean() + # 如果查找的还大于分位值,不调整 + if estate_charge > max_now: + estate_charge = None + return estate_charge + +def find_estate_charge(community, max_now): + """ + 查找小区周边物业费 + """ + estate_charge = None + if pd.isna(community['location']) or pd.isna(community['location']['lon']): + return estate_charge + location = [community['location']['lon'], community['location']['lat']] + docs = find_nearest_by_location(community['city'], location) + if len(docs) <= 0 or 'estate_charge' not in docs.columns: + return estate_charge + if 'estate_charge_fix' not in docs.columns: + docs['estate_charge_fix'] = None + # 用修正后的物业费补充 + docs['estate_charge_fix'].fillna(docs['estate_charge'], inplace=True) + if pd.notna(community['finish_building_date']) and 'finish_building_date' in docs.columns: + # 根据相近建筑年代赋值 + estate_charge = get_estate_charge(community['finish_building_date'], docs) + if pd.isna(estate_charge) or estate_charge > max_now: + # 取周边均值 + estate_charge = docs['estate_charge_fix'].mean() + if estate_charge > max_now: + estate_charge = None return estate_charge find_count = 0 @@ -225,7 +265,16 @@ def deal_fill(community): find_count += 1 if find_count % 1000 == 0: log.info("已查找" + str(find_count)) - estate_charge = find_estate_charge(community) + + max_now = 7 + city_short = community['city'] + region_short = community['region'] + if (city_short + region_short) in estate_charge_city_region_dict: + max_now = estate_charge_city_region_dict[city_short + region_short]['max'] + elif city_short in estate_charge_city_region_dict: + max_now = estate_charge_city_region_dict[city_short]['max'] + + estate_charge = find_estate_charge(community, max_now) if pd.notna(estate_charge): mutex.acquire(timeout=60) community["estate_charge"] = estate_charge @@ -233,7 +282,7 @@ def deal_fill(community): def fill_estate_charge(city_now): log.info(f"{city_now}开始补充物业费....") - sql = "select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='" + city_now + "' and (estate_charge is null or estate_charge_flag=1)" + sql = "select id, city, region, name, estate_charge, estate_charge_fix, finish_building_date, location from community_summary where city='" + city_now + "' and (estate_charge is null or estate_charge_flag=1)" community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) community_summary_list = list() @@ -242,7 +291,8 @@ def fill_estate_charge(city_now): with ThreadPoolExecutor(max_workers=30) as pool: for community_summary in community_summary_list: pool.submit(deal_fill, community_summary) - + # for community_summary in community_summary_list: + # deal_fill(community_summary) community_summary_df = pd.DataFrame(community_summary_list) save_fill(community_summary_df) log.info(f"{city_now}完成补充物业费....")