From c2455331a618525bf61843f0293d022605b804fb Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 14 Mar 2023 16:08:52 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=89=A9=E4=B8=9A=E8=B4=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../community/estate_charge_complete.py | 172 +++++++++++++++--- 1 file changed, 146 insertions(+), 26 deletions(-) diff --git a/dags/business/business/community/estate_charge_complete.py b/dags/business/business/community/estate_charge_complete.py index ed03faa..57bce79 100644 --- a/dags/business/business/community/estate_charge_complete.py +++ b/dags/business/business/community/estate_charge_complete.py @@ -13,10 +13,62 @@ log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" - +file_path = "/opt/airflow/dags/repo/dags/business/business/community/data" mutex = Lock() +estate_charge_min_max_df = pd.read_excel(file_path + '/全国各区域人均消费中位值.xlsx') +estate_charge_city_region_dict = dict() -def save(community_summary_df): + +# 加载各区域物业费最大最小值 +def deal_estate_charge(x): + city_now = x['city'] + region_now = x['region'] + if pd.notna(city_now) and pd.notna(region_now): + estate_charge_city_region_dict[city_now + region_now] = {'min': x['3'], 'max': x['97']} + elif pd.notna(city_now): + estate_charge_city_region_dict[city_now] = {'min': x['3'], 'max': x['97']} +estate_charge_min_max_df.apply(lambda x: deal_estate_charge(x), axis=1) + + +def save_fix(community_summary_df): + if 'estate_charge_fix' not in community_summary_df.columns: + log.info("该城市无补充物业费") + return + df = community_summary_df[['_id', 'city', 'estate_charge_fix']] + df = df.dropna(subset=['estate_charge_fix']) + actions = list() + count = 0 + + # 批量更新 + m_date = dateutils.iso_format() + for index, item in df.iterrows(): + row = dict() + _id = str(item["_id"]) + row['estate_charge_fix'] = item['estate_charge_fix'] + row['m_date'] = m_date + + action = {'_op_type': 'update', + '_index': index_name, + '_id': _id, # 请勿忘记ID + 'doc': row, + "_retry_on_conflict": 5 + } + actions.append(action) + if len(actions) == 5000: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("已更新{}".format(count)) + if len(actions) > 0: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("更新完成,共更新: {}".format(count)) + return df + + + +def save_fill(community_summary_df): if 'estate_charge' not in community_summary_df.columns: log.info("该城市无补充物业费") return @@ -108,33 +160,101 @@ def get_estate_charge(finish_building_date, docs): return None -def sync_by_city(city_now): +def adjust_estate_charge(city_now): + log.info(f"{city_now}开始调整物业费....") + sql = f"select id, city, region, name, estate_charge, location from community_summary where city='{city_now}' and estate_charge > 0 and estate_charge_flag != 1" + community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + community_summary_list = list() + community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1) + find_count = 0 + + def deal(community): + global find_count + find_count += 1 + if find_count % 1000 == 0: + log.info("已查找" + str(find_count)) + max_now = 7 + city_short = community['city'] + region_short = community['region'] + if (city_short + region_short) in estate_charge_city_region_dict: + max_now = estate_charge_city_region_dict[city_short + region_short]['max'] + elif city_short in estate_charge_city_region_dict: + max_now = estate_charge_city_region_dict[city_short]['max'] + + if community['estate_charge'] > max_now: + estate_charge = find_estate_charge(community) + # 如果查找的还大于分位值,不调整 + if estate_charge > max_now: + estate_charge = None + if pd.notna(estate_charge): + mutex.acquire(timeout=60) + community['estate_charge_fix'] = estate_charge + mutex.release() + + + with ThreadPoolExecutor(max_workers=30) as pool: + for community_summary in community_summary_list: + pool.submit(deal, community_summary) + + community_summary_df = pd.DataFrame(community_summary_list) + save_fix(community_summary_df) + + log.info(f"{city_now}完成调整物业费....") + + +def find_estate_charge(community): + """ + 查找小区周边物业费 + """ + estate_charge = None + if pd.isna(community['location']) or pd.isna(community['location']['lon']): + return estate_charge + location = [community['location']['lon'], community['location']['lat']] + docs = find_nearest_by_location(community['city'], location) + if len(docs) <= 0 or 'estate_charge' not in docs.columns: + return estate_charge + if pd.notna(community['finish_building_date']) and 'finish_building_date' in docs.columns: + # 根据相近建筑年代赋值 + estate_charge = get_estate_charge(community['finish_building_date'], docs) + if pd.isna(estate_charge): + # 取周边均值 + estate_charge = docs['estate_charge'].mean() + return estate_charge + + +def fill_estate_charge(city_now): log.info(f"{city_now}开始补充物业费....") sql = "select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='" + city_now + "' and (estate_charge is null or estate_charge_flag=1)" community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) - with ThreadPoolExecutor(max_workers=20) as pool: - def deal(index, item): - if index % 1000 == 0 and index > 0: - log.info(f"已查找{index}") - if pd.notna(item['location']) and pd.notna(item['location']['lon']): - location = [item['location']['lon'], item['location']['lat']] - docs = find_nearest_by_location(city_now, location) - estate_charge = None - if len(docs) <= 0 or 'estate_charge' not in docs.columns: - return - if pd.notna(item['finish_building_date']) and 'finish_building_date' in docs.columns: - # 根据相近建筑年代赋值 - estate_charge = get_estate_charge(item['finish_building_date'], docs) - if pd.isna(estate_charge): - # 取周边均值 - estate_charge = docs['estate_charge'].mean() - if pd.notna(estate_charge): - mutex.acquire(timeout=60) - community_summary_df.loc[index, "estate_charge"] = estate_charge - mutex.release() - for index, item in community_summary_df.iterrows(): - pool.submit(deal, index, item) - save(community_summary_df) + + community_summary_list = list() + community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1) + + find_count = 0 + + def deal(community): + global find_count + find_count += 1 + if find_count % 1000 == 0: + log.info("已查找" + str(find_count)) + estate_charge = find_estate_charge(community) + if pd.notna(estate_charge): + mutex.acquire(timeout=60) + community["estate_charge"] = estate_charge + mutex.release() + + with ThreadPoolExecutor(max_workers=30) as pool: + for community_summary in community_summary_list: + pool.submit(deal, community_summary) + + community_summary_df = pd.DataFrame(community_summary_list) + save_fill(community_summary_df) + log.info(f"{city_now}完成补充物业费....") + + +def sync_by_city(city_now): + adjust_estate_charge(city_now) + fill_estate_charge(city_now) def sync(city=None):