优化物业费

This commit is contained in:
shenjian@fangjia.com
2023-03-14 16:08:52 +08:00
parent 3e23948677
commit c2455331a6
@@ -13,10 +13,62 @@ log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
file_path = "/opt/airflow/dags/repo/dags/business/business/community/data"
mutex = Lock()
estate_charge_min_max_df = pd.read_excel(file_path + '/全国各区域人均消费中位值.xlsx')
estate_charge_city_region_dict = dict()
def save(community_summary_df):
# 加载各区域物业费最大最小值
def deal_estate_charge(x):
city_now = x['city']
region_now = x['region']
if pd.notna(city_now) and pd.notna(region_now):
estate_charge_city_region_dict[city_now + region_now] = {'min': x['3'], 'max': x['97']}
elif pd.notna(city_now):
estate_charge_city_region_dict[city_now] = {'min': x['3'], 'max': x['97']}
estate_charge_min_max_df.apply(lambda x: deal_estate_charge(x), axis=1)
def save_fix(community_summary_df):
if 'estate_charge_fix' not in community_summary_df.columns:
log.info("该城市无补充物业费")
return
df = community_summary_df[['_id', 'city', 'estate_charge_fix']]
df = df.dropna(subset=['estate_charge_fix'])
actions = list()
count = 0
# 批量更新
m_date = dateutils.iso_format()
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['estate_charge_fix'] = item['estate_charge_fix']
row['m_date'] = m_date
action = {'_op_type': 'update',
'_index': index_name,
'_id': _id, # 请勿忘记ID
'doc': row,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("已更新{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("更新完成,共更新: {}".format(count))
return df
def save_fill(community_summary_df):
if 'estate_charge' not in community_summary_df.columns:
log.info("该城市无补充物业费")
return
@@ -108,33 +160,101 @@ def get_estate_charge(finish_building_date, docs):
return None
def sync_by_city(city_now):
def adjust_estate_charge(city_now):
log.info(f"{city_now}开始调整物业费....")
sql = f"select id, city, region, name, estate_charge, location from community_summary where city='{city_now}' and estate_charge > 0 and estate_charge_flag != 1"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
community_summary_list = list()
community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1)
find_count = 0
def deal(community):
global find_count
find_count += 1
if find_count % 1000 == 0:
log.info("已查找" + str(find_count))
max_now = 7
city_short = community['city']
region_short = community['region']
if (city_short + region_short) in estate_charge_city_region_dict:
max_now = estate_charge_city_region_dict[city_short + region_short]['max']
elif city_short in estate_charge_city_region_dict:
max_now = estate_charge_city_region_dict[city_short]['max']
if community['estate_charge'] > max_now:
estate_charge = find_estate_charge(community)
# 如果查找的还大于分位值,不调整
if estate_charge > max_now:
estate_charge = None
if pd.notna(estate_charge):
mutex.acquire(timeout=60)
community['estate_charge_fix'] = estate_charge
mutex.release()
with ThreadPoolExecutor(max_workers=30) as pool:
for community_summary in community_summary_list:
pool.submit(deal, community_summary)
community_summary_df = pd.DataFrame(community_summary_list)
save_fix(community_summary_df)
log.info(f"{city_now}完成调整物业费....")
def find_estate_charge(community):
"""
查找小区周边物业费
"""
estate_charge = None
if pd.isna(community['location']) or pd.isna(community['location']['lon']):
return estate_charge
location = [community['location']['lon'], community['location']['lat']]
docs = find_nearest_by_location(community['city'], location)
if len(docs) <= 0 or 'estate_charge' not in docs.columns:
return estate_charge
if pd.notna(community['finish_building_date']) and 'finish_building_date' in docs.columns:
# 根据相近建筑年代赋值
estate_charge = get_estate_charge(community['finish_building_date'], docs)
if pd.isna(estate_charge):
# 取周边均值
estate_charge = docs['estate_charge'].mean()
return estate_charge
def fill_estate_charge(city_now):
log.info(f"{city_now}开始补充物业费....")
sql = "select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='" + city_now + "' and (estate_charge is null or estate_charge_flag=1)"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
with ThreadPoolExecutor(max_workers=20) as pool:
def deal(index, item):
if index % 1000 == 0 and index > 0:
log.info(f"已查找{index}")
if pd.notna(item['location']) and pd.notna(item['location']['lon']):
location = [item['location']['lon'], item['location']['lat']]
docs = find_nearest_by_location(city_now, location)
estate_charge = None
if len(docs) <= 0 or 'estate_charge' not in docs.columns:
return
if pd.notna(item['finish_building_date']) and 'finish_building_date' in docs.columns:
# 根据相近建筑年代赋值
estate_charge = get_estate_charge(item['finish_building_date'], docs)
if pd.isna(estate_charge):
# 取周边均值
estate_charge = docs['estate_charge'].mean()
if pd.notna(estate_charge):
mutex.acquire(timeout=60)
community_summary_df.loc[index, "estate_charge"] = estate_charge
mutex.release()
for index, item in community_summary_df.iterrows():
pool.submit(deal, index, item)
save(community_summary_df)
community_summary_list = list()
community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1)
find_count = 0
def deal(community):
global find_count
find_count += 1
if find_count % 1000 == 0:
log.info("已查找" + str(find_count))
estate_charge = find_estate_charge(community)
if pd.notna(estate_charge):
mutex.acquire(timeout=60)
community["estate_charge"] = estate_charge
mutex.release()
with ThreadPoolExecutor(max_workers=30) as pool:
for community_summary in community_summary_list:
pool.submit(deal, community_summary)
community_summary_df = pd.DataFrame(community_summary_list)
save_fill(community_summary_df)
log.info(f"{city_now}完成补充物业费....")
def sync_by_city(city_now):
adjust_estate_charge(city_now)
fill_estate_charge(city_now)
def sync(city=None):