优化物业费
This commit is contained in:
@@ -13,10 +13,62 @@ log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
index_name = "community_summary"
|
||||
|
||||
file_path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||
mutex = Lock()
|
||||
estate_charge_min_max_df = pd.read_excel(file_path + '/全国各区域人均消费中位值.xlsx')
|
||||
estate_charge_city_region_dict = dict()
|
||||
|
||||
def save(community_summary_df):
|
||||
|
||||
# 加载各区域物业费最大最小值
|
||||
def deal_estate_charge(x):
|
||||
city_now = x['city']
|
||||
region_now = x['region']
|
||||
if pd.notna(city_now) and pd.notna(region_now):
|
||||
estate_charge_city_region_dict[city_now + region_now] = {'min': x['3'], 'max': x['97']}
|
||||
elif pd.notna(city_now):
|
||||
estate_charge_city_region_dict[city_now] = {'min': x['3'], 'max': x['97']}
|
||||
estate_charge_min_max_df.apply(lambda x: deal_estate_charge(x), axis=1)
|
||||
|
||||
|
||||
def save_fix(community_summary_df):
|
||||
if 'estate_charge_fix' not in community_summary_df.columns:
|
||||
log.info("该城市无补充物业费")
|
||||
return
|
||||
df = community_summary_df[['_id', 'city', 'estate_charge_fix']]
|
||||
df = df.dropna(subset=['estate_charge_fix'])
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
# 批量更新
|
||||
m_date = dateutils.iso_format()
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
row['estate_charge_fix'] = item['estate_charge_fix']
|
||||
row['m_date'] = m_date
|
||||
|
||||
action = {'_op_type': 'update',
|
||||
'_index': index_name,
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': row,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("已更新{}".format(count))
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("更新完成,共更新: {}".format(count))
|
||||
return df
|
||||
|
||||
|
||||
|
||||
def save_fill(community_summary_df):
|
||||
if 'estate_charge' not in community_summary_df.columns:
|
||||
log.info("该城市无补充物业费")
|
||||
return
|
||||
@@ -108,33 +160,101 @@ def get_estate_charge(finish_building_date, docs):
|
||||
return None
|
||||
|
||||
|
||||
def sync_by_city(city_now):
|
||||
def adjust_estate_charge(city_now):
|
||||
log.info(f"{city_now}开始调整物业费....")
|
||||
sql = f"select id, city, region, name, estate_charge, location from community_summary where city='{city_now}' and estate_charge > 0 and estate_charge_flag != 1"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
community_summary_list = list()
|
||||
community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1)
|
||||
find_count = 0
|
||||
|
||||
def deal(community):
|
||||
global find_count
|
||||
find_count += 1
|
||||
if find_count % 1000 == 0:
|
||||
log.info("已查找" + str(find_count))
|
||||
max_now = 7
|
||||
city_short = community['city']
|
||||
region_short = community['region']
|
||||
if (city_short + region_short) in estate_charge_city_region_dict:
|
||||
max_now = estate_charge_city_region_dict[city_short + region_short]['max']
|
||||
elif city_short in estate_charge_city_region_dict:
|
||||
max_now = estate_charge_city_region_dict[city_short]['max']
|
||||
|
||||
if community['estate_charge'] > max_now:
|
||||
estate_charge = find_estate_charge(community)
|
||||
# 如果查找的还大于分位值,不调整
|
||||
if estate_charge > max_now:
|
||||
estate_charge = None
|
||||
if pd.notna(estate_charge):
|
||||
mutex.acquire(timeout=60)
|
||||
community['estate_charge_fix'] = estate_charge
|
||||
mutex.release()
|
||||
|
||||
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
for community_summary in community_summary_list:
|
||||
pool.submit(deal, community_summary)
|
||||
|
||||
community_summary_df = pd.DataFrame(community_summary_list)
|
||||
save_fix(community_summary_df)
|
||||
|
||||
log.info(f"{city_now}完成调整物业费....")
|
||||
|
||||
|
||||
def find_estate_charge(community):
|
||||
"""
|
||||
查找小区周边物业费
|
||||
"""
|
||||
estate_charge = None
|
||||
if pd.isna(community['location']) or pd.isna(community['location']['lon']):
|
||||
return estate_charge
|
||||
location = [community['location']['lon'], community['location']['lat']]
|
||||
docs = find_nearest_by_location(community['city'], location)
|
||||
if len(docs) <= 0 or 'estate_charge' not in docs.columns:
|
||||
return estate_charge
|
||||
if pd.notna(community['finish_building_date']) and 'finish_building_date' in docs.columns:
|
||||
# 根据相近建筑年代赋值
|
||||
estate_charge = get_estate_charge(community['finish_building_date'], docs)
|
||||
if pd.isna(estate_charge):
|
||||
# 取周边均值
|
||||
estate_charge = docs['estate_charge'].mean()
|
||||
return estate_charge
|
||||
|
||||
|
||||
def fill_estate_charge(city_now):
|
||||
log.info(f"{city_now}开始补充物业费....")
|
||||
sql = "select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='" + city_now + "' and (estate_charge is null or estate_charge_flag=1)"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
with ThreadPoolExecutor(max_workers=20) as pool:
|
||||
def deal(index, item):
|
||||
if index % 1000 == 0 and index > 0:
|
||||
log.info(f"已查找{index}")
|
||||
if pd.notna(item['location']) and pd.notna(item['location']['lon']):
|
||||
location = [item['location']['lon'], item['location']['lat']]
|
||||
docs = find_nearest_by_location(city_now, location)
|
||||
estate_charge = None
|
||||
if len(docs) <= 0 or 'estate_charge' not in docs.columns:
|
||||
return
|
||||
if pd.notna(item['finish_building_date']) and 'finish_building_date' in docs.columns:
|
||||
# 根据相近建筑年代赋值
|
||||
estate_charge = get_estate_charge(item['finish_building_date'], docs)
|
||||
if pd.isna(estate_charge):
|
||||
# 取周边均值
|
||||
estate_charge = docs['estate_charge'].mean()
|
||||
if pd.notna(estate_charge):
|
||||
mutex.acquire(timeout=60)
|
||||
community_summary_df.loc[index, "estate_charge"] = estate_charge
|
||||
mutex.release()
|
||||
for index, item in community_summary_df.iterrows():
|
||||
pool.submit(deal, index, item)
|
||||
save(community_summary_df)
|
||||
|
||||
community_summary_list = list()
|
||||
community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1)
|
||||
|
||||
find_count = 0
|
||||
|
||||
def deal(community):
|
||||
global find_count
|
||||
find_count += 1
|
||||
if find_count % 1000 == 0:
|
||||
log.info("已查找" + str(find_count))
|
||||
estate_charge = find_estate_charge(community)
|
||||
if pd.notna(estate_charge):
|
||||
mutex.acquire(timeout=60)
|
||||
community["estate_charge"] = estate_charge
|
||||
mutex.release()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
for community_summary in community_summary_list:
|
||||
pool.submit(deal, community_summary)
|
||||
|
||||
community_summary_df = pd.DataFrame(community_summary_list)
|
||||
save_fill(community_summary_df)
|
||||
log.info(f"{city_now}完成补充物业费....")
|
||||
|
||||
|
||||
def sync_by_city(city_now):
|
||||
adjust_estate_charge(city_now)
|
||||
fill_estate_charge(city_now)
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
|
||||
Reference in New Issue
Block a user