新增物业费补充

This commit is contained in:
shenjian@fangjia.com
2022-11-28 13:33:40 +08:00
parent 4a849da890
commit 9f8ed155b5
@@ -0,0 +1,131 @@
"""
物业费补全
"""
from goojia_utils import ElasticSearchClient, areautils, dateutils
import pandas as pd
import logging
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
def save(community_summary_df):
df = community_summary_df[['_id', 'city', 'estate_charge']]
df = df.dropna(subset=['estate_charge'])
actions = list()
count = 0
# 批量更新
m_date = dateutils.iso_format()
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['estate_charge'] = item['estate_charge']
# 1 表示为物业费补充的,在实际应用中不输出
row['estate_charge_flag'] = 1
row['m_date'] = m_date
action = {'_op_type': 'update',
'_index': index_name,
'_id': _id, # 请勿忘记ID
'doc': row,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("已更新{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("更新完成,共更新: {}".format(count))
return df
def find_nearest_by_location(city, location):
docs = list()
try:
scan = es206_client.scan(index_name, {
"size": 3,
"_source": ["finish_building_date", "estate_charge"],
"query": {
"bool": {
"filter": [
{
"term": {
"city": city
}
}
]
}
},
"sort": [
{
"_geo_distance": {
"location": location,
"order": "asc"
}
}
]
}, preserve_order=True)
for doc in scan:
docs.append(doc['_source'])
if len(docs) == 3:
return pd.DataFrame(docs)
return None
except Exception as e:
logging.error(e)
return None
def get_estate_charge(finish_building_date, docs):
aim_year = int(finish_building_date / 10)
for index, item in docs.iterrows():
if pd.notna(item['finish_building_date']) and pd.notna(item['estate_charge']) and aim_year == int(item['finish_building_date'] / 10):
return item['estate_charge']
return None
def sync_by_city(city_now):
log.info(f"{city_now}开始补充物业费....")
sql = "select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='" + city_now + "' and (estate_charge is null or estate_charge_flag=1)"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
with ThreadPoolExecutor(max_workers=50) as pool:
def deal(index, item):
if index % 1000 == 0:
log.info(f"已补充{index}")
if pd.notna(item['location']) and pd.notna(item['location']['lon']):
location = [item['location']['lon'], item['location']['lat']]
docs = find_nearest_by_location(city_now, location)
estate_charge = None
if len(docs) <= 0 or 'estate_charge' not in docs.columns:
return
if pd.notna(item['finish_building_date']) and 'finish_building_date' in docs.columns:
# 根据相近建筑年代赋值
estate_charge = get_estate_charge(item['finish_building_date'], docs)
if pd.isna(estate_charge):
# 取周边均值
estate_charge = docs['estate_charge'].mean()
if pd.notna(estate_charge):
community_summary_df.loc[index, "estate_charge"] = estate_charge
for index, item in community_summary_df.iterrows():
pool.submit(deal, index, item)
save(community_summary_df)
def sync(city=None):
if pd.isna(city):
for city_now in areautils.get_all_cities():
sync_by_city(city_now)
else:
sync_by_city(city)
sync('上海')