新增物业费补充
This commit is contained in:
@@ -0,0 +1,131 @@
|
||||
"""
|
||||
物业费补全
|
||||
"""
|
||||
from goojia_utils import ElasticSearchClient, areautils, dateutils
|
||||
import pandas as pd
|
||||
import logging
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
index_name = "community_summary"
|
||||
|
||||
|
||||
def save(community_summary_df):
|
||||
df = community_summary_df[['_id', 'city', 'estate_charge']]
|
||||
df = df.dropna(subset=['estate_charge'])
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
# 批量更新
|
||||
m_date = dateutils.iso_format()
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
row['estate_charge'] = item['estate_charge']
|
||||
# 1 表示为物业费补充的,在实际应用中不输出
|
||||
row['estate_charge_flag'] = 1
|
||||
row['m_date'] = m_date
|
||||
|
||||
action = {'_op_type': 'update',
|
||||
'_index': index_name,
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': row,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("已更新{}".format(count))
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("更新完成,共更新: {}".format(count))
|
||||
return df
|
||||
|
||||
|
||||
def find_nearest_by_location(city, location):
|
||||
docs = list()
|
||||
try:
|
||||
scan = es206_client.scan(index_name, {
|
||||
"size": 3,
|
||||
"_source": ["finish_building_date", "estate_charge"],
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{
|
||||
"term": {
|
||||
"city": city
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"sort": [
|
||||
{
|
||||
"_geo_distance": {
|
||||
"location": location,
|
||||
"order": "asc"
|
||||
}
|
||||
}
|
||||
]
|
||||
}, preserve_order=True)
|
||||
for doc in scan:
|
||||
docs.append(doc['_source'])
|
||||
if len(docs) == 3:
|
||||
return pd.DataFrame(docs)
|
||||
return None
|
||||
except Exception as e:
|
||||
logging.error(e)
|
||||
return None
|
||||
|
||||
|
||||
def get_estate_charge(finish_building_date, docs):
|
||||
aim_year = int(finish_building_date / 10)
|
||||
for index, item in docs.iterrows():
|
||||
if pd.notna(item['finish_building_date']) and pd.notna(item['estate_charge']) and aim_year == int(item['finish_building_date'] / 10):
|
||||
return item['estate_charge']
|
||||
return None
|
||||
|
||||
|
||||
def sync_by_city(city_now):
|
||||
log.info(f"{city_now}开始补充物业费....")
|
||||
sql = "select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='" + city_now + "' and (estate_charge is null or estate_charge_flag=1)"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
with ThreadPoolExecutor(max_workers=50) as pool:
|
||||
def deal(index, item):
|
||||
if index % 1000 == 0:
|
||||
log.info(f"已补充{index}")
|
||||
if pd.notna(item['location']) and pd.notna(item['location']['lon']):
|
||||
location = [item['location']['lon'], item['location']['lat']]
|
||||
docs = find_nearest_by_location(city_now, location)
|
||||
estate_charge = None
|
||||
if len(docs) <= 0 or 'estate_charge' not in docs.columns:
|
||||
return
|
||||
if pd.notna(item['finish_building_date']) and 'finish_building_date' in docs.columns:
|
||||
# 根据相近建筑年代赋值
|
||||
estate_charge = get_estate_charge(item['finish_building_date'], docs)
|
||||
if pd.isna(estate_charge):
|
||||
# 取周边均值
|
||||
estate_charge = docs['estate_charge'].mean()
|
||||
if pd.notna(estate_charge):
|
||||
community_summary_df.loc[index, "estate_charge"] = estate_charge
|
||||
for index, item in community_summary_df.iterrows():
|
||||
pool.submit(deal, index, item)
|
||||
save(community_summary_df)
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
if pd.isna(city):
|
||||
for city_now in areautils.get_all_cities():
|
||||
sync_by_city(city_now)
|
||||
else:
|
||||
sync_by_city(city)
|
||||
|
||||
|
||||
sync('上海')
|
||||
Reference in New Issue
Block a user