From 9f8ed155b53f3779dd65d79b6ea03de3e83adb00 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 28 Nov 2022 13:33:40 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=89=A9=E4=B8=9A=E8=B4=B9?= =?UTF-8?q?=E8=A1=A5=E5=85=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../community/estate_charge_complete.py | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 dags/business/business/community/estate_charge_complete.py diff --git a/dags/business/business/community/estate_charge_complete.py b/dags/business/business/community/estate_charge_complete.py new file mode 100644 index 0000000..18ef988 --- /dev/null +++ b/dags/business/business/community/estate_charge_complete.py @@ -0,0 +1,131 @@ +""" +物业费补全 +""" +from goojia_utils import ElasticSearchClient, areautils, dateutils +import pandas as pd +import logging +from elasticsearch import helpers +from concurrent.futures import ThreadPoolExecutor + +log = logging.getLogger(__name__) + +es206_client = ElasticSearchClient(host='192.168.0.206') +index_name = "community_summary" + + +def save(community_summary_df): + df = community_summary_df[['_id', 'city', 'estate_charge']] + df = df.dropna(subset=['estate_charge']) + actions = list() + count = 0 + + # 批量更新 + m_date = dateutils.iso_format() + for index, item in df.iterrows(): + row = dict() + _id = str(item["_id"]) + row['estate_charge'] = item['estate_charge'] + # 1 表示为物业费补充的,在实际应用中不输出 + row['estate_charge_flag'] = 1 + row['m_date'] = m_date + + action = {'_op_type': 'update', + '_index': index_name, + '_id': _id, # 请勿忘记ID + 'doc': row, + "_retry_on_conflict": 5 + } + actions.append(action) + if len(actions) == 5000: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("已更新{}".format(count)) + if len(actions) > 0: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("更新完成,共更新: {}".format(count)) + return df + + +def find_nearest_by_location(city, location): + docs = list() + try: + scan = es206_client.scan(index_name, { + "size": 3, + "_source": ["finish_building_date", "estate_charge"], + "query": { + "bool": { + "filter": [ + { + "term": { + "city": city + } + } + ] + } + }, + "sort": [ + { + "_geo_distance": { + "location": location, + "order": "asc" + } + } + ] + }, preserve_order=True) + for doc in scan: + docs.append(doc['_source']) + if len(docs) == 3: + return pd.DataFrame(docs) + return None + except Exception as e: + logging.error(e) + return None + + +def get_estate_charge(finish_building_date, docs): + aim_year = int(finish_building_date / 10) + for index, item in docs.iterrows(): + if pd.notna(item['finish_building_date']) and pd.notna(item['estate_charge']) and aim_year == int(item['finish_building_date'] / 10): + return item['estate_charge'] + return None + + +def sync_by_city(city_now): + log.info(f"{city_now}开始补充物业费....") + sql = "select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='" + city_now + "' and (estate_charge is null or estate_charge_flag=1)" + community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + with ThreadPoolExecutor(max_workers=50) as pool: + def deal(index, item): + if index % 1000 == 0: + log.info(f"已补充{index}") + if pd.notna(item['location']) and pd.notna(item['location']['lon']): + location = [item['location']['lon'], item['location']['lat']] + docs = find_nearest_by_location(city_now, location) + estate_charge = None + if len(docs) <= 0 or 'estate_charge' not in docs.columns: + return + if pd.notna(item['finish_building_date']) and 'finish_building_date' in docs.columns: + # 根据相近建筑年代赋值 + estate_charge = get_estate_charge(item['finish_building_date'], docs) + if pd.isna(estate_charge): + # 取周边均值 + estate_charge = docs['estate_charge'].mean() + if pd.notna(estate_charge): + community_summary_df.loc[index, "estate_charge"] = estate_charge + for index, item in community_summary_df.iterrows(): + pool.submit(deal, index, item) + save(community_summary_df) + + +def sync(city=None): + if pd.isna(city): + for city_now in areautils.get_all_cities(): + sync_by_city(city_now) + else: + sync_by_city(city) + + +sync('上海') \ No newline at end of file