diff --git a/dags/business/business/community/extra.py b/dags/business/business/community/extra.py new file mode 100644 index 0000000..2d0be9a --- /dev/null +++ b/dags/business/business/community/extra.py @@ -0,0 +1,138 @@ +""" +附加标签同步至206 + # economic 中心城区 普通城区 郊区 郊外 + # tag 豪华住宅 高档住宅 低档住宅 普通住宅 + # 交通便利: 附近一公里有地铁 + # 靠近学校: 附近一公里有学校 + # 繁华:500米内有商场、麦当劳、肯德基、星巴克 + # 别墅:物业类型包含别墅 + # 县城:区域级别是县城 +""" +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils, areautils, poiutils +import logging +import pandas as pd +from elasticsearch import helpers +from concurrent.futures import ThreadPoolExecutor +from threading import Lock + +log = logging.getLogger(__name__) +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" +path = "/home/airflow/data/community" +m_date = dateutils.iso_format() + +district_analy_data_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_analy_data') + +count = 0 +mutex = Lock() + +def build_filter_info(city, distance, location, bis_type, type_value, secondary_type, source): + filter_info = [{"term": {"city": {"value": city}}}] + if pd.notna(bis_type): + filter_info.append({"term": {"bis_type": {"value": bis_type}}}) + filter_info.append({"terms": {"type": type_value}}) + if pd.notna(secondary_type): + filter_info.append({"term": {"secondary_type": {"value": secondary_type}}}) + filter_info.append({"term": {"source": {"value": source}}}) + filter_info.append({ + "geo_distance": { + "distance": distance, + "location": location + } + }) + return filter_info + + +def find_by_location(city, distance, location, bis_type=None, type_value=None, secondary_type=None, source='ES206|fangjia|amap_poi'): + count = es206_client.elastic_client.count({ + "query": { + "bool": { + "filter": build_filter_info(city, distance, location, bis_type, type_value, secondary_type, source) + } + } + }, 'dws_poi') + return count + + +def save(actions, item): + doc_info = dict() + extra_info = list() + global count + + city = item['city'] + region = item['region'] + estate_type = item['estate_type'] + location = item['location'] + + if pd.notna(item['tag']): + extra_info.append(item['tag']) + + region_display = areautils.get_district_display(city, region) + + if item['subway_station_num'] > 0: + extra_info.append("交通便利") + + if item['kindergarten_num'] > 0 or item['key_primary_school_num'] > 0 or item['ordinary_primary_school_num'] > 0 or item['key_middle_school_num'] > 0 or item['ordinary_middle_school_num'] > 0: + extra_info.append("靠近学校") + + good_count = find_by_location(city, 500, [location['lon'], location['lat']], '生活配套', ['其他', '商场']) + if good_count['count'] > 0: + extra_info.append("繁华") + if pd.notna(item['economic']): + extra_info.append(item['economic']) + if '别墅' in estate_type: + extra_info.append("别墅") + if pd.notna(region_display) and '县' in region_display: + extra_info.append("县城") + + doc_info['extra_info'] = ",".join(extra_info) + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['_id'], + "doc": doc_info + } + mutex.acquire(timeout=60) + actions.append(action) + if len(actions) == 5000: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) + actions.clear() + mutex.release() + + +def sync(city=None): + log.info(f"开始同步附加标签至206 {index_name}") + + sql = "select city, region, name, estate_type, location, subway_station_num, kindergarten_num, key_primary_school_num, ordinary_primary_school_num, key_middle_school_num, ordinary_middle_school_num from community_summary " + if pd.notna(city): + sql += " where city='" + city + "'" + community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + + query_filter = {} + if pd.notna(city): + query_filter = {'city': city} + + district_analy_data_df = pd.DataFrame(list(district_analy_data_conn.find(query_filter, {'_id': 0, 'city': 1, 'region': 1, 'district_name': 1, 'economic': 1, 'tag': 1}))) + district_analy_data_df = district_analy_data_df.rename(columns={'district_name': 'name'}) + + community_summary_df = pd.merge(community_summary_df, district_analy_data_df, how='left', on=['city', 'region', 'name']) + + # with ThreadPoolExecutor(max_workers=30) as pool: + # actions = list() + # + # for index, item in community_summary_df.iterrows(): + # pool.submit(save, actions, item) + # if len(actions) > 0: + # helpers.bulk(es206_client.elastic_client, actions) + # actions.clear() + actions = list() + for index, item in community_summary_df.iterrows(): + save(actions, item) + + log.info(f"完成同步附加标签至206 {index_name}") + + +sync('上海') \ No newline at end of file