新增附加标签

This commit is contained in:
shenjian@fangjia.com
2022-11-04 11:20:17 +08:00
parent 356949e6cc
commit 3293cf10f1
+138
View File
@@ -0,0 +1,138 @@
"""
附加标签同步至206
# economic 中心城区 普通城区 郊区 郊外
# tag 豪华住宅 高档住宅 低档住宅 普通住宅
# 交通便利: 附近一公里有地铁
# 靠近学校: 附近一公里有学校
# 繁华:500米内有商场、麦当劳、肯德基、星巴克
# 别墅:物业类型包含别墅
# 县城:区域级别是县城
"""
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils, areautils, poiutils
import logging
import pandas as pd
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
district_analy_data_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_analy_data')
count = 0
mutex = Lock()
def build_filter_info(city, distance, location, bis_type, type_value, secondary_type, source):
filter_info = [{"term": {"city": {"value": city}}}]
if pd.notna(bis_type):
filter_info.append({"term": {"bis_type": {"value": bis_type}}})
filter_info.append({"terms": {"type": type_value}})
if pd.notna(secondary_type):
filter_info.append({"term": {"secondary_type": {"value": secondary_type}}})
filter_info.append({"term": {"source": {"value": source}}})
filter_info.append({
"geo_distance": {
"distance": distance,
"location": location
}
})
return filter_info
def find_by_location(city, distance, location, bis_type=None, type_value=None, secondary_type=None, source='ES206|fangjia|amap_poi'):
count = es206_client.elastic_client.count({
"query": {
"bool": {
"filter": build_filter_info(city, distance, location, bis_type, type_value, secondary_type, source)
}
}
}, 'dws_poi')
return count
def save(actions, item):
doc_info = dict()
extra_info = list()
global count
city = item['city']
region = item['region']
estate_type = item['estate_type']
location = item['location']
if pd.notna(item['tag']):
extra_info.append(item['tag'])
region_display = areautils.get_district_display(city, region)
if item['subway_station_num'] > 0:
extra_info.append("交通便利")
if item['kindergarten_num'] > 0 or item['key_primary_school_num'] > 0 or item['ordinary_primary_school_num'] > 0 or item['key_middle_school_num'] > 0 or item['ordinary_middle_school_num'] > 0:
extra_info.append("靠近学校")
good_count = find_by_location(city, 500, [location['lon'], location['lat']], '生活配套', ['其他', '商场'])
if good_count['count'] > 0:
extra_info.append("繁华")
if pd.notna(item['economic']):
extra_info.append(item['economic'])
if '别墅' in estate_type:
extra_info.append("别墅")
if pd.notna(region_display) and '县' in region_display:
extra_info.append("县城")
doc_info['extra_info'] = ",".join(extra_info)
action = {
"_op_type": "update",
"_index": index_name,
"_id": item['_id'],
"doc": doc_info
}
mutex.acquire(timeout=60)
actions.append(action)
if len(actions) == 5000:
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
log.info("更新:" + str(count))
actions.clear()
mutex.release()
def sync(city=None):
log.info(f"开始同步附加标签至206 {index_name}")
sql = "select city, region, name, estate_type, location, subway_station_num, kindergarten_num, key_primary_school_num, ordinary_primary_school_num, key_middle_school_num, ordinary_middle_school_num from community_summary "
if pd.notna(city):
sql += " where city='" + city + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
query_filter = {}
if pd.notna(city):
query_filter = {'city': city}
district_analy_data_df = pd.DataFrame(list(district_analy_data_conn.find(query_filter, {'_id': 0, 'city': 1, 'region': 1, 'district_name': 1, 'economic': 1, 'tag': 1})))
district_analy_data_df = district_analy_data_df.rename(columns={'district_name': 'name'})
community_summary_df = pd.merge(community_summary_df, district_analy_data_df, how='left', on=['city', 'region', 'name'])
# with ThreadPoolExecutor(max_workers=30) as pool:
# actions = list()
#
# for index, item in community_summary_df.iterrows():
# pool.submit(save, actions, item)
# if len(actions) > 0:
# helpers.bulk(es206_client.elastic_client, actions)
# actions.clear()
actions = list()
for index, item in community_summary_df.iterrows():
save(actions, item)
log.info(f"完成同步附加标签至206 {index_name}")
sync('上海')