新增附加标签
This commit is contained in:
@@ -0,0 +1,138 @@
|
||||
"""
|
||||
附加标签同步至206
|
||||
# economic 中心城区 普通城区 郊区 郊外
|
||||
# tag 豪华住宅 高档住宅 低档住宅 普通住宅
|
||||
# 交通便利: 附近一公里有地铁
|
||||
# 靠近学校: 附近一公里有学校
|
||||
# 繁华:500米内有商场、麦当劳、肯德基、星巴克
|
||||
# 别墅:物业类型包含别墅
|
||||
# 县城:区域级别是县城
|
||||
"""
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils, areautils, poiutils
|
||||
import logging
|
||||
import pandas as pd
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
m_date = dateutils.iso_format()
|
||||
|
||||
district_analy_data_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_analy_data')
|
||||
|
||||
count = 0
|
||||
mutex = Lock()
|
||||
|
||||
def build_filter_info(city, distance, location, bis_type, type_value, secondary_type, source):
|
||||
filter_info = [{"term": {"city": {"value": city}}}]
|
||||
if pd.notna(bis_type):
|
||||
filter_info.append({"term": {"bis_type": {"value": bis_type}}})
|
||||
filter_info.append({"terms": {"type": type_value}})
|
||||
if pd.notna(secondary_type):
|
||||
filter_info.append({"term": {"secondary_type": {"value": secondary_type}}})
|
||||
filter_info.append({"term": {"source": {"value": source}}})
|
||||
filter_info.append({
|
||||
"geo_distance": {
|
||||
"distance": distance,
|
||||
"location": location
|
||||
}
|
||||
})
|
||||
return filter_info
|
||||
|
||||
|
||||
def find_by_location(city, distance, location, bis_type=None, type_value=None, secondary_type=None, source='ES206|fangjia|amap_poi'):
|
||||
count = es206_client.elastic_client.count({
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": build_filter_info(city, distance, location, bis_type, type_value, secondary_type, source)
|
||||
}
|
||||
}
|
||||
}, 'dws_poi')
|
||||
return count
|
||||
|
||||
|
||||
def save(actions, item):
|
||||
doc_info = dict()
|
||||
extra_info = list()
|
||||
global count
|
||||
|
||||
city = item['city']
|
||||
region = item['region']
|
||||
estate_type = item['estate_type']
|
||||
location = item['location']
|
||||
|
||||
if pd.notna(item['tag']):
|
||||
extra_info.append(item['tag'])
|
||||
|
||||
region_display = areautils.get_district_display(city, region)
|
||||
|
||||
if item['subway_station_num'] > 0:
|
||||
extra_info.append("交通便利")
|
||||
|
||||
if item['kindergarten_num'] > 0 or item['key_primary_school_num'] > 0 or item['ordinary_primary_school_num'] > 0 or item['key_middle_school_num'] > 0 or item['ordinary_middle_school_num'] > 0:
|
||||
extra_info.append("靠近学校")
|
||||
|
||||
good_count = find_by_location(city, 500, [location['lon'], location['lat']], '生活配套', ['其他', '商场'])
|
||||
if good_count['count'] > 0:
|
||||
extra_info.append("繁华")
|
||||
if pd.notna(item['economic']):
|
||||
extra_info.append(item['economic'])
|
||||
if '别墅' in estate_type:
|
||||
extra_info.append("别墅")
|
||||
if pd.notna(region_display) and '县' in region_display:
|
||||
extra_info.append("县城")
|
||||
|
||||
doc_info['extra_info'] = ",".join(extra_info)
|
||||
action = {
|
||||
"_op_type": "update",
|
||||
"_index": index_name,
|
||||
"_id": item['_id'],
|
||||
"doc": doc_info
|
||||
}
|
||||
mutex.acquire(timeout=60)
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
log.info("更新:" + str(count))
|
||||
actions.clear()
|
||||
mutex.release()
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步附加标签至206 {index_name}")
|
||||
|
||||
sql = "select city, region, name, estate_type, location, subway_station_num, kindergarten_num, key_primary_school_num, ordinary_primary_school_num, key_middle_school_num, ordinary_middle_school_num from community_summary "
|
||||
if pd.notna(city):
|
||||
sql += " where city='" + city + "'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
query_filter = {}
|
||||
if pd.notna(city):
|
||||
query_filter = {'city': city}
|
||||
|
||||
district_analy_data_df = pd.DataFrame(list(district_analy_data_conn.find(query_filter, {'_id': 0, 'city': 1, 'region': 1, 'district_name': 1, 'economic': 1, 'tag': 1})))
|
||||
district_analy_data_df = district_analy_data_df.rename(columns={'district_name': 'name'})
|
||||
|
||||
community_summary_df = pd.merge(community_summary_df, district_analy_data_df, how='left', on=['city', 'region', 'name'])
|
||||
|
||||
# with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
# actions = list()
|
||||
#
|
||||
# for index, item in community_summary_df.iterrows():
|
||||
# pool.submit(save, actions, item)
|
||||
# if len(actions) > 0:
|
||||
# helpers.bulk(es206_client.elastic_client, actions)
|
||||
# actions.clear()
|
||||
actions = list()
|
||||
for index, item in community_summary_df.iterrows():
|
||||
save(actions, item)
|
||||
|
||||
log.info(f"完成同步附加标签至206 {index_name}")
|
||||
|
||||
|
||||
sync('上海')
|
||||
Reference in New Issue
Block a user