diff --git a/dags/poi/business/district_bank.py b/dags/poi/business/district_bank.py index 2cc6d9e..0d8b1d9 100644 --- a/dags/poi/business/district_bank.py +++ b/dags/poi/business/district_bank.py @@ -26,14 +26,23 @@ bank_typecode = ["160100", "160101", "160102", "160103", "160104", "160105", "16 "160144", "160145", "160146", "160147", "160148", "160149", "160150", "160151", "160152", "160200"] +small_type = ["银行", "中国人民银行", "国家开发银行", "中国进出口银行", "中国银行", "中国工商银行", "中国建设银行", + "中国农业银行", "交通银行", "招商银行", "华夏银行", "中信银行", "中国民生银行", "中国光大银行", + "上海银行", "上海浦东发展银行", "平安银行", "兴业银行", "北京银行", "广发银行", "农村商业银行", + "香港恒生银行", "东亚银行", "花旗银行", "渣打银行", "汇丰银行", "荷兰银行", "美国运通银行", + "瑞士友邦银行", "美国银行", "蒙特利尔银行", "纽约银行", "苏格兰皇家银行", "法国兴业银行", "德意志银行", + "日本三菱东京日联银行", "巴克莱银行", "摩根大通银行", "中国邮政储蓄银行", "香港星展银行", "南洋商业银行", + "上海商业银行", "永亨银行", "香港永隆银行", "创兴银行", "大新银行", "中信银行(国际)", "大众银行(香港)", + "北京农商银行", "上海农商银行", "广州农商银行", "深圳农村商业银行", "银行相关"] + def deal3(item): location = [item['lng'], item['lat']] - item['bank'] = poiutils.find_around_by_location(item['city_display'], location, bank_typecode, 3000) + item['bank'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 3000) return item def deal(item): location = [item['lng'], item['lat']] - item['bank'] = poiutils.find_around_by_location(item['city_display'], location, bank_typecode, 1000) + item['bank'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 1000) return item def do(): @@ -113,4 +122,4 @@ def sync1(): if len(actions) > 0: district_facilitys_conn.bulk_write(actions) actions.clear() - log.info(city + '周边银行入库完毕') + log.info(city + '周边银行入库完毕') \ No newline at end of file diff --git a/dags/poi/business/district_bus.py b/dags/poi/business/district_bus.py index e5d8fc9..a1a4fd0 100644 --- a/dags/poi/business/district_bus.py +++ b/dags/poi/business/district_bus.py @@ -16,16 +16,17 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', ' # 交通code bus_typecode = ["150700"] +small_type = ["公交车站相关"] def deal3(item): location = [item['lng'], item['lat']] - item['bus'] = poiutils.find_around_by_location(item['city_display'], location, bus_typecode, 3000) + item['bus'] = poiutils.find_around_by_location_for_bus(item['city_display'], location, small_type, 3000) return item def deal(item): location = [item['lng'], item['lat']] - item['bus'] = poiutils.find_around_by_location(item['city_display'], location, bus_typecode, 1000) + item['bus'] = poiutils.find_around_by_location_for_bus(item['city_display'], location, small_type, 1000) return item def do(): diff --git a/dags/poi/business/district_hospital.py b/dags/poi/business/district_hospital.py index a610336..7c19365 100644 --- a/dags/poi/business/district_hospital.py +++ b/dags/poi/business/district_hospital.py @@ -18,16 +18,16 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', ' # 医院code hospital_typecode = ["090100", "090101", "090200"] - +small_type = ["综合医院", "三级甲等医院", "专科医院"] def deal3(item): location = [item['lng'], item['lat']] - item['hospital'] = poiutils.find_around_by_location(item['city_display'], location, hospital_typecode, 3000) + item['hospital'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 3000) return item def deal(item): location = [item['lng'], item['lat']] - item['hospital'] = poiutils.find_around_by_location(item['city_display'], location, hospital_typecode, 1000) + item['hospital'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 1000) return item diff --git a/dags/poi/business/district_school.py b/dags/poi/business/district_school.py index db37394..61996f8 100644 --- a/dags/poi/business/district_school.py +++ b/dags/poi/business/district_school.py @@ -16,16 +16,16 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', ' # 教育code school_typecode = ["141200", "141201", "141202", "141203", "141204"] - +small_type = ["学校", "高等院校", "中学", "小学", "幼儿园"] def deal3(item): location = [item['lng'], item['lat']] - item['school'] = poiutils.find_around_by_location(item['city_display'], location, school_typecode, 3000) + item['school'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 3000) return item def deal(item): location = [item['lng'], item['lat']] - item['school'] = poiutils.find_around_by_location(item['city_display'], location, school_typecode, 1000) + item['school'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 1000) return item diff --git a/dags/poi/business/district_subway.py b/dags/poi/business/district_subway.py index f8d332e..6e2826c 100644 --- a/dags/poi/business/district_subway.py +++ b/dags/poi/business/district_subway.py @@ -9,7 +9,6 @@ import poiutils log = logging.getLogger(__name__) -elastic_client = ElasticSearchClient(host='192.168.0.206') district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') surrounding_facilities_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'surrounding_facilities') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -17,44 +16,52 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', ' # 地铁code subway_typecode = ["150500"] +small_type = ["地铁站"] + def deal3(item): location = [item['lng'], item['lat']] - item['subway'] = poiutils.find_around_by_location(item['city_display'], location, subway_typecode, 3000) + item['subway'] = poiutils.find_around_by_location_for_bus(item['city_display'], location, small_type, 3000) return item + def deal(item): location = [item['lng'], item['lat']] - item['subway'] = poiutils.find_around_by_location(item['city_display'], location, subway_typecode, 1000) + item['subway'] = poiutils.find_around_by_location_for_bus(item['city_display'], location, small_type, 1000) return item + def do(): sync_new() sync1() + def sync_new(distance=3000): city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: city = city_item['city_short'] city_display = city_item['city'] - seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, - {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( - 10000))) + seaweed_df = pd.DataFrame( + list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, + {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( + 10000))) if len(seaweed_df) <= 0: continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 seaweed_df['city_display'] = city_display seaweed_df = process_dataframe_multithread(seaweed_df, deal3, 30) - #seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) + # seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) actions = list() for index, item in seaweed_df.iterrows(): try: # 周边配套库中历史原因,id可能不为小区id action = UpdateOne({'city': item['city'], 'region': item['region'], - 'district_name': item['name']}, {'$set': {'subway': item['subway'], 'city': item['city'], 'region': item['region'], - 'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat'], 'distance': distance}}, upsert=True) + 'district_name': item['name']}, + {'$set': {'subway': item['subway'], 'city': item['city'], 'region': item['region'], + 'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat'], + 'distance': distance}}, upsert=True) actions.append(action) except Exception as e: print(e) @@ -62,33 +69,37 @@ def sync_new(distance=3000): surrounding_facilities_conn.bulk_write(actions) actions.clear() if len(actions) > 0: - surrounding_facilities_conn.bulk_write(actions) - actions.clear() + surrounding_facilities_conn.bulk_write(actions) + actions.clear() log.info(city + '周边地铁入库完毕') + def sync1(): city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: city = city_item['city_short'] city_display = city_item['city'] - seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, - {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( - 10000))) + seaweed_df = pd.DataFrame( + list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, + {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( + 10000))) if len(seaweed_df) <= 0: continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 seaweed_df['city_display'] = city_display seaweed_df = process_dataframe_multithread(seaweed_df, deal, 30) - #seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) + # seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) actions = list() for index, item in seaweed_df.iterrows(): try: # 周边配套库中历史原因,id可能不为小区id action = UpdateOne({'city': item['city'], 'region': item['region'], - 'district_name': item['name']}, {'$set': {'subway': item['subway'], 'city': item['city'], 'region': item['region'], - 'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True) + 'district_name': item['name']}, + {'$set': {'subway': item['subway'], 'city': item['city'], 'region': item['region'], + 'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, + upsert=True) actions.append(action) except Exception as e: print(e) @@ -96,6 +107,8 @@ def sync1(): district_facilitys_conn.bulk_write(actions) actions.clear() if len(actions) > 0: - district_facilitys_conn.bulk_write(actions) - actions.clear() + district_facilitys_conn.bulk_write(actions) + actions.clear() log.info(city + '周边地铁入库完毕') + + diff --git a/dags/poi/business/district_supermarket.py b/dags/poi/business/district_supermarket.py index 17d7001..2e3606e 100644 --- a/dags/poi/business/district_supermarket.py +++ b/dags/poi/business/district_supermarket.py @@ -16,17 +16,17 @@ surrounding_facilities_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', ' # 商场code -supermarket_typecode = ["060100","060101","060102"] - +supermarket_typecode = ["060100", "060101", "060102"] +small_type = ["商场", "购物中心", "普通商场"] def deal3(item): location = [item['lng'], item['lat']] - item['supermarket'] = poiutils.find_around_by_location(item['city_display'], location, supermarket_typecode, 3000) + item['supermarket'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 3000) return item def deal(item): location = [item['lng'], item['lat']] - item['supermarket'] = poiutils.find_around_by_location(item['city_display'], location, supermarket_typecode, 1000) + item['supermarket'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 1000) return item def do(): diff --git a/dags/poi/business/poiutils.py b/dags/poi/business/poiutils.py index a770eed..f18ef96 100644 --- a/dags/poi/business/poiutils.py +++ b/dags/poi/business/poiutils.py @@ -1,8 +1,8 @@ from goojia_utils import ElasticSearchClient -elastic_client = ElasticSearchClient(host='192.168.0.206') +elastic_client = ElasticSearchClient(host='192.168.0.89') -def find_around_by_location(city, location, typecode, distance=1000): +def find_around_by_location_for_bus(city, location, small_type, distance=1000): """ 查找周边poi :param city: @@ -12,7 +12,58 @@ def find_around_by_location(city, location, typecode, distance=1000): """ try: - ret = elastic_client.scan_to_data_frame('amap_poi', { + ret = elastic_client.scan_to_data_frame('gaode_map_poi', { + "_source": ["name"], + "query": { + "bool": { + "must_not": [ + { + "term": { + "main_type": { + "value": 0 # 表示清洗后的数据 + } + } + } + ], + "filter": [ + {"terms": { + "small_type": small_type + }}, + { + "term": { + "city": { + "value": city + } + } + }, + { + "geo_distance": { + "distance": distance, + "location": location + } + } + ] + } + } + }); + if len(ret) > 0: + return ret['name'].tolist() + return [] + except Exception as e: + print(e) + return [] + +def find_around_by_location(city, location, small_type, distance=1000): + """ + 查找周边poi + :param city: + :param location: + :param distance: + :return: + """ + + try: + ret = elastic_client.scan_to_data_frame('gaode_map_poi', { "_source": ["name"], "query": { "bool": { @@ -23,15 +74,22 @@ def find_around_by_location(city, location, typecode, distance=1000): "value": 1 } } + }, + { + "term": { + "main_type": { + "value": 0 # 表示清洗后的数据 + } + } } ], "filter": [ {"terms": { - "typecode": typecode + "small_type": small_type }}, { "term": { - "cityname": { + "city": { "value": city } }