From 1de9f959e4a354f55d2dfe26f4ae92bfbe0622ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 2 Aug 2024 16:14:31 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9A=82=E6=97=B6=E5=86=99=E6=AD=BB=E9=BB=91?= =?UTF-8?q?=E9=BE=99=E6=B1=9F=E7=9C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/poi/business/district_bank.py | 10 +++++----- dags/poi/business/district_bus.py | 10 +++++----- dags/poi/business/district_hospital.py | 10 +++++----- dags/poi/business/district_school.py | 11 +++++------ dags/poi/business/district_subway.py | 11 +++++------ dags/poi/business/district_supermarket.py | 11 +++++------ dags/poi/dag_district_facilities.py | 2 +- 7 files changed, 31 insertions(+), 34 deletions(-) diff --git a/dags/poi/business/district_bank.py b/dags/poi/business/district_bank.py index a7dc998..1591d82 100644 --- a/dags/poi/business/district_bank.py +++ b/dags/poi/business/district_bank.py @@ -11,7 +11,7 @@ log = logging.getLogger(__name__) elastic_client = ElasticSearchClient(host='192.168.0.206') district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') +administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping') # 银行code bank_typecode = ["160100", "160101", "160102", "160103", "160104", "160105", "160106", "160107", "160108", "160109", @@ -82,9 +82,9 @@ def deal(city, lng, lat): def do(): log.info("开始执行周边银行配套入库") - city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1})) - for city_item in ["哈尔滨市","齐齐哈尔市","大庆市","牡丹江市","佳木斯市","绥化市","鸡西市","七台河市","双鸭山市","鹤岗市","黑河市","伊春市","大兴安岭地区"]: - city = city_item + city_dict = list(administrative_mapping_conn.find({'province': '黑龙江省', 'type': '城市'})) + for city_item in city_dict: + city = city_item['city_short'] log.info('开始查找' + city + '周边银行') seaweed_df = pd.DataFrame( list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, @@ -93,7 +93,7 @@ def do(): continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1) + seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) actions = list() log.info(city + '周边银行开始入库') diff --git a/dags/poi/business/district_bus.py b/dags/poi/business/district_bus.py index 54c5426..d63d56c 100644 --- a/dags/poi/business/district_bus.py +++ b/dags/poi/business/district_bus.py @@ -11,7 +11,7 @@ log = logging.getLogger(__name__) elastic_client = ElasticSearchClient(host='192.168.0.206') district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') +administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping') # 交通code bus_typecode = ["150700"] @@ -75,9 +75,9 @@ def deal(city, lng, lat): def do(): log.info("开始执行周边交通配套入库") - city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1})) - for city_item in ["哈尔滨市","齐齐哈尔市","大庆市","牡丹江市","佳木斯市","绥化市","鸡西市","七台河市","双鸭山市","鹤岗市","黑河市","伊春市","大兴安岭地区"]: - city = city_item + city_dict = list(administrative_mapping_conn.find({'province': '黑龙江省', 'type': '城市'})) + for city_item in city_dict: + city = city_item['city_short'] log.info('开始查找' + city + '周边交通') seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( @@ -86,7 +86,7 @@ def do(): continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['bus'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1) + seaweed_df['bus'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) actions = list() log.info(city + '周边交通开始入库') diff --git a/dags/poi/business/district_hospital.py b/dags/poi/business/district_hospital.py index 4628326..904328b 100644 --- a/dags/poi/business/district_hospital.py +++ b/dags/poi/business/district_hospital.py @@ -12,6 +12,7 @@ elastic_client = ElasticSearchClient(host='192.168.0.206') district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') +administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping') # 医院code hospital_typecode = ["090100", "090101", "090200"] @@ -75,10 +76,9 @@ def deal(city, lng, lat): def do(): log.info("开始执行周边医院配套入库") - city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1})) - for city_item in ["哈尔滨市", "齐齐哈尔市", "大庆市", "牡丹江市", "佳木斯市", "绥化市", "鸡西市", "七台河市", - "双鸭山市", "鹤岗市", "黑河市", "伊春市", "大兴安岭地区"]: - city = city_item + city_dict = list(administrative_mapping_conn.find({'province': '黑龙江省', 'type': '城市'})) + for city_item in city_dict: + city = city_item['city_short'] log.info('开始查找' + city + '周边医院') seaweed_df = pd.DataFrame( list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, @@ -87,7 +87,7 @@ def do(): continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['hospital'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1) + seaweed_df['hospital'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) actions = list() log.info(city + '周边医院开始入库') diff --git a/dags/poi/business/district_school.py b/dags/poi/business/district_school.py index 150cf8e..01173c0 100644 --- a/dags/poi/business/district_school.py +++ b/dags/poi/business/district_school.py @@ -11,7 +11,7 @@ log = logging.getLogger(__name__) elastic_client = ElasticSearchClient(host='192.168.0.206') district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') +administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping') # 教育code school_typecode = ["141200", "141201", "141202", "141203", "141204"] @@ -74,10 +74,9 @@ def deal(city, lng, lat): def do(): log.info("开始执行周边教育配套入库") - city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1})) - for city_item in ["哈尔滨市", "齐齐哈尔市", "大庆市", "牡丹江市", "佳木斯市", "绥化市", "鸡西市", "七台河市", - "双鸭山市", "鹤岗市", "黑河市", "伊春市", "大兴安岭地区"]: - city = city_item + city_dict = list(administrative_mapping_conn.find({'province': '黑龙江省', 'type': '城市'})) + for city_item in city_dict: + city = city_item['city_short'] log.info('开始查找' + city + '周边教育') seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000))) @@ -85,7 +84,7 @@ def do(): continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['school'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1) + seaweed_df['school'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) actions = list() log.info(city + '周边教育开始入库') diff --git a/dags/poi/business/district_subway.py b/dags/poi/business/district_subway.py index 60901d5..3f46090 100644 --- a/dags/poi/business/district_subway.py +++ b/dags/poi/business/district_subway.py @@ -11,7 +11,7 @@ log = logging.getLogger(__name__) elastic_client = ElasticSearchClient(host='192.168.0.206') district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') +administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping') # 地铁code @@ -75,10 +75,9 @@ def deal(city, lng, lat): def do(): log.info("开始执行周边地铁配套入库") - city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1})) - for city_item in ["哈尔滨市", "齐齐哈尔市", "大庆市", "牡丹江市", "佳木斯市", "绥化市", "鸡西市", "七台河市", - "双鸭山市", "鹤岗市", "黑河市", "伊春市", "大兴安岭地区"]: - city = city_item + city_dict = list(administrative_mapping_conn.find({'province': '黑龙江省', 'type': '城市'})) + for city_item in city_dict: + city = city_item['city_short'] log.info('开始查找' + city + '周边地铁') seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( @@ -87,7 +86,7 @@ def do(): continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1) + seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) actions = list() log.info(city + '周边地铁开始入库') diff --git a/dags/poi/business/district_supermarket.py b/dags/poi/business/district_supermarket.py index 1132936..dfd91b2 100644 --- a/dags/poi/business/district_supermarket.py +++ b/dags/poi/business/district_supermarket.py @@ -11,7 +11,7 @@ log = logging.getLogger(__name__) elastic_client = ElasticSearchClient(host='192.168.0.206') district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') +administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping') # 商场code @@ -76,10 +76,9 @@ def deal(city, lng, lat): def do(): log.info("开始执行周边商场配套入库") - city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1})) - for city_item in ["哈尔滨市", "齐齐哈尔市", "大庆市", "牡丹江市", "佳木斯市", "绥化市", "鸡西市", "七台河市", - "双鸭山市", "鹤岗市", "黑河市", "伊春市", "大兴安岭地区"]: - city = city_item + city_dict = list(administrative_mapping_conn.find({'province': '黑龙江省', 'type': '城市'})) + for city_item in city_dict: + city = city_item['city_short'] log.info('开始查找' + city + '周边商场') seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( @@ -88,7 +87,7 @@ def do(): continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['supermarket'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1) + seaweed_df['supermarket'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) actions = list() log.info(city + '周边商场开始入库') diff --git a/dags/poi/dag_district_facilities.py b/dags/poi/dag_district_facilities.py index 723d986..105498e 100644 --- a/dags/poi/dag_district_facilities.py +++ b/dags/poi/dag_district_facilities.py @@ -16,7 +16,7 @@ with DAG( ) as dag: t1 = BashOperator( task_id='district_facilities', - bash_command='python /opt/airflow/dags/repo/dags/poi/business/district_facilities.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/poi/business/district_facilities.py {{dag_run.conf["type"] if "type" in dag_run.conf}} ', ) t1