暂时写死黑龙江省
This commit is contained in:
@@ -11,7 +11,7 @@ log = logging.getLogger(__name__)
|
||||
elastic_client = ElasticSearchClient(host='192.168.0.206')
|
||||
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
|
||||
administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping')
|
||||
|
||||
# 银行code
|
||||
bank_typecode = ["160100", "160101", "160102", "160103", "160104", "160105", "160106", "160107", "160108", "160109",
|
||||
@@ -82,9 +82,9 @@ def deal(city, lng, lat):
|
||||
|
||||
def do():
|
||||
log.info("开始执行周边银行配套入库")
|
||||
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
|
||||
for city_item in ["哈尔滨市","齐齐哈尔市","大庆市","牡丹江市","佳木斯市","绥化市","鸡西市","七台河市","双鸭山市","鹤岗市","黑河市","伊春市","大兴安岭地区"]:
|
||||
city = city_item
|
||||
city_dict = list(administrative_mapping_conn.find({'province': '黑龙江省', 'type': '城市'}))
|
||||
for city_item in city_dict:
|
||||
city = city_item['city_short']
|
||||
log.info('开始查找' + city + '周边银行')
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
|
||||
@@ -93,7 +93,7 @@ def do():
|
||||
continue
|
||||
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
|
||||
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
|
||||
seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
|
||||
seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
|
||||
|
||||
actions = list()
|
||||
log.info(city + '周边银行开始入库')
|
||||
|
||||
@@ -11,7 +11,7 @@ log = logging.getLogger(__name__)
|
||||
elastic_client = ElasticSearchClient(host='192.168.0.206')
|
||||
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
|
||||
administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping')
|
||||
|
||||
# 交通code
|
||||
bus_typecode = ["150700"]
|
||||
@@ -75,9 +75,9 @@ def deal(city, lng, lat):
|
||||
|
||||
def do():
|
||||
log.info("开始执行周边交通配套入库")
|
||||
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
|
||||
for city_item in ["哈尔滨市","齐齐哈尔市","大庆市","牡丹江市","佳木斯市","绥化市","鸡西市","七台河市","双鸭山市","鹤岗市","黑河市","伊春市","大兴安岭地区"]:
|
||||
city = city_item
|
||||
city_dict = list(administrative_mapping_conn.find({'province': '黑龙江省', 'type': '城市'}))
|
||||
for city_item in city_dict:
|
||||
city = city_item['city_short']
|
||||
log.info('开始查找' + city + '周边交通')
|
||||
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
|
||||
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
|
||||
@@ -86,7 +86,7 @@ def do():
|
||||
continue
|
||||
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
|
||||
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
|
||||
seaweed_df['bus'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
|
||||
seaweed_df['bus'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
|
||||
|
||||
actions = list()
|
||||
log.info(city + '周边交通开始入库')
|
||||
|
||||
@@ -12,6 +12,7 @@ elastic_client = ElasticSearchClient(host='192.168.0.206')
|
||||
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
|
||||
administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping')
|
||||
|
||||
# 医院code
|
||||
hospital_typecode = ["090100", "090101", "090200"]
|
||||
@@ -75,10 +76,9 @@ def deal(city, lng, lat):
|
||||
|
||||
def do():
|
||||
log.info("开始执行周边医院配套入库")
|
||||
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
|
||||
for city_item in ["哈尔滨市", "齐齐哈尔市", "大庆市", "牡丹江市", "佳木斯市", "绥化市", "鸡西市", "七台河市",
|
||||
"双鸭山市", "鹤岗市", "黑河市", "伊春市", "大兴安岭地区"]:
|
||||
city = city_item
|
||||
city_dict = list(administrative_mapping_conn.find({'province': '黑龙江省', 'type': '城市'}))
|
||||
for city_item in city_dict:
|
||||
city = city_item['city_short']
|
||||
log.info('开始查找' + city + '周边医院')
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
|
||||
@@ -87,7 +87,7 @@ def do():
|
||||
continue
|
||||
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
|
||||
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
|
||||
seaweed_df['hospital'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
|
||||
seaweed_df['hospital'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
|
||||
|
||||
actions = list()
|
||||
log.info(city + '周边医院开始入库')
|
||||
|
||||
@@ -11,7 +11,7 @@ log = logging.getLogger(__name__)
|
||||
elastic_client = ElasticSearchClient(host='192.168.0.206')
|
||||
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
|
||||
administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping')
|
||||
|
||||
# 教育code
|
||||
school_typecode = ["141200", "141201", "141202", "141203", "141204"]
|
||||
@@ -74,10 +74,9 @@ def deal(city, lng, lat):
|
||||
|
||||
def do():
|
||||
log.info("开始执行周边教育配套入库")
|
||||
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
|
||||
for city_item in ["哈尔滨市", "齐齐哈尔市", "大庆市", "牡丹江市", "佳木斯市", "绥化市", "鸡西市", "七台河市",
|
||||
"双鸭山市", "鹤岗市", "黑河市", "伊春市", "大兴安岭地区"]:
|
||||
city = city_item
|
||||
city_dict = list(administrative_mapping_conn.find({'province': '黑龙江省', 'type': '城市'}))
|
||||
for city_item in city_dict:
|
||||
city = city_item['city_short']
|
||||
log.info('开始查找' + city + '周边教育')
|
||||
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
|
||||
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000)))
|
||||
@@ -85,7 +84,7 @@ def do():
|
||||
continue
|
||||
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
|
||||
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
|
||||
seaweed_df['school'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
|
||||
seaweed_df['school'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
|
||||
|
||||
actions = list()
|
||||
log.info(city + '周边教育开始入库')
|
||||
|
||||
@@ -11,7 +11,7 @@ log = logging.getLogger(__name__)
|
||||
elastic_client = ElasticSearchClient(host='192.168.0.206')
|
||||
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
|
||||
administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping')
|
||||
|
||||
|
||||
# 地铁code
|
||||
@@ -75,10 +75,9 @@ def deal(city, lng, lat):
|
||||
|
||||
def do():
|
||||
log.info("开始执行周边地铁配套入库")
|
||||
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
|
||||
for city_item in ["哈尔滨市", "齐齐哈尔市", "大庆市", "牡丹江市", "佳木斯市", "绥化市", "鸡西市", "七台河市",
|
||||
"双鸭山市", "鹤岗市", "黑河市", "伊春市", "大兴安岭地区"]:
|
||||
city = city_item
|
||||
city_dict = list(administrative_mapping_conn.find({'province': '黑龙江省', 'type': '城市'}))
|
||||
for city_item in city_dict:
|
||||
city = city_item['city_short']
|
||||
log.info('开始查找' + city + '周边地铁')
|
||||
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
|
||||
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
|
||||
@@ -87,7 +86,7 @@ def do():
|
||||
continue
|
||||
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
|
||||
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
|
||||
seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
|
||||
seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
|
||||
|
||||
actions = list()
|
||||
log.info(city + '周边地铁开始入库')
|
||||
|
||||
@@ -11,7 +11,7 @@ log = logging.getLogger(__name__)
|
||||
elastic_client = ElasticSearchClient(host='192.168.0.206')
|
||||
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
|
||||
administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping')
|
||||
|
||||
|
||||
# 商场code
|
||||
@@ -76,10 +76,9 @@ def deal(city, lng, lat):
|
||||
|
||||
def do():
|
||||
log.info("开始执行周边商场配套入库")
|
||||
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
|
||||
for city_item in ["哈尔滨市", "齐齐哈尔市", "大庆市", "牡丹江市", "佳木斯市", "绥化市", "鸡西市", "七台河市",
|
||||
"双鸭山市", "鹤岗市", "黑河市", "伊春市", "大兴安岭地区"]:
|
||||
city = city_item
|
||||
city_dict = list(administrative_mapping_conn.find({'province': '黑龙江省', 'type': '城市'}))
|
||||
for city_item in city_dict:
|
||||
city = city_item['city_short']
|
||||
log.info('开始查找' + city + '周边商场')
|
||||
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
|
||||
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
|
||||
@@ -88,7 +87,7 @@ def do():
|
||||
continue
|
||||
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
|
||||
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
|
||||
seaweed_df['supermarket'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
|
||||
seaweed_df['supermarket'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
|
||||
|
||||
actions = list()
|
||||
log.info(city + '周边商场开始入库')
|
||||
|
||||
@@ -16,7 +16,7 @@ with DAG(
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='district_facilities',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/poi/business/district_facilities.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/poi/business/district_facilities.py {{dag_run.conf["type"] if "type" in dag_run.conf}} ',
|
||||
)
|
||||
t1
|
||||
|
||||
|
||||
Reference in New Issue
Block a user