周边配套计算库修改

This commit is contained in:
沈健
2024-12-23 15:26:44 +08:00
parent 075577afd7
commit 528a257113
7 changed files with 120 additions and 39 deletions
+12 -3
View File
@@ -26,14 +26,23 @@ bank_typecode = ["160100", "160101", "160102", "160103", "160104", "160105", "16
"160144",
"160145", "160146", "160147", "160148", "160149", "160150", "160151", "160152", "160200"]
small_type = ["银行", "中国人民银行", "国家开发银行", "中国进出口银行", "中国银行", "中国工商银行", "中国建设银行",
"中国农业银行", "交通银行", "招商银行", "华夏银行", "中信银行", "中国民生银行", "中国光大银行",
"上海银行", "上海浦东发展银行", "平安银行", "兴业银行", "北京银行", "广发银行", "农村商业银行",
"香港恒生银行", "东亚银行", "花旗银行", "渣打银行", "汇丰银行", "荷兰银行", "美国运通银行",
"瑞士友邦银行", "美国银行", "蒙特利尔银行", "纽约银行", "苏格兰皇家银行", "法国兴业银行", "德意志银行",
"日本三菱东京日联银行", "巴克莱银行", "摩根大通银行", "中国邮政储蓄银行", "香港星展银行", "南洋商业银行",
"上海商业银行", "永亨银行", "香港永隆银行", "创兴银行", "大新银行", "中信银行(国际)", "大众银行(香港)",
"北京农商银行", "上海农商银行", "广州农商银行", "深圳农村商业银行", "银行相关"]
def deal3(item):
location = [item['lng'], item['lat']]
item['bank'] = poiutils.find_around_by_location(item['city_display'], location, bank_typecode, 3000)
item['bank'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 3000)
return item
def deal(item):
location = [item['lng'], item['lat']]
item['bank'] = poiutils.find_around_by_location(item['city_display'], location, bank_typecode, 1000)
item['bank'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 1000)
return item
def do():
@@ -113,4 +122,4 @@ def sync1():
if len(actions) > 0:
district_facilitys_conn.bulk_write(actions)
actions.clear()
log.info(city + '周边银行入库完毕')
log.info(city + '周边银行入库完毕')
+3 -2
View File
@@ -16,16 +16,17 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', '
# 交通code
bus_typecode = ["150700"]
small_type = ["公交车站相关"]
def deal3(item):
location = [item['lng'], item['lat']]
item['bus'] = poiutils.find_around_by_location(item['city_display'], location, bus_typecode, 3000)
item['bus'] = poiutils.find_around_by_location_for_bus(item['city_display'], location, small_type, 3000)
return item
def deal(item):
location = [item['lng'], item['lat']]
item['bus'] = poiutils.find_around_by_location(item['city_display'], location, bus_typecode, 1000)
item['bus'] = poiutils.find_around_by_location_for_bus(item['city_display'], location, small_type, 1000)
return item
def do():
+3 -3
View File
@@ -18,16 +18,16 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', '
# 医院code
hospital_typecode = ["090100", "090101", "090200"]
small_type = ["综合医院", "三级甲等医院", "专科医院"]
def deal3(item):
location = [item['lng'], item['lat']]
item['hospital'] = poiutils.find_around_by_location(item['city_display'], location, hospital_typecode, 3000)
item['hospital'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 3000)
return item
def deal(item):
location = [item['lng'], item['lat']]
item['hospital'] = poiutils.find_around_by_location(item['city_display'], location, hospital_typecode, 1000)
item['hospital'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 1000)
return item
+3 -3
View File
@@ -16,16 +16,16 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', '
# 教育code
school_typecode = ["141200", "141201", "141202", "141203", "141204"]
small_type = ["学校", "高等院校", "中学", "小学", "幼儿园"]
def deal3(item):
location = [item['lng'], item['lat']]
item['school'] = poiutils.find_around_by_location(item['city_display'], location, school_typecode, 3000)
item['school'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 3000)
return item
def deal(item):
location = [item['lng'], item['lat']]
item['school'] = poiutils.find_around_by_location(item['city_display'], location, school_typecode, 1000)
item['school'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 1000)
return item
+32 -19
View File
@@ -9,7 +9,6 @@ import poiutils
log = logging.getLogger(__name__)
elastic_client = ElasticSearchClient(host='192.168.0.206')
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
surrounding_facilities_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'surrounding_facilities')
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -17,44 +16,52 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', '
# 地铁code
subway_typecode = ["150500"]
small_type = ["地铁站"]
def deal3(item):
location = [item['lng'], item['lat']]
item['subway'] = poiutils.find_around_by_location(item['city_display'], location, subway_typecode, 3000)
item['subway'] = poiutils.find_around_by_location_for_bus(item['city_display'], location, small_type, 3000)
return item
def deal(item):
location = [item['lng'], item['lat']]
item['subway'] = poiutils.find_around_by_location(item['city_display'], location, subway_typecode, 1000)
item['subway'] = poiutils.find_around_by_location_for_bus(item['city_display'], location, small_type, 1000)
return item
def do():
sync_new()
sync1()
def sync_new(distance=3000):
city_dict = list(administrative_mapping_conn.find({'type': '城市'}))
for city_item in city_dict:
city = city_item['city_short']
city_display = city_item['city']
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
10000)))
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['city_display'] = city_display
seaweed_df = process_dataframe_multithread(seaweed_df, deal3, 30)
#seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1)
# seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1)
actions = list()
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']}, {'$set': {'subway': item['subway'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat'], 'distance': distance}}, upsert=True)
'district_name': item['name']},
{'$set': {'subway': item['subway'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat'],
'distance': distance}}, upsert=True)
actions.append(action)
except Exception as e:
print(e)
@@ -62,33 +69,37 @@ def sync_new(distance=3000):
surrounding_facilities_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
surrounding_facilities_conn.bulk_write(actions)
actions.clear()
surrounding_facilities_conn.bulk_write(actions)
actions.clear()
log.info(city + '周边地铁入库完毕')
def sync1():
city_dict = list(administrative_mapping_conn.find({'type': '城市'}))
for city_item in city_dict:
city = city_item['city_short']
city_display = city_item['city']
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
10000)))
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['city_display'] = city_display
seaweed_df = process_dataframe_multithread(seaweed_df, deal, 30)
#seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
# seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
actions = list()
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']}, {'$set': {'subway': item['subway'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True)
'district_name': item['name']},
{'$set': {'subway': item['subway'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}},
upsert=True)
actions.append(action)
except Exception as e:
print(e)
@@ -96,6 +107,8 @@ def sync1():
district_facilitys_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
district_facilitys_conn.bulk_write(actions)
actions.clear()
district_facilitys_conn.bulk_write(actions)
actions.clear()
log.info(city + '周边地铁入库完毕')
+4 -4
View File
@@ -16,17 +16,17 @@ surrounding_facilities_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', '
# 商场code
supermarket_typecode = ["060100","060101","060102"]
supermarket_typecode = ["060100", "060101", "060102"]
small_type = ["商场", "购物中心", "普通商场"]
def deal3(item):
location = [item['lng'], item['lat']]
item['supermarket'] = poiutils.find_around_by_location(item['city_display'], location, supermarket_typecode, 3000)
item['supermarket'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 3000)
return item
def deal(item):
location = [item['lng'], item['lat']]
item['supermarket'] = poiutils.find_around_by_location(item['city_display'], location, supermarket_typecode, 1000)
item['supermarket'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 1000)
return item
def do():
+63 -5
View File
@@ -1,8 +1,8 @@
from goojia_utils import ElasticSearchClient
elastic_client = ElasticSearchClient(host='192.168.0.206')
elastic_client = ElasticSearchClient(host='192.168.0.89')
def find_around_by_location(city, location, typecode, distance=1000):
def find_around_by_location_for_bus(city, location, small_type, distance=1000):
"""
查找周边poi
:param city:
@@ -12,7 +12,58 @@ def find_around_by_location(city, location, typecode, distance=1000):
"""
try:
ret = elastic_client.scan_to_data_frame('amap_poi', {
ret = elastic_client.scan_to_data_frame('gaode_map_poi', {
"_source": ["name"],
"query": {
"bool": {
"must_not": [
{
"term": {
"main_type": {
"value": 0 # 表示清洗后的数据
}
}
}
],
"filter": [
{"terms": {
"small_type": small_type
}},
{
"term": {
"city": {
"value": city
}
}
},
{
"geo_distance": {
"distance": distance,
"location": location
}
}
]
}
}
});
if len(ret) > 0:
return ret['name'].tolist()
return []
except Exception as e:
print(e)
return []
def find_around_by_location(city, location, small_type, distance=1000):
"""
查找周边poi
:param city:
:param location:
:param distance:
:return:
"""
try:
ret = elastic_client.scan_to_data_frame('gaode_map_poi', {
"_source": ["name"],
"query": {
"bool": {
@@ -23,15 +74,22 @@ def find_around_by_location(city, location, typecode, distance=1000):
"value": 1
}
}
},
{
"term": {
"main_type": {
"value": 0 # 表示清洗后的数据
}
}
}
],
"filter": [
{"terms": {
"typecode": typecode
"small_type": small_type
}},
{
"term": {
"cityname": {
"city": {
"value": city
}
}