周边POI改为多线程

This commit is contained in:
sjshenjian
2024-11-20 18:59:55 +08:00
parent 3ed3bf7641
commit 67eed5a9df
6 changed files with 97 additions and 73 deletions
+17 -12
View File
@@ -2,7 +2,7 @@
小区周边1公里银行入库业务逻辑
"""
import logging
from goojia_utils import MAIN_MONGO_CLIENT
from goojia_utils import MAIN_MONGO_CLIENT, process_dataframe_multithread
import pandas as pd
from pymongo import UpdateOne
@@ -26,10 +26,15 @@ bank_typecode = ["160100", "160101", "160102", "160103", "160104", "160105", "16
"160144",
"160145", "160146", "160147", "160148", "160149", "160150", "160151", "160152", "160200"]
def deal3(item):
location = [item['lng'], item['lat']]
item['bank'] = poiutils.find_around_by_location(item['city_display'], location, bank_typecode, 3000)
return item
def deal(city, lng, lat, distance=1000):
location = [lng, lat]
return poiutils.find_around_by_location(city, location, bank_typecode, distance)
def deal(item):
location = [item['lng'], item['lat']]
item['bank'] = poiutils.find_around_by_location(item['city_display'], location, bank_typecode, 1000)
return item
def do():
sync_new()
@@ -39,11 +44,10 @@ def sync_new(distance=3000):
"""
周边3公里POI入库
"""
log.info("开始执行周边银行配套入库")
city_dict = list(administrative_mapping_conn.find({'type': '城市'}))
for city_item in city_dict:
city = city_item['city_short']
log.info('开始查找' + city + '周边银行')
city_display = city_item['city']
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000)))
@@ -51,10 +55,11 @@ def sync_new(distance=3000):
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1)
seaweed_df['city_display'] = city_display
seaweed_df = process_dataframe_multithread(seaweed_df, deal3, 30)
#seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1)
actions = list()
log.info(city + '周边银行开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
@@ -75,11 +80,10 @@ def sync_new(distance=3000):
log.info(city + '周边银行入库完毕')
def sync1():
log.info("开始执行周边银行配套入库")
city_dict = list(administrative_mapping_conn.find({'type': '城市'}))
for city_item in city_dict:
city = city_item['city_short']
log.info('开始查找' + city + '周边银行')
city_display = city_item['city']
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000)))
@@ -87,10 +91,11 @@ def sync1():
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
seaweed_df['city_display'] = city_display
seaweed_df = process_dataframe_multithread(seaweed_df, deal, 30)
#seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边银行开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
+16 -12
View File
@@ -2,7 +2,7 @@
小区周边1公里交通地铁入库业务逻辑
"""
import logging
from goojia_utils import MAIN_MONGO_CLIENT
from goojia_utils import MAIN_MONGO_CLIENT, process_dataframe_multithread
import pandas as pd
from pymongo import UpdateOne
import poiutils
@@ -18,9 +18,15 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', '
bus_typecode = ["150700"]
def deal(city, lng, lat, distance=1000):
location = [lng, lat]
return poiutils.find_around_by_location(city, location, bus_typecode, distance)
def deal3(item):
location = [item['lng'], item['lat']]
item['bus'] = poiutils.find_around_by_location(item['city_display'], location, bus_typecode, 3000)
return item
def deal(item):
location = [item['lng'], item['lat']]
item['bus'] = poiutils.find_around_by_location(item['city_display'], location, bus_typecode, 1000)
return item
def do():
sync_new()
@@ -28,11 +34,10 @@ def do():
def sync_new(distance=3000):
log.info("开始执行周边交通配套入库")
city_dict = list(administrative_mapping_conn.find({'type': '城市'}))
for city_item in city_dict:
city = city_item['city_short']
log.info('开始查找' + city + '周边交通')
city_display = city_item['city']
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
@@ -41,10 +46,10 @@ def sync_new(distance=3000):
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['bus'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1)
seaweed_df = process_dataframe_multithread(seaweed_df, deal3, 30)
#seaweed_df['bus'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1)
actions = list()
log.info(city + '周边交通开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
@@ -65,11 +70,10 @@ def sync_new(distance=3000):
log.info(city + '周边交通入库完毕')
def sync1():
log.info("开始执行周边交通配套入库")
city_dict = list(administrative_mapping_conn.find({'type': '城市'}))
for city_item in city_dict:
city = city_item['city_short']
log.info('开始查找' + city + '周边交通')
city_display = city_item['city']
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
@@ -78,10 +82,10 @@ def sync1():
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['bus'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
seaweed_df = process_dataframe_multithread(seaweed_df, deal, 30)
#seaweed_df['bus'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边交通开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
+16 -10
View File
@@ -2,7 +2,7 @@
小区周边1公里医院入库业务逻辑
"""
import logging
from goojia_utils import MAIN_MONGO_CLIENT
from goojia_utils import MAIN_MONGO_CLIENT, process_dataframe_multithread
import poiutils
import pandas as pd
@@ -20,10 +20,15 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', '
hospital_typecode = ["090100", "090101", "090200"]
def deal(city, lng, lat, distance=1000):
location = [lng, lat]
hospital_df = poiutils.find_around_by_location(city, location, hospital_typecode, distance)
return hospital_df
def deal3(item):
location = [item['lng'], item['lat']]
item['hospital'] = poiutils.find_around_by_location(item['city_display'], location, hospital_typecode, 3000)
return item
def deal(item):
location = [item['lng'], item['lat']]
item['hospital'] = poiutils.find_around_by_location(item['city_display'], location, hospital_typecode, 1000)
return item
def do():
@@ -35,6 +40,7 @@ def sync_new(distance=3000):
city_dict = list(administrative_mapping_conn.find({'type': '城市'}))
for city_item in city_dict:
city = city_item['city_short']
city_display = city_item['city']
log.info('开始查找' + city + '周边医院')
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
@@ -43,7 +49,8 @@ def sync_new(distance=3000):
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['hospital'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1)
seaweed_df = process_dataframe_multithread(seaweed_df, deal3, 30)
#seaweed_df['hospital'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1)
actions = list()
log.info(city + '周边医院开始入库')
@@ -69,11 +76,10 @@ def sync_new(distance=3000):
def sync1():
log.info("开始执行周边医院配套入库")
city_dict = list(administrative_mapping_conn.find({'type': '城市'}))
for city_item in city_dict:
city = city_item['city_short']
log.info('开始查找' + city + '周边医院')
city_display = city_item['city']
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000)))
@@ -81,10 +87,10 @@ def sync1():
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['hospital'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
seaweed_df = process_dataframe_multithread(seaweed_df, deal, 30)
#seaweed_df['hospital'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边医院开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
+16 -13
View File
@@ -2,7 +2,7 @@
小区周边1公里教育入库业务逻辑
"""
import logging
from goojia_utils import MAIN_MONGO_CLIENT
from goojia_utils import MAIN_MONGO_CLIENT, process_dataframe_multithread
import pandas as pd
from pymongo import UpdateOne
import poiutils
@@ -18,10 +18,15 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', '
school_typecode = ["141200", "141201", "141202", "141203", "141204"]
def deal(city, lng, lat, distance=1000):
location = [lng, lat]
school_df = poiutils.find_around_by_location(city, location, school_typecode, distance)
return school_df
def deal3(item):
location = [item['lng'], item['lat']]
item['school'] = poiutils.find_around_by_location(item['city_display'], location, school_typecode, 3000)
return item
def deal(item):
location = [item['lng'], item['lat']]
item['school'] = poiutils.find_around_by_location(item['city_display'], location, school_typecode, 1000)
return item
def do():
@@ -29,21 +34,20 @@ def do():
sync1()
def sync_new(distance=3000):
log.info("开始执行周边教育配套入库")
city_dict = list(administrative_mapping_conn.find({'type': '城市'}))
for city_item in city_dict:
city = city_item['city_short']
log.info('开始查找' + city + '周边教育')
city_display = city_item['city']
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['school'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1)
seaweed_df = process_dataframe_multithread(seaweed_df, deal3, 30)
#seaweed_df['school'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1)
actions = list()
log.info(city + '周边教育开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
@@ -62,21 +66,20 @@ def sync_new(distance=3000):
log.info(city + '周边教育入库完毕')
def sync1():
log.info("开始执行周边教育配套入库")
city_dict = list(administrative_mapping_conn.find({'type': '城市'}))
for city_item in city_dict:
city = city_item['city_short']
log.info('开始查找' + city + '周边教育')
city_display = city_item['city']
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['school'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
seaweed_df = process_dataframe_multithread(seaweed_df, deal, 30)
#seaweed_df['school'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边教育开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
+16 -13
View File
@@ -2,7 +2,7 @@
小区周边1公里地铁地铁入库业务逻辑
"""
import logging
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, process_dataframe_multithread
import pandas as pd
from pymongo import UpdateOne
import poiutils
@@ -18,21 +18,25 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', '
# 地铁code
subway_typecode = ["150500"]
def deal(city, lng, lat, distance=1000):
location = [lng, lat]
subway_df = poiutils.find_around_by_location(city, location, subway_typecode, distance)
return subway_df
def deal3(item):
location = [item['lng'], item['lat']]
item['subway'] = poiutils.find_around_by_location(item['city_display'], location, subway_typecode, 3000)
return item
def deal(item):
location = [item['lng'], item['lat']]
item['subway'] = poiutils.find_around_by_location(item['city_display'], location, subway_typecode, 1000)
return item
def do():
sync_new()
sync1()
def sync_new(distance=3000):
log.info("开始执行周边地铁配套入库")
city_dict = list(administrative_mapping_conn.find({'type': '城市'}))
for city_item in city_dict:
city = city_item['city_short']
log.info('开始查找' + city + '周边地铁')
city_display = city_item['city']
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
10000)))
@@ -40,10 +44,10 @@ def sync_new(distance=3000):
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1)
seaweed_df = process_dataframe_multithread(seaweed_df, deal3, 30)
#seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1)
actions = list()
log.info(city + '周边地铁开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
@@ -62,11 +66,10 @@ def sync_new(distance=3000):
log.info(city + '周边地铁入库完毕')
def sync1():
log.info("开始执行周边地铁配套入库")
city_dict = list(administrative_mapping_conn.find({'type': '城市'}))
for city_item in city_dict:
city = city_item['city_short']
log.info('开始查找' + city + '周边地铁')
city_display = city_item['city']
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
10000)))
@@ -74,10 +77,10 @@ def sync1():
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
seaweed_df = process_dataframe_multithread(seaweed_df, deal, 30)
#seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边地铁开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
+16 -13
View File
@@ -2,7 +2,7 @@
小区周边1公里商场入库业务逻辑
"""
import logging
from goojia_utils import MAIN_MONGO_CLIENT
from goojia_utils import MAIN_MONGO_CLIENT, process_dataframe_multithread
import pandas as pd
from pymongo import UpdateOne
import poiutils
@@ -19,21 +19,25 @@ surrounding_facilities_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', '
supermarket_typecode = ["060100","060101","060102"]
def deal(city, lng, lat, distance=1000):
location = [lng, lat]
supermarket_df = poiutils.find_around_by_location(city, location, supermarket_typecode, distance)
return supermarket_df
def deal3(item):
location = [item['lng'], item['lat']]
item['supermarket'] = poiutils.find_around_by_location(item['city_display'], location, supermarket_typecode, 3000)
return item
def deal(item):
location = [item['lng'], item['lat']]
item['supermarket'] = poiutils.find_around_by_location(item['city_display'], location, supermarket_typecode, 1000)
return item
def do():
sync_new()
sync1()
def sync_new():
log.info("开始执行周边商场配套入库")
city_dict = list(administrative_mapping_conn.find({'type': '城市'}))
for city_item in city_dict:
city = city_item['city_short']
log.info('开始查找' + city + '周边商场')
city_display = city_item['city']
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
10000)))
@@ -41,10 +45,10 @@ def sync_new():
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['supermarket'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
seaweed_df = process_dataframe_multithread(seaweed_df, deal3, 30)
#seaweed_df['supermarket'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边商场开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
@@ -63,11 +67,10 @@ def sync_new():
log.info(city + '周边商场入库完毕')
def sync1():
log.info("开始执行周边商场配套入库")
city_dict = list(administrative_mapping_conn.find({'type': '城市'}))
for city_item in city_dict:
city = city_item['city_short']
log.info('开始查找' + city + '周边商场')
city_display = city_item['city']
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
10000)))
@@ -75,10 +78,10 @@ def sync1():
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['supermarket'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
seaweed_df = process_dataframe_multithread(seaweed_df, deal, 30)
#seaweed_df['supermarket'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边商场开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id