优化同步问题

This commit is contained in:
Jian Shen
2022-08-23 20:41:06 +08:00
parent 65aed9d6b9
commit 2466dcf5a7
7 changed files with 217 additions and 208 deletions
+34 -29
View File
@@ -13,8 +13,6 @@ district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
# 银行code
bank_typecode = ["160100", "160101", "160102", "160103", "160104", "160105", "160106", "160107", "160108", "160109",
"160110",
@@ -26,7 +24,6 @@ bank_typecode = ["160100", "160101", "160102", "160103", "160104", "160105", "16
"160144",
"160145", "160146", "160147", "160148", "160149", "160150", "160151", "160152", "160200"]
log.info("开始执行周边银行配套入库")
def find_around_by_location(city, location):
"""
@@ -77,37 +74,45 @@ def find_around_by_location(city, location):
print(e)
return []
def deal(city, lng, lat):
location = [lng, lat]
bank_df = find_around_by_location(city, location)
return bank_df
for city_item in city_dict:
city = city_item['display']
log.info('开始查找' + city + '周边银行')
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边银行开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']}, {'$set': {'bank': item['bank'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True)
actions.append(action)
except Exception as e:
print(e)
if len(actions) == 5000:
def do():
log.info("开始执行周边银行配套入库")
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
for city_item in city_dict:
city = city_item['display']
log.info('开始查找' + city + '周边银行')
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边银行开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']},
{'$set': {'bank': item['bank'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}},
upsert=True)
actions.append(action)
except Exception as e:
print(e)
if len(actions) == 5000:
district_facilitys_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
district_facilitys_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
district_facilitys_conn.bulk_write(actions)
actions.clear()
log.info(city + '周边银行入库完毕')
log.info(city + '周边银行入库完毕')
+33 -32
View File
@@ -13,12 +13,9 @@ district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
# 交通code
bus_typecode = ["150700","150500"]
log.info("开始执行周边交通配套入库")
def find_around_by_location(city, location):
"""
@@ -75,33 +72,37 @@ def deal(city, lng, lat):
bus_df = find_around_by_location(city, location)
return bus_df
for city_item in city_dict:
city = city_item['display']
log.info('开始查找' + city + '周边交通')
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['bus'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边交通开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']}, {'$set': {'bus': item['bus'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True)
actions.append(action)
except Exception as e:
print(e)
if len(actions) == 5000:
district_facilitys_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
district_facilitys_conn.bulk_write(actions)
actions.clear()
log.info(city + '周边交通入库完毕')
def do():
log.info("开始执行周边交通配套入库")
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
for city_item in city_dict:
city = city_item['display']
log.info('开始查找' + city + '周边交通')
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['bus'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边交通开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']}, {'$set': {'bus': item['bus'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True)
actions.append(action)
except Exception as e:
print(e)
if len(actions) == 5000:
district_facilitys_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
district_facilitys_conn.bulk_write(actions)
actions.clear()
log.info(city + '周边交通入库完毕')
+18 -24
View File
@@ -1,4 +1,10 @@
import sys
import district_bank as bank
import district_bus as bus
import district_hospital as hospital
import district_school as school
import district_subway as subway
import district_supermarket as supermarket
type = 'all'
if len(sys.argv) > 1:
@@ -8,34 +14,22 @@ print("输出类型" + type)
if type is not None:
if type == 'bank':
import district_bank as bank
bank
bank.do()
elif type == 'bus':
import district_bus as bus
bus
bus.do()
elif type == 'hospital':
import district_hospital as hospital
hospital
hospital.do()
elif type == 'school':
import district_school as school
school
school.do()
elif type == 'subway':
import district_subway as subway
subway
subway.do()
elif type == 'supermarket':
import district_supermarket as supermarket
supermarket
supermarket.do()
else:
import district_bank as bank
import district_bus as bus
import district_hospital as hospital
import district_school as school
import district_subway as subway
import district_supermarket as supermarket
bank
bus
hospital
school
subway
supermarket
bank.do()
bus.do()
hospital.do()
school.do()
subway.do()
supermarket.do()
+35 -29
View File
@@ -13,12 +13,9 @@ district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
# 医院code
hospital_typecode = ["090100", "090101", "090200"]
log.info("开始执行周边医院配套入库")
def find_around_by_location(city, location):
"""
@@ -69,37 +66,46 @@ def find_around_by_location(city, location):
print(e)
return []
def deal(city, lng, lat):
location = [lng, lat]
hospital_df = find_around_by_location(city, location)
return hospital_df
for city_item in city_dict:
city = city_item['display']
log.info('开始查找' + city + '周边医院')
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['hospital'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边医院开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']}, {'$set': {'hospital': item['hospital'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True)
actions.append(action)
except Exception as e:
print(e)
if len(actions) == 5000:
def do():
log.info("开始执行周边医院配套入库")
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
for city_item in city_dict:
city = city_item['display']
log.info('开始查找' + city + '周边医院')
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['hospital'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边医院开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']}, {
'$set': {'hospital': item['hospital'], 'city': item['city'],
'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}},
upsert=True)
actions.append(action)
except Exception as e:
print(e)
if len(actions) == 5000:
district_facilitys_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
district_facilitys_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
district_facilitys_conn.bulk_write(actions)
actions.clear()
log.info(city + '周边医院入库完毕')
log.info(city + '周边医院入库完毕')
+32 -31
View File
@@ -13,12 +13,9 @@ district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
# 教育code
school_typecode = ["141200", "141201", "141202", "141203", "141204"]
log.info("开始执行周边教育配套入库")
def find_around_by_location(city, location):
"""
@@ -74,32 +71,36 @@ def deal(city, lng, lat):
school_df = find_around_by_location(city, location)
return school_df
for city_item in city_dict:
city = city_item['display']
log.info('开始查找' + city + '周边教育')
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['school'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边教育开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']}, {'$set': {'school': item['school'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True)
actions.append(action)
except Exception as e:
print(e)
if len(actions) == 5000:
district_facilitys_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
district_facilitys_conn.bulk_write(actions)
actions.clear()
log.info(city + '周边教育入库完毕')
def do():
log.info("开始执行周边教育配套入库")
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
for city_item in city_dict:
city = city_item['display']
log.info('开始查找' + city + '周边教育')
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['school'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边教育开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']}, {'$set': {'school': item['school'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True)
actions.append(action)
except Exception as e:
print(e)
if len(actions) == 5000:
district_facilitys_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
district_facilitys_conn.bulk_write(actions)
actions.clear()
log.info(city + '周边教育入库完毕')
+33 -32
View File
@@ -13,13 +13,10 @@ district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
# 地铁code
subway_typecode = ["150500"]
log.info("开始执行周边地铁配套入库")
def find_around_by_location(city, location):
"""
查找周边poi
@@ -75,33 +72,37 @@ def deal(city, lng, lat):
subway_df = find_around_by_location(city, location)
return subway_df
for city_item in city_dict:
city = city_item['display']
log.info('开始查找' + city + '周边地铁')
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边地铁开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']}, {'$set': {'subway': item['subway'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True)
actions.append(action)
except Exception as e:
print(e)
if len(actions) == 5000:
district_facilitys_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
district_facilitys_conn.bulk_write(actions)
actions.clear()
log.info(city + '周边地铁入库完毕')
def do():
log.info("开始执行周边地铁配套入库")
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
for city_item in city_dict:
city = city_item['display']
log.info('开始查找' + city + '周边地铁')
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边地铁开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']}, {'$set': {'subway': item['subway'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True)
actions.append(action)
except Exception as e:
print(e)
if len(actions) == 5000:
district_facilitys_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
district_facilitys_conn.bulk_write(actions)
actions.clear()
log.info(city + '周边地铁入库完毕')
+32 -31
View File
@@ -13,12 +13,10 @@ district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
# 商场code
supermarket_typecode = ["060100","060101","060102"]
log.info("开始执行周边商场配套入库")
def find_around_by_location(city, location):
"""
@@ -76,33 +74,36 @@ def deal(city, lng, lat):
return supermarket_df
for city_item in city_dict:
city = city_item['display']
log.info('开始查找' + city + '周边商场')
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['supermarket'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
def do():
log.info("开始执行周边商场配套入库")
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
for city_item in city_dict:
city = city_item['display']
log.info('开始查找' + city + '周边商场')
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['supermarket'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边商场开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']}, {'$set': {'supermarket': item['supermarket'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True)
actions.append(action)
except Exception as e:
print(e)
if len(actions) == 5000:
district_facilitys_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
district_facilitys_conn.bulk_write(actions)
actions.clear()
log.info(city + '周边商场入库完毕')
actions = list()
log.info(city + '周边商场开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']}, {'$set': {'supermarket': item['supermarket'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True)
actions.append(action)
except Exception as e:
print(e)
if len(actions) == 5000:
district_facilitys_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
district_facilitys_conn.bulk_write(actions)
actions.clear()
log.info(city + '周边商场入库完毕')