diff --git a/dags/poi/business/district_bank.py b/dags/poi/business/district_bank.py index aa7d10f..e8e18b0 100644 --- a/dags/poi/business/district_bank.py +++ b/dags/poi/business/district_bank.py @@ -9,6 +9,9 @@ from pymongo import UpdateOne elastic_client = ElasticSearchClient(host='192.168.0.206') district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') + +city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1})) # 银行code bank_typecode = ["160100", "160101", "160102", "160103", "160104", "160105", "160106", "160107", "160108", "160109", @@ -21,13 +24,6 @@ bank_typecode = ["160100", "160101", "160102", "160103", "160104", "160105", "16 "160144", "160145", "160146", "160147", "160148", "160149", "160150", "160151", "160152", "160200"] -city = '临沂' -seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0}, - {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( - 10000))) -seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 -seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - def find_around_by_location(city, location): """ @@ -83,24 +79,31 @@ def deal(city, lng, lat): return [] -print('开始查找' + city + '周边银行') -seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1) +for city_item in city_dict: + city = city_item['display'] + print('开始查找' + city + '周边银行') + seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0}, + {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( + 10000))) + seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 + seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 + seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1) -actions = list() -print(city + '周边银行开始入库') -for index, item in seaweed_df.iterrows(): - try: - # 周边配套库中历史原因,id可能不为小区id - action = UpdateOne({'city': item['city'], 'region': item['region'], - 'district_name': item['name']}, {'$set': {'bank': item['bank'], 'city': item['city'], 'region': item['region'], - 'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True) - actions.append(action) - except Exception as e: - print(e) - if len(actions) == 5000: - district_facilitys_conn.bulk_write(actions) - actions.clear() -if len(actions) > 0: - district_facilitys_conn.bulk_write(actions) - actions.clear() -print(city + '周边银行入库完毕') + actions = list() + print(city + '周边银行开始入库') + for index, item in seaweed_df.iterrows(): + try: + # 周边配套库中历史原因,id可能不为小区id + action = UpdateOne({'city': item['city'], 'region': item['region'], + 'district_name': item['name']}, {'$set': {'bank': item['bank'], 'city': item['city'], 'region': item['region'], + 'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True) + actions.append(action) + except Exception as e: + print(e) + if len(actions) == 5000: + district_facilitys_conn.bulk_write(actions) + actions.clear() + if len(actions) > 0: + district_facilitys_conn.bulk_write(actions) + actions.clear() + print(city + '周边银行入库完毕')