新增小区43同步至100高效方式

This commit is contained in:
Jian Shen
2022-08-22 15:36:15 +08:00
parent d5c3c8af94
commit f9fbb84065
2 changed files with 87 additions and 0 deletions
+70
View File
@@ -0,0 +1,70 @@
import logging
import sys
from goojia_utils import MAIN_MONGO_CLIENT, DIDATA_MONGO_CLIENT, areautils
from pymongo import UpdateOne
from bson.objectid import ObjectId
import pandas as pd
import time
log = logging.getLogger(__name__)
type = sys.argv[1] if len(sys.argv) > 1 else ''
log.info("同步类型:" + type)
def sync_district_office(type):
if type != 'district' and type != 'office':
return
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
seaweed_didata_conn = DIDATA_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
cities = areautils.get_all_cities()
update_date = int(round(time.time() * 1000))
for city in cities:
actions = list()
id_list = list()
for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0}, {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1,
'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1, 'status': 1})\
.batch_size(10000):
updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'],
'estate_type': doc['estate_type2'], 'status': doc['status']}
id_list.append({'_id': doc['_id']})
if 'block' in doc:
updateDict['block'] = doc['block']
if 'address' in doc:
updateDict['address'] = doc['address']
if 'township' in doc:
updateDict['township'] = doc['township']
if 'alias' in doc:
updateDict['alias'] = doc['alias']
if 'addr_alias' in doc:
updateDict['addr_alias'] = doc['addr_alias']
if 'lng' in doc and str(doc['lng']) != 'nan':
updateDict['location'] = {'lon': doc['lng'] / 10**10, 'lat': doc['lat'] / 10**10}
updateDict['update_date'] = update_date
action = UpdateOne({'_id': doc['_id']}, {'$set': updateDict}, upsert=True)
actions.append(action)
if len(actions) == 10000:
seaweed_didata_conn.bulk_write(actions)
actions.clear()
didata_id_df = pd.DataFrame(list(seaweed_didata_conn.find({'city': city, 'cat': type}, {'_id': 1}).batch_size(10000)))[['_id']]
id_df = pd.DataFrame(id_list)
deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False)
for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows():
action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, {'$set': {'status': 1, 'update_date': update_date}}, upsert=False)
actions.append(action)
if len(actions) == 10000:
seaweed_didata_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
seaweed_didata_conn.bulk_write(actions)
actions.clear()
log.info(city + "-" + type + "-同步完成")
sync_district_office(type)
+17
View File
@@ -0,0 +1,17 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
"didata_sync_dag",
# 每月28日3点整分运行
schedule_interval="0 3 28 * *",
start_date=pendulum.datetime(2022, 8, 22, tz="Asia/Shanghai"),
catchup=True,
description="地数相关数据同步type: district或office"
) as dag:
t1 = BashOperator(
task_id='didata_sync_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/didata_sync.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t1