版本修改
This commit is contained in:
@@ -1,82 +1,82 @@
|
||||
import logging
|
||||
import sys
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, DIDATA_MONGO_CLIENT, areautils
|
||||
from pymongo import UpdateOne
|
||||
from bson.objectid import ObjectId
|
||||
import pandas as pd
|
||||
import time
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
type = sys.argv[1] if len(sys.argv) > 1 else 'all'
|
||||
log.info("同步类型:" + type)
|
||||
|
||||
|
||||
def sync_district_office(type):
|
||||
if type != 'district' and type != 'office':
|
||||
return
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
seaweed_didata_conn = DIDATA_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
|
||||
cities = areautils.get_all_cities()
|
||||
update_date = int(round(time.time() * 1000))
|
||||
for city in cities:
|
||||
actions = list()
|
||||
id_list = list()
|
||||
for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0},
|
||||
{'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1,
|
||||
'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1,
|
||||
'status': 1}) \
|
||||
.batch_size(10000):
|
||||
updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'],
|
||||
'status': doc['status']}
|
||||
if 'estate_type2' in doc:
|
||||
updateDict['estate_type'] = doc['estate_type2']
|
||||
|
||||
id_list.append({'_id': doc['_id']})
|
||||
if 'block' in doc:
|
||||
updateDict['block'] = doc['block']
|
||||
if 'address' in doc:
|
||||
updateDict['address'] = doc['address']
|
||||
if 'township' in doc:
|
||||
updateDict['township'] = doc['township']
|
||||
if 'alias' in doc:
|
||||
updateDict['alias'] = doc['alias']
|
||||
if 'addr_alias' in doc:
|
||||
updateDict['addr_alias'] = doc['addr_alias']
|
||||
if 'lng' in doc and doc['lng'] is not None and str(doc['lng']) != 'nan':
|
||||
updateDict['location'] = {'lon': doc['lng'] / 10 ** 10, 'lat': doc['lat'] / 10 ** 10}
|
||||
updateDict['update_date'] = update_date
|
||||
action = UpdateOne({'_id': doc['_id']}, {'$set': updateDict}, upsert=True)
|
||||
|
||||
actions.append(action)
|
||||
if len(actions) == 10000:
|
||||
seaweed_didata_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
|
||||
didata_id_df = pd.DataFrame(
|
||||
list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000)))
|
||||
if len(didata_id_df) > 0:
|
||||
didata_id_df = didata_id_df[['_id']]
|
||||
id_df = pd.DataFrame(id_list)
|
||||
deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False)
|
||||
|
||||
for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows():
|
||||
action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])},
|
||||
{'$set': {'status': 1, 'update_date': update_date}}, upsert=False)
|
||||
actions.append(action)
|
||||
if len(actions) == 10000:
|
||||
seaweed_didata_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
|
||||
if len(actions) > 0:
|
||||
seaweed_didata_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
log.info(city + "-" + type + "-同步完成")
|
||||
|
||||
|
||||
if type == 'all':
|
||||
sync_district_office("district")
|
||||
sync_district_office("office")
|
||||
else:
|
||||
sync_district_office(type)
|
||||
# import logging
|
||||
# import sys
|
||||
# from goojia_utils import MAIN_MONGO_CLIENT, DIDATA_MONGO_CLIENT, areautils
|
||||
# from pymongo import UpdateOne
|
||||
# from bson.objectid import ObjectId
|
||||
# import pandas as pd
|
||||
# import time
|
||||
#
|
||||
# log = logging.getLogger(__name__)
|
||||
#
|
||||
# type = sys.argv[1] if len(sys.argv) > 1 else 'all'
|
||||
# log.info("同步类型:" + type)
|
||||
#
|
||||
#
|
||||
# def sync_district_office(type):
|
||||
# if type != 'district' and type != 'office':
|
||||
# return
|
||||
# seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
# seaweed_didata_conn = DIDATA_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
#
|
||||
# cities = areautils.get_all_cities()
|
||||
# update_date = int(round(time.time() * 1000))
|
||||
# for city in cities:
|
||||
# actions = list()
|
||||
# id_list = list()
|
||||
# for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0},
|
||||
# {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1,
|
||||
# 'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1,
|
||||
# 'status': 1}) \
|
||||
# .batch_size(10000):
|
||||
# updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'],
|
||||
# 'status': doc['status']}
|
||||
# if 'estate_type2' in doc:
|
||||
# updateDict['estate_type'] = doc['estate_type2']
|
||||
#
|
||||
# id_list.append({'_id': doc['_id']})
|
||||
# if 'block' in doc:
|
||||
# updateDict['block'] = doc['block']
|
||||
# if 'address' in doc:
|
||||
# updateDict['address'] = doc['address']
|
||||
# if 'township' in doc:
|
||||
# updateDict['township'] = doc['township']
|
||||
# if 'alias' in doc:
|
||||
# updateDict['alias'] = doc['alias']
|
||||
# if 'addr_alias' in doc:
|
||||
# updateDict['addr_alias'] = doc['addr_alias']
|
||||
# if 'lng' in doc and doc['lng'] is not None and str(doc['lng']) != 'nan':
|
||||
# updateDict['location'] = {'lon': doc['lng'] / 10 ** 10, 'lat': doc['lat'] / 10 ** 10}
|
||||
# updateDict['update_date'] = update_date
|
||||
# action = UpdateOne({'_id': doc['_id']}, {'$set': updateDict}, upsert=True)
|
||||
#
|
||||
# actions.append(action)
|
||||
# if len(actions) == 10000:
|
||||
# seaweed_didata_conn.bulk_write(actions)
|
||||
# actions.clear()
|
||||
#
|
||||
# didata_id_df = pd.DataFrame(
|
||||
# list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000)))
|
||||
# if len(didata_id_df) > 0:
|
||||
# didata_id_df = didata_id_df[['_id']]
|
||||
# id_df = pd.DataFrame(id_list)
|
||||
# deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False)
|
||||
#
|
||||
# for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows():
|
||||
# action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])},
|
||||
# {'$set': {'status': 1, 'update_date': update_date}}, upsert=False)
|
||||
# actions.append(action)
|
||||
# if len(actions) == 10000:
|
||||
# seaweed_didata_conn.bulk_write(actions)
|
||||
# actions.clear()
|
||||
#
|
||||
# if len(actions) > 0:
|
||||
# seaweed_didata_conn.bulk_write(actions)
|
||||
# actions.clear()
|
||||
# log.info(city + "-" + type + "-同步完成")
|
||||
#
|
||||
#
|
||||
# if type == 'all':
|
||||
# sync_district_office("district")
|
||||
# sync_district_office("office")
|
||||
# else:
|
||||
# sync_district_office(type)
|
||||
|
||||
@@ -2,16 +2,16 @@ from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="didata_sync_dag",
|
||||
# 每月28日3点整分运行
|
||||
schedule="0 3 28 * *",
|
||||
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="地数相关数据同步type: district或office, 不填默认同步全部"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='didata_sync_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/didata_sync.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t1
|
||||
# with DAG(
|
||||
# dag_id="didata_sync_dag",
|
||||
# # 每月28日3点整分运行
|
||||
# schedule="0 3 28 * *",
|
||||
# start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
|
||||
# catchup=True,
|
||||
# description="地数相关数据同步type: district或office, 不填默认同步全部"
|
||||
# ) as dag:
|
||||
# t1 = BashOperator(
|
||||
# task_id='didata_sync_dag',
|
||||
# bash_command='python /opt/airflow/dags/repo/dags/business/business/didata_sync.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
# )
|
||||
# t1
|
||||
Reference in New Issue
Block a user