From daf0eac325c86d46bb0eaf2b064e6fd05c18a0e0 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 10 Feb 2023 17:29:40 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=96=B0=E6=88=BF=E5=8F=98?= =?UTF-8?q?=E4=BA=8C=E6=89=8B=E6=88=BF=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/change_new_to_old_seaweed.py | 70 +++++++++++++++++++ .../business/change_new_to_old_seaweed_dag.py | 17 +++++ 2 files changed, 87 insertions(+) create mode 100644 dags/business/business/change_new_to_old_seaweed.py create mode 100644 dags/business/change_new_to_old_seaweed_dag.py diff --git a/dags/business/business/change_new_to_old_seaweed.py b/dags/business/business/change_new_to_old_seaweed.py new file mode 100644 index 0000000..14e6e21 --- /dev/null +++ b/dags/business/business/change_new_to_old_seaweed.py @@ -0,0 +1,70 @@ +""" +新房变二手房定时脚本 +创建年代、入住/交付时间三年后,或者有样本就变成二手房 +""" +from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils +from pymongo import UpdateOne +import pandas as pd +import logging +import datetime +import time + +log = logging.getLogger(__name__) + + +def do(): + log.info("开始执行新房变二手房脚本") + seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') + df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'is_new': 1}, + {'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'open_date': 1, 'live_date': 1, + 'create_year': 1}).batch_size(100000))) + if len(df) <= 0: + log.info("执行新房变二手房脚本完毕,未发现新房") + return False + + actions = list() + count = 0 + + def deal(x): + global count + is_old = False + time_now = int(time.mktime(time.strptime(datetime.datetime.now().strftime('%Y-%m-%d'), '%Y-%m-%d'))) + if 'open_date' in x and pd.notna(x['open_date']): + time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(x['open_date'], years=3), '%Y-%m-%d'), '%Y-%m-%d'))) + if time_old < time_now: + is_old = True + if is_old is False and 'live_date' in x and pd.notna(x['live_date']): + time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['live_date']), years=3), '%Y-%m-%d'), '%Y-%m-%d'))) + if time_old < time_now: + is_old = True + if is_old is False and 'create_year' in x and pd.notna(x['create_year']): + time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['create_year'], '%Y-%m'), years=3), '%Y-%m-%d'), '%Y-%m-%d'))) + if time_old < time_now: + is_old = True + # 查询样本库 + names = [x['name']] + if type(x['alias']) == list: + for alias_item in x['alias']: + names.append(alias_item) + names_str = "','".join(names) + names_str = "'" + names_str + "'" + if is_old is False: + nums = commutils.get_sample(x['city'], x['region'], names_str) + if nums > 0: + is_old = True + if is_old: + actions.append(UpdateOne({'_id': x['_id']}, {'$set': {'is_new': 0, 'm_date': datetime.datetime.now()}})) + if len(actions) == 5000: + seaweed_conn.bulk_write(actions) + count += len(actions) + actions.clear() + + df.apply(lambda x: deal(x), axis=1) + if len(actions) > 0: + seaweed_conn.bulk_write(actions) + count += len(actions) + actions.clear() + log.info(f"执行新房变二手房脚本完毕,共更新{count}") + + +do() diff --git a/dags/business/change_new_to_old_seaweed_dag.py b/dags/business/change_new_to_old_seaweed_dag.py new file mode 100644 index 0000000..f9ee05a --- /dev/null +++ b/dags/business/change_new_to_old_seaweed_dag.py @@ -0,0 +1,17 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + "change_new_to_old_seaweed_dag", + # 每周六1点运行 + schedule_interval="0 1 * * 6", + start_date=pendulum.datetime(2023, 2, 10, tz="Asia/Shanghai"), + catchup=True, + description="新房变二手房定时脚本" +) as dag: + t1 = BashOperator( + task_id='change_new_to_old_seaweed_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/change_new_to_old_seaweed.py', + ) + t1 \ No newline at end of file