新增新房变二手房脚本
This commit is contained in:
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
新房变二手房定时脚本
|
||||
创建年代、入住/交付时间三年后,或者有样本就变成二手房
|
||||
"""
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
|
||||
from pymongo import UpdateOne
|
||||
import pandas as pd
|
||||
import logging
|
||||
import datetime
|
||||
import time
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do():
|
||||
log.info("开始执行新房变二手房脚本")
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'is_new': 1},
|
||||
{'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'open_date': 1, 'live_date': 1,
|
||||
'create_year': 1}).batch_size(100000)))
|
||||
if len(df) <= 0:
|
||||
log.info("执行新房变二手房脚本完毕,未发现新房")
|
||||
return False
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
def deal(x):
|
||||
global count
|
||||
is_old = False
|
||||
time_now = int(time.mktime(time.strptime(datetime.datetime.now().strftime('%Y-%m-%d'), '%Y-%m-%d')))
|
||||
if 'open_date' in x and pd.notna(x['open_date']):
|
||||
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(x['open_date'], years=3), '%Y-%m-%d'), '%Y-%m-%d')))
|
||||
if time_old < time_now:
|
||||
is_old = True
|
||||
if is_old is False and 'live_date' in x and pd.notna(x['live_date']):
|
||||
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['live_date']), years=3), '%Y-%m-%d'), '%Y-%m-%d')))
|
||||
if time_old < time_now:
|
||||
is_old = True
|
||||
if is_old is False and 'create_year' in x and pd.notna(x['create_year']):
|
||||
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['create_year'], '%Y-%m'), years=3), '%Y-%m-%d'), '%Y-%m-%d')))
|
||||
if time_old < time_now:
|
||||
is_old = True
|
||||
# 查询样本库
|
||||
names = [x['name']]
|
||||
if type(x['alias']) == list:
|
||||
for alias_item in x['alias']:
|
||||
names.append(alias_item)
|
||||
names_str = "','".join(names)
|
||||
names_str = "'" + names_str + "'"
|
||||
if is_old is False:
|
||||
nums = commutils.get_sample(x['city'], x['region'], names_str)
|
||||
if nums > 0:
|
||||
is_old = True
|
||||
if is_old:
|
||||
actions.append(UpdateOne({'_id': x['_id']}, {'$set': {'is_new': 0, 'm_date': datetime.datetime.now()}}))
|
||||
if len(actions) == 5000:
|
||||
seaweed_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
|
||||
df.apply(lambda x: deal(x), axis=1)
|
||||
if len(actions) > 0:
|
||||
seaweed_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info(f"执行新房变二手房脚本完毕,共更新{count}")
|
||||
|
||||
|
||||
do()
|
||||
@@ -0,0 +1,17 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
"change_new_to_old_seaweed_dag",
|
||||
# 每周六1点运行
|
||||
schedule_interval="0 1 * * 6",
|
||||
start_date=pendulum.datetime(2023, 2, 10, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="新房变二手房定时脚本"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='change_new_to_old_seaweed_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/change_new_to_old_seaweed.py',
|
||||
)
|
||||
t1
|
||||
Reference in New Issue
Block a user