新增新房变二手房脚本

This commit is contained in:
shenjian@fangjia.com
2023-02-10 17:29:40 +08:00
parent f435f5c5b0
commit daf0eac325
2 changed files with 87 additions and 0 deletions
@@ -0,0 +1,70 @@
"""
新房变二手房定时脚本
创建年代、入住/交付时间三年后,或者有样本就变成二手房
"""
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
from pymongo import UpdateOne
import pandas as pd
import logging
import datetime
import time
log = logging.getLogger(__name__)
def do():
log.info("开始执行新房变二手房脚本")
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'is_new': 1},
{'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'open_date': 1, 'live_date': 1,
'create_year': 1}).batch_size(100000)))
if len(df) <= 0:
log.info("执行新房变二手房脚本完毕,未发现新房")
return False
actions = list()
count = 0
def deal(x):
global count
is_old = False
time_now = int(time.mktime(time.strptime(datetime.datetime.now().strftime('%Y-%m-%d'), '%Y-%m-%d')))
if 'open_date' in x and pd.notna(x['open_date']):
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(x['open_date'], years=3), '%Y-%m-%d'), '%Y-%m-%d')))
if time_old < time_now:
is_old = True
if is_old is False and 'live_date' in x and pd.notna(x['live_date']):
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['live_date']), years=3), '%Y-%m-%d'), '%Y-%m-%d')))
if time_old < time_now:
is_old = True
if is_old is False and 'create_year' in x and pd.notna(x['create_year']):
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['create_year'], '%Y-%m'), years=3), '%Y-%m-%d'), '%Y-%m-%d')))
if time_old < time_now:
is_old = True
# 查询样本库
names = [x['name']]
if type(x['alias']) == list:
for alias_item in x['alias']:
names.append(alias_item)
names_str = "','".join(names)
names_str = "'" + names_str + "'"
if is_old is False:
nums = commutils.get_sample(x['city'], x['region'], names_str)
if nums > 0:
is_old = True
if is_old:
actions.append(UpdateOne({'_id': x['_id']}, {'$set': {'is_new': 0, 'm_date': datetime.datetime.now()}}))
if len(actions) == 5000:
seaweed_conn.bulk_write(actions)
count += len(actions)
actions.clear()
df.apply(lambda x: deal(x), axis=1)
if len(actions) > 0:
seaweed_conn.bulk_write(actions)
count += len(actions)
actions.clear()
log.info(f"执行新房变二手房脚本完毕,共更新{count}")
do()
@@ -0,0 +1,17 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
"change_new_to_old_seaweed_dag",
# 每周六1点运行
schedule_interval="0 1 * * 6",
start_date=pendulum.datetime(2023, 2, 10, tz="Asia/Shanghai"),
catchup=True,
description="新房变二手房定时脚本"
) as dag:
t1 = BashOperator(
task_id='change_new_to_old_seaweed_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/change_new_to_old_seaweed.py',
)
t1