From f0d77086273339c02b4955646b65f4678f9f67d8 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 21 Nov 2022 16:37:20 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=AC=AC=E4=B8=89=E6=96=B9?= =?UTF-8?q?=E8=BF=87=E6=BB=A4=E4=BE=9D=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/price/third_price.py | 38 +++--------------------------------- dags/jh/third_price_dag.py | 2 +- requirements.txt | 4 +++- 3 files changed, 7 insertions(+), 37 deletions(-) diff --git a/dags/jh/price/third_price.py b/dags/jh/price/third_price.py index f594838..985b14f 100644 --- a/dags/jh/price/third_price.py +++ b/dags/jh/price/third_price.py @@ -1,52 +1,21 @@ -#!/usr/bin/env python -# coding: utf-8 - -# In[45]: - - -import matplotlib.pyplot as plt from datetime import datetime -from goojia_utils import MAIN_MONGO_CLIENT, LOG_MONGO_CLIENT, data_utils, dateutils -from sklearn import preprocessing -from sklearn.preprocessing import OneHotEncoder +from goojia_utils import MAIN_MONGO_CLIENT, dateutils, oracle from sklearn.cluster import DBSCAN import numpy as np import pandas as pd -import os -from sqlalchemy import create_engine from pykalman import KalmanFilter from pymongo import UpdateOne import transbigdata as tbd from sklearn.neighbors import NearestNeighbors -os.environ["NLS_LANG"] = "AMERICAN_AMERICA.AL32UTF8" -engine = create_engine('oracle+cx_oracle://GOOJIA:GOOJIA@192.168.0.236:1521/orcl?encoding=UTF-8', echo=False) - -# In[2]: - - -get_ipython().run_line_magic('matplotlib', 'inline') - -# In[3]: - - year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-2), '%Y%m')) before_year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), years=-1, months=-1), '%Y%m')) -# In[4]: - - average_statistics_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "average_statistics") -# In[5]: - - average_statistics = pd.DataFrame( list(average_statistics_client.find({"s_date": {"$gt": before_year_month}}).batch_size(50000))) -# In[6]: - - data = list() keys = [key for key in average_statistics.columns if "esf_price" in key] @@ -116,9 +85,8 @@ monthly_data = monthly_data.sort_values(by=["city", "region", "name"]) # In[14]: -community_base_price = pd.read_sql( - f"select commid, year_month, avg_price, calc_type,confidence, cityname from community_base_price_gj where year_month > {before_year_month}", - con=engine) +community_base_price = oracle.scan_by_sql_to_data_frame( + f"select commid, year_month, avg_price, calc_type,confidence, cityname from community_base_price_gj where year_month > {before_year_month}") community_base_price = community_base_price.sort_values(by=['commid', 'year_month'], ascending=[True, True]) # In[15]: diff --git a/dags/jh/third_price_dag.py b/dags/jh/third_price_dag.py index 4404a40..ef4af4b 100644 --- a/dags/jh/third_price_dag.py +++ b/dags/jh/third_price_dag.py @@ -6,7 +6,7 @@ with DAG( dag_id="third_price_dag", # 每月23日运行 schedule_interval="0 16 18 * *", - start_date=pendulum.datetime(2022, 18, 16, tz="Asia/Shanghai"), + start_date=pendulum.datetime(2022, 11, 21, tz="Asia/Shanghai"), catchup=True, description="建行过滤第三方均价" ) as dag: diff --git a/requirements.txt b/requirements.txt index 5847119..42f474e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ nextcloud-api-wrapper elasticsearch==7.11.0 -goojia_utils +transbigdata==0.4.15 +pykalman==0.9.5 +goojia_utils \ No newline at end of file