diff --git a/dags/jh/price/third_price.py b/dags/jh/price/third_price.py index f594838..985b14f 100644 --- a/dags/jh/price/third_price.py +++ b/dags/jh/price/third_price.py @@ -1,52 +1,21 @@ -#!/usr/bin/env python -# coding: utf-8 - -# In[45]: - - -import matplotlib.pyplot as plt from datetime import datetime -from goojia_utils import MAIN_MONGO_CLIENT, LOG_MONGO_CLIENT, data_utils, dateutils -from sklearn import preprocessing -from sklearn.preprocessing import OneHotEncoder +from goojia_utils import MAIN_MONGO_CLIENT, dateutils, oracle from sklearn.cluster import DBSCAN import numpy as np import pandas as pd -import os -from sqlalchemy import create_engine from pykalman import KalmanFilter from pymongo import UpdateOne import transbigdata as tbd from sklearn.neighbors import NearestNeighbors -os.environ["NLS_LANG"] = "AMERICAN_AMERICA.AL32UTF8" -engine = create_engine('oracle+cx_oracle://GOOJIA:GOOJIA@192.168.0.236:1521/orcl?encoding=UTF-8', echo=False) - -# In[2]: - - -get_ipython().run_line_magic('matplotlib', 'inline') - -# In[3]: - - year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-2), '%Y%m')) before_year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), years=-1, months=-1), '%Y%m')) -# In[4]: - - average_statistics_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "average_statistics") -# In[5]: - - average_statistics = pd.DataFrame( list(average_statistics_client.find({"s_date": {"$gt": before_year_month}}).batch_size(50000))) -# In[6]: - - data = list() keys = [key for key in average_statistics.columns if "esf_price" in key] @@ -116,9 +85,8 @@ monthly_data = monthly_data.sort_values(by=["city", "region", "name"]) # In[14]: -community_base_price = pd.read_sql( - f"select commid, year_month, avg_price, calc_type,confidence, cityname from community_base_price_gj where year_month > {before_year_month}", - con=engine) +community_base_price = oracle.scan_by_sql_to_data_frame( + f"select commid, year_month, avg_price, calc_type,confidence, cityname from community_base_price_gj where year_month > {before_year_month}") community_base_price = community_base_price.sort_values(by=['commid', 'year_month'], ascending=[True, True]) # In[15]: diff --git a/requirements.txt b/requirements.txt index 5847119..42f474e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ nextcloud-api-wrapper elasticsearch==7.11.0 -goojia_utils +transbigdata==0.4.15 +pykalman==0.9.5 +goojia_utils \ No newline at end of file