Merge branch 'master' of http://gitlab.fangjia.com/dev0/airflow
This commit is contained in:
@@ -1,52 +1,21 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[45]:
|
||||
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
from datetime import datetime
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, LOG_MONGO_CLIENT, data_utils, dateutils
|
||||
from sklearn import preprocessing
|
||||
from sklearn.preprocessing import OneHotEncoder
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, oracle
|
||||
from sklearn.cluster import DBSCAN
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import os
|
||||
from sqlalchemy import create_engine
|
||||
from pykalman import KalmanFilter
|
||||
from pymongo import UpdateOne
|
||||
import transbigdata as tbd
|
||||
from sklearn.neighbors import NearestNeighbors
|
||||
|
||||
os.environ["NLS_LANG"] = "AMERICAN_AMERICA.AL32UTF8"
|
||||
engine = create_engine('oracle+cx_oracle://GOOJIA:GOOJIA@192.168.0.236:1521/orcl?encoding=UTF-8', echo=False)
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
get_ipython().run_line_magic('matplotlib', 'inline')
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-2), '%Y%m'))
|
||||
before_year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), years=-1, months=-1), '%Y%m'))
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
average_statistics_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "average_statistics")
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
average_statistics = pd.DataFrame(
|
||||
list(average_statistics_client.find({"s_date": {"$gt": before_year_month}}).batch_size(50000)))
|
||||
|
||||
# In[6]:
|
||||
|
||||
|
||||
data = list()
|
||||
keys = [key for key in average_statistics.columns if "esf_price" in key]
|
||||
|
||||
@@ -116,9 +85,8 @@ monthly_data = monthly_data.sort_values(by=["city", "region", "name"])
|
||||
# In[14]:
|
||||
|
||||
|
||||
community_base_price = pd.read_sql(
|
||||
f"select commid, year_month, avg_price, calc_type,confidence, cityname from community_base_price_gj where year_month > {before_year_month}",
|
||||
con=engine)
|
||||
community_base_price = oracle.scan_by_sql_to_data_frame(
|
||||
f"select commid, year_month, avg_price, calc_type,confidence, cityname from community_base_price_gj where year_month > {before_year_month}")
|
||||
community_base_price = community_base_price.sort_values(by=['commid', 'year_month'], ascending=[True, True])
|
||||
|
||||
# In[15]:
|
||||
|
||||
+3
-1
@@ -1,3 +1,5 @@
|
||||
nextcloud-api-wrapper
|
||||
elasticsearch==7.11.0
|
||||
goojia_utils
|
||||
transbigdata==0.4.15
|
||||
pykalman==0.9.5
|
||||
goojia_utils
|
||||
Reference in New Issue
Block a user