diff --git a/dags/jh/price/third_price.py b/dags/jh/price/third_price.py new file mode 100644 index 0000000..f594838 --- /dev/null +++ b/dags/jh/price/third_price.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[45]: + + +import matplotlib.pyplot as plt +from datetime import datetime +from goojia_utils import MAIN_MONGO_CLIENT, LOG_MONGO_CLIENT, data_utils, dateutils +from sklearn import preprocessing +from sklearn.preprocessing import OneHotEncoder +from sklearn.cluster import DBSCAN +import numpy as np +import pandas as pd +import os +from sqlalchemy import create_engine +from pykalman import KalmanFilter +from pymongo import UpdateOne +import transbigdata as tbd +from sklearn.neighbors import NearestNeighbors + +os.environ["NLS_LANG"] = "AMERICAN_AMERICA.AL32UTF8" +engine = create_engine('oracle+cx_oracle://GOOJIA:GOOJIA@192.168.0.236:1521/orcl?encoding=UTF-8', echo=False) + +# In[2]: + + +get_ipython().run_line_magic('matplotlib', 'inline') + +# In[3]: + + +year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-2), '%Y%m')) +before_year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), years=-1, months=-1), '%Y%m')) + +# In[4]: + + +average_statistics_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "average_statistics") + +# In[5]: + + +average_statistics = pd.DataFrame( + list(average_statistics_client.find({"s_date": {"$gt": before_year_month}}).batch_size(50000))) + +# In[6]: + + +data = list() +keys = [key for key in average_statistics.columns if "esf_price" in key] + + +def assemble(item, data): + for website in keys: + if pd.notna(item[website]): + data.append({"city": item["city"], "region": item["region"], "name": item["name"], "s_date": item["s_date"], + "price": item[website], "source": website, "commid": item["seaweed_id"]}) + + +average_statistics.apply(lambda x: assemble(x, data), axis=1) +data = pd.DataFrame(data) + +# In[7]: + + +monthly_data = data[data.s_date == data.s_date.max()].copy() +monthly_data['region_mean'] = monthly_data.groupby(by=['city', 'region'])["price"].transform('mean').map(int) +monthly_data['region_min'] = monthly_data.groupby(by=['city', 'region'])["price"].transform(lambda x: x.quantile(0.01)) +monthly_data['region_max'] = monthly_data.groupby(by=['city', 'region'])["price"].transform(lambda x: x.quantile(0.99)) + +# In[8]: + + +monthly_data.loc[ + (monthly_data.region_min >= monthly_data.price) | (monthly_data.price >= monthly_data.region_max), "异常第三方"] = 1 + +# In[9]: + + +monthly_data['web_count'] = monthly_data.groupby(by=['city', 'region', "name"])["price"].transform('count') +monthly_data['comm_mean'] = monthly_data.groupby(by=['city', 'region', "name"])["price"].transform('mean').map(int) + +# In[10]: + + +monthly_data.loc[(monthly_data.web_count >= 3) & (monthly_data["异常第三方"].isna()) & ( + (monthly_data.price > monthly_data.comm_mean * 1.4) | ( + (monthly_data.price < monthly_data.comm_mean * 0.6))), "异常第三方"] = 2 + + +# In[11]: + + +def dbscan_denose(x): + db = DBSCAN(eps=x.max() / 10, min_samples=2).fit([[a] for a in x]) + if (db.labels_ == -1).all() or len(x) < 3: + return np.zeros_like(db.labels_) + labels_ = db.labels_ + labels_[np.where(labels_ > -1)] = 0 + return db.labels_ + + +# In[12]: + + +monthly_data["dbscan_denose"] = monthly_data.groupby(by=['city', 'region', 'name'])["price"].transform( + lambda x: dbscan_denose(x)) +monthly_data.loc[(monthly_data["异常第三方"].isna()) & (monthly_data.dbscan_denose == -1), "异常第三方"] = 3 + +# In[13]: + + +monthly_data = monthly_data.sort_values(by=["city", "region", "name"]) + +# In[14]: + + +community_base_price = pd.read_sql( + f"select commid, year_month, avg_price, calc_type,confidence, cityname from community_base_price_gj where year_month > {before_year_month}", + con=engine) +community_base_price = community_base_price.sort_values(by=['commid', 'year_month'], ascending=[True, True]) + +# In[15]: + + +city_prices = dict() +for entry in community_base_price.groupby(by="cityname")["avg_price"].mean().reset_index().to_dict(orient='records'): + city_prices[entry['cityname']] = int(entry['avg_price']) + + +def denose_kalman(stats): + stats = stats.reset_index(drop=True) + delta_t = np.pi * 2 / len(stats["avg_price"]) + observations_data = stats["avg_price"] + var = stats["avg_price"].var() + initial_value_guess = stats[stats.confidence == stats.confidence.max()][['avg_price']][0:1] + if initial_value_guess.empty: + initial_value_guess = stats[['avg_price']][0:1] + transition_matrix = 1 + cityname = stats.iloc[0, :]['cityname'] + if cityname in city_prices: + transition_covariance = city_prices[cityname] * 0.15 + else: + transition_covariance = 100 + kf = KalmanFilter( + initial_state_mean=initial_value_guess, + initial_state_covariance=1, + observation_covariance=1, + transition_covariance=transition_covariance, + transition_matrices=1 + ) + pred_state, state_cov = kf.smooth(observations_data) + return int(pred_state[-1][0]) + + +# In[16]: + + +kalman_stats = list() +i = 0 +for commid, items in community_base_price.groupby(by="commid"): + i = i + 1 + if i % 1000 == 0: + print(i) + kalman_stats.append({"commid": commid, "kalman_price": denose_kalman(items)}) +kalman_stats = pd.DataFrame(kalman_stats) + +# In[17]: + + +monthly_data = monthly_data.merge(kalman_stats, on="commid", how='left') + +# In[18]: + + +monthly_data["kalman_price_ratio"] = monthly_data["price"] / monthly_data["kalman_price"] + +# In[19]: + + +monthly_data.loc[(monthly_data.kalman_price_ratio >= 1.5) | (monthly_data.kalman_price_ratio < 0.5), "异常第三方"] = 4 + +# In[20]: + + +stat = community_base_price[community_base_price.year_month == community_base_price.year_month.max()] + +# In[21]: + + +seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed") +seaweed = pd.DataFrame( + list(seaweed_client.find({"cat": "district", "status": 0}, {"_id": 1, "lat": 1, "lng": 1, "estate_type2": "1"}))) +seaweed["lng"] = seaweed["lng"] / 10000000000.0 +seaweed["lat"] = seaweed["lat"] / 10000000000.0 +seaweed = seaweed.rename(columns={"_id": "commid"}) +seaweed["commid"] = seaweed["commid"].apply(lambda x: str(x)) + +# In[22]: + + +stat = stat.merge(seaweed, on="commid", how="left") + +# In[23]: + + +stat = stat.reset_index(drop=True) +stat = stat.dropna() + +# In[24]: + +# 计算经纬度距离 +def get_distance(x, y): + return tbd.getdistance(x[0], x[1], y[0], y[1]) + + +# In[25]: + + +stat_zhuzhai = stat[stat.estate_type2 == '普通住宅'] +stat_zhuzhai = stat_zhuzhai.reset_index(drop=True) +stat_shangzhu = stat[stat.estate_type2 == '商住'] +stat_shangzhu = stat_shangzhu.reset_index(drop=True) + +# In[26]: + + +monthly_data = monthly_data.merge(stat[['lng', 'lat', 'estate_type2', 'commid']], on='commid', how='left') + +# In[27]: + + +zhuzhai_model = NearestNeighbors(metric=get_distance, radius=1000) +zhuzhai_model.fit(stat_zhuzhai[['lng', 'lat']].values) +shangzhu_model = NearestNeighbors(metric=get_distance, radius=1000) +shangzhu_model.fit(stat_shangzhu[['lng', 'lat']].values) + + +# In[28]: + + +def calc(item): + if pd.isna(item['lng']): + return None + if item["estate_type2"] == "商住": + distances, indices = shangzhu_model.radius_neighbors(X=[[item['lng'], item['lat']]]) + pois = stat_shangzhu.iloc[indices[0]].copy() + return pois["avg_price"].mean() + else: + distances, indices = zhuzhai_model.radius_neighbors(X=[[item['lng'], item['lat']]]) + pois = stat_zhuzhai.iloc[indices[0]].copy() + return pois["avg_price"].mean() + + +# In[29]: + + +monthly_data['around_price'] = monthly_data.apply(lambda x: calc(x), axis=1) + +# In[30]: + + +monthly_data["around_price_ratio"] = monthly_data["price"] / monthly_data["around_price"] +monthly_data.loc[(monthly_data.around_price_ratio >= 1.75) | (monthly_data.kalman_price_ratio < 0.5), "异常第三方"] = 5 + +# In[ ]: + + +# 去除诸葛找房独家数据 + + +# In[ ]: + + +monthly_data.loc[(monthly_data["异常第三方"].isna()) & (monthly_data.web_count == 1) & ( + monthly_data.source == "zhugezhaofang_esf_price"), "异常第三方"] = 6 + +# In[63]: + + +print(len(monthly_data[monthly_data["异常第三方"] == 1])) +print(len(monthly_data[monthly_data["异常第三方"] == 2])) +print(len(monthly_data[monthly_data["异常第三方"] == 3])) +print(len(monthly_data[monthly_data["异常第三方"] == 4])) +print(len(monthly_data[monthly_data["异常第三方"] == 5])) +print(len(monthly_data[monthly_data["异常第三方"] == 6])) +print(len(monthly_data[~monthly_data["异常第三方"].isna()])) +monthly_data + +# In[39]: + + +# monthly_data[monthly_data["异常第三方"]==3].to_excel("异常第三方-3.xlsx") +commids = monthly_data[~monthly_data["异常第三方"].isna()].groupby(by=['commid']).count() +print(len(commids)) + +# In[40]: + + +monthly_data[monthly_data.around_price_ratio < 2]["around_price_ratio"].hist(bins=100) + +# In[41]: + + +monthly_data["around_price"] = monthly_data["around_price"].apply(lambda x: int(x) if pd.notna(x) else x) + +# In[68]: + + +data = monthly_data[~monthly_data["异常第三方"].isna()] +# data = monthly_data[monthly_data["异常第三方"] == 6] + + +# In[69]: + + +print(len(data)) + +# In[67]: +actions = [] +for i, d in data.iterrows(): + print(i) + # average_statistics_client.update_one({'s_date':d['s_date'],'city':d['city'],'region':d['region'], 'name':d['name']},{'$set':{d['source']:None}}) + action = UpdateOne({'s_date': d['s_date'], 'city': d['city'], 'region': d['region'], 'name': d['name']}, + {'$set': {d['source']: None}}) + actions.append(action) + +average_statistics_client.bulk_write(actions) +# rets1=monthly_data[(monthly_data["异常第三方"].isna()) & (monthly_data.web_count==1) & (monthly_data.source=="zhugezhaofang_esf_price")] + + + + + diff --git a/dags/jh/third_price_dag.py b/dags/jh/third_price_dag.py new file mode 100644 index 0000000..4404a40 --- /dev/null +++ b/dags/jh/third_price_dag.py @@ -0,0 +1,17 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + dag_id="third_price_dag", + # 每月23日运行 + schedule_interval="0 16 18 * *", + start_date=pendulum.datetime(2022, 18, 16, tz="Asia/Shanghai"), + catchup=True, + description="建行过滤第三方均价" +) as dag: + t1 = BashOperator( + task_id='third_price', + bash_command='python /opt/airflow/dags/repo/dags/jh/price/third_price.py', + ) + t1 \ No newline at end of file