新增第三方过滤
This commit is contained in:
@@ -0,0 +1,335 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[45]:
|
||||
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
from datetime import datetime
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, LOG_MONGO_CLIENT, data_utils, dateutils
|
||||
from sklearn import preprocessing
|
||||
from sklearn.preprocessing import OneHotEncoder
|
||||
from sklearn.cluster import DBSCAN
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import os
|
||||
from sqlalchemy import create_engine
|
||||
from pykalman import KalmanFilter
|
||||
from pymongo import UpdateOne
|
||||
import transbigdata as tbd
|
||||
from sklearn.neighbors import NearestNeighbors
|
||||
|
||||
os.environ["NLS_LANG"] = "AMERICAN_AMERICA.AL32UTF8"
|
||||
engine = create_engine('oracle+cx_oracle://GOOJIA:GOOJIA@192.168.0.236:1521/orcl?encoding=UTF-8', echo=False)
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
get_ipython().run_line_magic('matplotlib', 'inline')
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-2), '%Y%m'))
|
||||
before_year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), years=-1, months=-1), '%Y%m'))
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
average_statistics_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "average_statistics")
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
average_statistics = pd.DataFrame(
|
||||
list(average_statistics_client.find({"s_date": {"$gt": before_year_month}}).batch_size(50000)))
|
||||
|
||||
# In[6]:
|
||||
|
||||
|
||||
data = list()
|
||||
keys = [key for key in average_statistics.columns if "esf_price" in key]
|
||||
|
||||
|
||||
def assemble(item, data):
|
||||
for website in keys:
|
||||
if pd.notna(item[website]):
|
||||
data.append({"city": item["city"], "region": item["region"], "name": item["name"], "s_date": item["s_date"],
|
||||
"price": item[website], "source": website, "commid": item["seaweed_id"]})
|
||||
|
||||
|
||||
average_statistics.apply(lambda x: assemble(x, data), axis=1)
|
||||
data = pd.DataFrame(data)
|
||||
|
||||
# In[7]:
|
||||
|
||||
|
||||
monthly_data = data[data.s_date == data.s_date.max()].copy()
|
||||
monthly_data['region_mean'] = monthly_data.groupby(by=['city', 'region'])["price"].transform('mean').map(int)
|
||||
monthly_data['region_min'] = monthly_data.groupby(by=['city', 'region'])["price"].transform(lambda x: x.quantile(0.01))
|
||||
monthly_data['region_max'] = monthly_data.groupby(by=['city', 'region'])["price"].transform(lambda x: x.quantile(0.99))
|
||||
|
||||
# In[8]:
|
||||
|
||||
|
||||
monthly_data.loc[
|
||||
(monthly_data.region_min >= monthly_data.price) | (monthly_data.price >= monthly_data.region_max), "异常第三方"] = 1
|
||||
|
||||
# In[9]:
|
||||
|
||||
|
||||
monthly_data['web_count'] = monthly_data.groupby(by=['city', 'region', "name"])["price"].transform('count')
|
||||
monthly_data['comm_mean'] = monthly_data.groupby(by=['city', 'region', "name"])["price"].transform('mean').map(int)
|
||||
|
||||
# In[10]:
|
||||
|
||||
|
||||
monthly_data.loc[(monthly_data.web_count >= 3) & (monthly_data["异常第三方"].isna()) & (
|
||||
(monthly_data.price > monthly_data.comm_mean * 1.4) | (
|
||||
(monthly_data.price < monthly_data.comm_mean * 0.6))), "异常第三方"] = 2
|
||||
|
||||
|
||||
# In[11]:
|
||||
|
||||
|
||||
def dbscan_denose(x):
|
||||
db = DBSCAN(eps=x.max() / 10, min_samples=2).fit([[a] for a in x])
|
||||
if (db.labels_ == -1).all() or len(x) < 3:
|
||||
return np.zeros_like(db.labels_)
|
||||
labels_ = db.labels_
|
||||
labels_[np.where(labels_ > -1)] = 0
|
||||
return db.labels_
|
||||
|
||||
|
||||
# In[12]:
|
||||
|
||||
|
||||
monthly_data["dbscan_denose"] = monthly_data.groupby(by=['city', 'region', 'name'])["price"].transform(
|
||||
lambda x: dbscan_denose(x))
|
||||
monthly_data.loc[(monthly_data["异常第三方"].isna()) & (monthly_data.dbscan_denose == -1), "异常第三方"] = 3
|
||||
|
||||
# In[13]:
|
||||
|
||||
|
||||
monthly_data = monthly_data.sort_values(by=["city", "region", "name"])
|
||||
|
||||
# In[14]:
|
||||
|
||||
|
||||
community_base_price = pd.read_sql(
|
||||
f"select commid, year_month, avg_price, calc_type,confidence, cityname from community_base_price_gj where year_month > {before_year_month}",
|
||||
con=engine)
|
||||
community_base_price = community_base_price.sort_values(by=['commid', 'year_month'], ascending=[True, True])
|
||||
|
||||
# In[15]:
|
||||
|
||||
|
||||
city_prices = dict()
|
||||
for entry in community_base_price.groupby(by="cityname")["avg_price"].mean().reset_index().to_dict(orient='records'):
|
||||
city_prices[entry['cityname']] = int(entry['avg_price'])
|
||||
|
||||
|
||||
def denose_kalman(stats):
|
||||
stats = stats.reset_index(drop=True)
|
||||
delta_t = np.pi * 2 / len(stats["avg_price"])
|
||||
observations_data = stats["avg_price"]
|
||||
var = stats["avg_price"].var()
|
||||
initial_value_guess = stats[stats.confidence == stats.confidence.max()][['avg_price']][0:1]
|
||||
if initial_value_guess.empty:
|
||||
initial_value_guess = stats[['avg_price']][0:1]
|
||||
transition_matrix = 1
|
||||
cityname = stats.iloc[0, :]['cityname']
|
||||
if cityname in city_prices:
|
||||
transition_covariance = city_prices[cityname] * 0.15
|
||||
else:
|
||||
transition_covariance = 100
|
||||
kf = KalmanFilter(
|
||||
initial_state_mean=initial_value_guess,
|
||||
initial_state_covariance=1,
|
||||
observation_covariance=1,
|
||||
transition_covariance=transition_covariance,
|
||||
transition_matrices=1
|
||||
)
|
||||
pred_state, state_cov = kf.smooth(observations_data)
|
||||
return int(pred_state[-1][0])
|
||||
|
||||
|
||||
# In[16]:
|
||||
|
||||
|
||||
kalman_stats = list()
|
||||
i = 0
|
||||
for commid, items in community_base_price.groupby(by="commid"):
|
||||
i = i + 1
|
||||
if i % 1000 == 0:
|
||||
print(i)
|
||||
kalman_stats.append({"commid": commid, "kalman_price": denose_kalman(items)})
|
||||
kalman_stats = pd.DataFrame(kalman_stats)
|
||||
|
||||
# In[17]:
|
||||
|
||||
|
||||
monthly_data = monthly_data.merge(kalman_stats, on="commid", how='left')
|
||||
|
||||
# In[18]:
|
||||
|
||||
|
||||
monthly_data["kalman_price_ratio"] = monthly_data["price"] / monthly_data["kalman_price"]
|
||||
|
||||
# In[19]:
|
||||
|
||||
|
||||
monthly_data.loc[(monthly_data.kalman_price_ratio >= 1.5) | (monthly_data.kalman_price_ratio < 0.5), "异常第三方"] = 4
|
||||
|
||||
# In[20]:
|
||||
|
||||
|
||||
stat = community_base_price[community_base_price.year_month == community_base_price.year_month.max()]
|
||||
|
||||
# In[21]:
|
||||
|
||||
|
||||
seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed")
|
||||
seaweed = pd.DataFrame(
|
||||
list(seaweed_client.find({"cat": "district", "status": 0}, {"_id": 1, "lat": 1, "lng": 1, "estate_type2": "1"})))
|
||||
seaweed["lng"] = seaweed["lng"] / 10000000000.0
|
||||
seaweed["lat"] = seaweed["lat"] / 10000000000.0
|
||||
seaweed = seaweed.rename(columns={"_id": "commid"})
|
||||
seaweed["commid"] = seaweed["commid"].apply(lambda x: str(x))
|
||||
|
||||
# In[22]:
|
||||
|
||||
|
||||
stat = stat.merge(seaweed, on="commid", how="left")
|
||||
|
||||
# In[23]:
|
||||
|
||||
|
||||
stat = stat.reset_index(drop=True)
|
||||
stat = stat.dropna()
|
||||
|
||||
# In[24]:
|
||||
|
||||
# 计算经纬度距离
|
||||
def get_distance(x, y):
|
||||
return tbd.getdistance(x[0], x[1], y[0], y[1])
|
||||
|
||||
|
||||
# In[25]:
|
||||
|
||||
|
||||
stat_zhuzhai = stat[stat.estate_type2 == '普通住宅']
|
||||
stat_zhuzhai = stat_zhuzhai.reset_index(drop=True)
|
||||
stat_shangzhu = stat[stat.estate_type2 == '商住']
|
||||
stat_shangzhu = stat_shangzhu.reset_index(drop=True)
|
||||
|
||||
# In[26]:
|
||||
|
||||
|
||||
monthly_data = monthly_data.merge(stat[['lng', 'lat', 'estate_type2', 'commid']], on='commid', how='left')
|
||||
|
||||
# In[27]:
|
||||
|
||||
|
||||
zhuzhai_model = NearestNeighbors(metric=get_distance, radius=1000)
|
||||
zhuzhai_model.fit(stat_zhuzhai[['lng', 'lat']].values)
|
||||
shangzhu_model = NearestNeighbors(metric=get_distance, radius=1000)
|
||||
shangzhu_model.fit(stat_shangzhu[['lng', 'lat']].values)
|
||||
|
||||
|
||||
# In[28]:
|
||||
|
||||
|
||||
def calc(item):
|
||||
if pd.isna(item['lng']):
|
||||
return None
|
||||
if item["estate_type2"] == "商住":
|
||||
distances, indices = shangzhu_model.radius_neighbors(X=[[item['lng'], item['lat']]])
|
||||
pois = stat_shangzhu.iloc[indices[0]].copy()
|
||||
return pois["avg_price"].mean()
|
||||
else:
|
||||
distances, indices = zhuzhai_model.radius_neighbors(X=[[item['lng'], item['lat']]])
|
||||
pois = stat_zhuzhai.iloc[indices[0]].copy()
|
||||
return pois["avg_price"].mean()
|
||||
|
||||
|
||||
# In[29]:
|
||||
|
||||
|
||||
monthly_data['around_price'] = monthly_data.apply(lambda x: calc(x), axis=1)
|
||||
|
||||
# In[30]:
|
||||
|
||||
|
||||
monthly_data["around_price_ratio"] = monthly_data["price"] / monthly_data["around_price"]
|
||||
monthly_data.loc[(monthly_data.around_price_ratio >= 1.75) | (monthly_data.kalman_price_ratio < 0.5), "异常第三方"] = 5
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
# 去除诸葛找房独家数据
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
monthly_data.loc[(monthly_data["异常第三方"].isna()) & (monthly_data.web_count == 1) & (
|
||||
monthly_data.source == "zhugezhaofang_esf_price"), "异常第三方"] = 6
|
||||
|
||||
# In[63]:
|
||||
|
||||
|
||||
print(len(monthly_data[monthly_data["异常第三方"] == 1]))
|
||||
print(len(monthly_data[monthly_data["异常第三方"] == 2]))
|
||||
print(len(monthly_data[monthly_data["异常第三方"] == 3]))
|
||||
print(len(monthly_data[monthly_data["异常第三方"] == 4]))
|
||||
print(len(monthly_data[monthly_data["异常第三方"] == 5]))
|
||||
print(len(monthly_data[monthly_data["异常第三方"] == 6]))
|
||||
print(len(monthly_data[~monthly_data["异常第三方"].isna()]))
|
||||
monthly_data
|
||||
|
||||
# In[39]:
|
||||
|
||||
|
||||
# monthly_data[monthly_data["异常第三方"]==3].to_excel("异常第三方-3.xlsx")
|
||||
commids = monthly_data[~monthly_data["异常第三方"].isna()].groupby(by=['commid']).count()
|
||||
print(len(commids))
|
||||
|
||||
# In[40]:
|
||||
|
||||
|
||||
monthly_data[monthly_data.around_price_ratio < 2]["around_price_ratio"].hist(bins=100)
|
||||
|
||||
# In[41]:
|
||||
|
||||
|
||||
monthly_data["around_price"] = monthly_data["around_price"].apply(lambda x: int(x) if pd.notna(x) else x)
|
||||
|
||||
# In[68]:
|
||||
|
||||
|
||||
data = monthly_data[~monthly_data["异常第三方"].isna()]
|
||||
# data = monthly_data[monthly_data["异常第三方"] == 6]
|
||||
|
||||
|
||||
# In[69]:
|
||||
|
||||
|
||||
print(len(data))
|
||||
|
||||
# In[67]:
|
||||
actions = []
|
||||
for i, d in data.iterrows():
|
||||
print(i)
|
||||
# average_statistics_client.update_one({'s_date':d['s_date'],'city':d['city'],'region':d['region'], 'name':d['name']},{'$set':{d['source']:None}})
|
||||
action = UpdateOne({'s_date': d['s_date'], 'city': d['city'], 'region': d['region'], 'name': d['name']},
|
||||
{'$set': {d['source']: None}})
|
||||
actions.append(action)
|
||||
|
||||
average_statistics_client.bulk_write(actions)
|
||||
# rets1=monthly_data[(monthly_data["异常第三方"].isna()) & (monthly_data.web_count==1) & (monthly_data.source=="zhugezhaofang_esf_price")]
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="third_price_dag",
|
||||
# 每月23日运行
|
||||
schedule_interval="0 16 18 * *",
|
||||
start_date=pendulum.datetime(2022, 18, 16, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="建行过滤第三方均价"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='third_price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/jh/price/third_price.py',
|
||||
)
|
||||
t1
|
||||
Reference in New Issue
Block a user