新增第三方过滤

This commit is contained in:
2022-11-21 16:12:05 +08:00
parent 0baa286567
commit fad0729bf3
2 changed files with 352 additions and 0 deletions
+335
View File
@@ -0,0 +1,335 @@
#!/usr/bin/env python
# coding: utf-8
# In[45]:
import matplotlib.pyplot as plt
from datetime import datetime
from goojia_utils import MAIN_MONGO_CLIENT, LOG_MONGO_CLIENT, data_utils, dateutils
from sklearn import preprocessing
from sklearn.preprocessing import OneHotEncoder
from sklearn.cluster import DBSCAN
import numpy as np
import pandas as pd
import os
from sqlalchemy import create_engine
from pykalman import KalmanFilter
from pymongo import UpdateOne
import transbigdata as tbd
from sklearn.neighbors import NearestNeighbors
os.environ["NLS_LANG"] = "AMERICAN_AMERICA.AL32UTF8"
engine = create_engine('oracle+cx_oracle://GOOJIA:GOOJIA@192.168.0.236:1521/orcl?encoding=UTF-8', echo=False)
# In[2]:
get_ipython().run_line_magic('matplotlib', 'inline')
# In[3]:
year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-2), '%Y%m'))
before_year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), years=-1, months=-1), '%Y%m'))
# In[4]:
average_statistics_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "average_statistics")
# In[5]:
average_statistics = pd.DataFrame(
list(average_statistics_client.find({"s_date": {"$gt": before_year_month}}).batch_size(50000)))
# In[6]:
data = list()
keys = [key for key in average_statistics.columns if "esf_price" in key]
def assemble(item, data):
for website in keys:
if pd.notna(item[website]):
data.append({"city": item["city"], "region": item["region"], "name": item["name"], "s_date": item["s_date"],
"price": item[website], "source": website, "commid": item["seaweed_id"]})
average_statistics.apply(lambda x: assemble(x, data), axis=1)
data = pd.DataFrame(data)
# In[7]:
monthly_data = data[data.s_date == data.s_date.max()].copy()
monthly_data['region_mean'] = monthly_data.groupby(by=['city', 'region'])["price"].transform('mean').map(int)
monthly_data['region_min'] = monthly_data.groupby(by=['city', 'region'])["price"].transform(lambda x: x.quantile(0.01))
monthly_data['region_max'] = monthly_data.groupby(by=['city', 'region'])["price"].transform(lambda x: x.quantile(0.99))
# In[8]:
monthly_data.loc[
(monthly_data.region_min >= monthly_data.price) | (monthly_data.price >= monthly_data.region_max), "异常第三方"] = 1
# In[9]:
monthly_data['web_count'] = monthly_data.groupby(by=['city', 'region', "name"])["price"].transform('count')
monthly_data['comm_mean'] = monthly_data.groupby(by=['city', 'region', "name"])["price"].transform('mean').map(int)
# In[10]:
monthly_data.loc[(monthly_data.web_count >= 3) & (monthly_data["异常第三方"].isna()) & (
(monthly_data.price > monthly_data.comm_mean * 1.4) | (
(monthly_data.price < monthly_data.comm_mean * 0.6))), "异常第三方"] = 2
# In[11]:
def dbscan_denose(x):
db = DBSCAN(eps=x.max() / 10, min_samples=2).fit([[a] for a in x])
if (db.labels_ == -1).all() or len(x) < 3:
return np.zeros_like(db.labels_)
labels_ = db.labels_
labels_[np.where(labels_ > -1)] = 0
return db.labels_
# In[12]:
monthly_data["dbscan_denose"] = monthly_data.groupby(by=['city', 'region', 'name'])["price"].transform(
lambda x: dbscan_denose(x))
monthly_data.loc[(monthly_data["异常第三方"].isna()) & (monthly_data.dbscan_denose == -1), "异常第三方"] = 3
# In[13]:
monthly_data = monthly_data.sort_values(by=["city", "region", "name"])
# In[14]:
community_base_price = pd.read_sql(
f"select commid, year_month, avg_price, calc_type,confidence, cityname from community_base_price_gj where year_month > {before_year_month}",
con=engine)
community_base_price = community_base_price.sort_values(by=['commid', 'year_month'], ascending=[True, True])
# In[15]:
city_prices = dict()
for entry in community_base_price.groupby(by="cityname")["avg_price"].mean().reset_index().to_dict(orient='records'):
city_prices[entry['cityname']] = int(entry['avg_price'])
def denose_kalman(stats):
stats = stats.reset_index(drop=True)
delta_t = np.pi * 2 / len(stats["avg_price"])
observations_data = stats["avg_price"]
var = stats["avg_price"].var()
initial_value_guess = stats[stats.confidence == stats.confidence.max()][['avg_price']][0:1]
if initial_value_guess.empty:
initial_value_guess = stats[['avg_price']][0:1]
transition_matrix = 1
cityname = stats.iloc[0, :]['cityname']
if cityname in city_prices:
transition_covariance = city_prices[cityname] * 0.15
else:
transition_covariance = 100
kf = KalmanFilter(
initial_state_mean=initial_value_guess,
initial_state_covariance=1,
observation_covariance=1,
transition_covariance=transition_covariance,
transition_matrices=1
)
pred_state, state_cov = kf.smooth(observations_data)
return int(pred_state[-1][0])
# In[16]:
kalman_stats = list()
i = 0
for commid, items in community_base_price.groupby(by="commid"):
i = i + 1
if i % 1000 == 0:
print(i)
kalman_stats.append({"commid": commid, "kalman_price": denose_kalman(items)})
kalman_stats = pd.DataFrame(kalman_stats)
# In[17]:
monthly_data = monthly_data.merge(kalman_stats, on="commid", how='left')
# In[18]:
monthly_data["kalman_price_ratio"] = monthly_data["price"] / monthly_data["kalman_price"]
# In[19]:
monthly_data.loc[(monthly_data.kalman_price_ratio >= 1.5) | (monthly_data.kalman_price_ratio < 0.5), "异常第三方"] = 4
# In[20]:
stat = community_base_price[community_base_price.year_month == community_base_price.year_month.max()]
# In[21]:
seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed")
seaweed = pd.DataFrame(
list(seaweed_client.find({"cat": "district", "status": 0}, {"_id": 1, "lat": 1, "lng": 1, "estate_type2": "1"})))
seaweed["lng"] = seaweed["lng"] / 10000000000.0
seaweed["lat"] = seaweed["lat"] / 10000000000.0
seaweed = seaweed.rename(columns={"_id": "commid"})
seaweed["commid"] = seaweed["commid"].apply(lambda x: str(x))
# In[22]:
stat = stat.merge(seaweed, on="commid", how="left")
# In[23]:
stat = stat.reset_index(drop=True)
stat = stat.dropna()
# In[24]:
# 计算经纬度距离
def get_distance(x, y):
return tbd.getdistance(x[0], x[1], y[0], y[1])
# In[25]:
stat_zhuzhai = stat[stat.estate_type2 == '普通住宅']
stat_zhuzhai = stat_zhuzhai.reset_index(drop=True)
stat_shangzhu = stat[stat.estate_type2 == '商住']
stat_shangzhu = stat_shangzhu.reset_index(drop=True)
# In[26]:
monthly_data = monthly_data.merge(stat[['lng', 'lat', 'estate_type2', 'commid']], on='commid', how='left')
# In[27]:
zhuzhai_model = NearestNeighbors(metric=get_distance, radius=1000)
zhuzhai_model.fit(stat_zhuzhai[['lng', 'lat']].values)
shangzhu_model = NearestNeighbors(metric=get_distance, radius=1000)
shangzhu_model.fit(stat_shangzhu[['lng', 'lat']].values)
# In[28]:
def calc(item):
if pd.isna(item['lng']):
return None
if item["estate_type2"] == "商住":
distances, indices = shangzhu_model.radius_neighbors(X=[[item['lng'], item['lat']]])
pois = stat_shangzhu.iloc[indices[0]].copy()
return pois["avg_price"].mean()
else:
distances, indices = zhuzhai_model.radius_neighbors(X=[[item['lng'], item['lat']]])
pois = stat_zhuzhai.iloc[indices[0]].copy()
return pois["avg_price"].mean()
# In[29]:
monthly_data['around_price'] = monthly_data.apply(lambda x: calc(x), axis=1)
# In[30]:
monthly_data["around_price_ratio"] = monthly_data["price"] / monthly_data["around_price"]
monthly_data.loc[(monthly_data.around_price_ratio >= 1.75) | (monthly_data.kalman_price_ratio < 0.5), "异常第三方"] = 5
# In[ ]:
# 去除诸葛找房独家数据
# In[ ]:
monthly_data.loc[(monthly_data["异常第三方"].isna()) & (monthly_data.web_count == 1) & (
monthly_data.source == "zhugezhaofang_esf_price"), "异常第三方"] = 6
# In[63]:
print(len(monthly_data[monthly_data["异常第三方"] == 1]))
print(len(monthly_data[monthly_data["异常第三方"] == 2]))
print(len(monthly_data[monthly_data["异常第三方"] == 3]))
print(len(monthly_data[monthly_data["异常第三方"] == 4]))
print(len(monthly_data[monthly_data["异常第三方"] == 5]))
print(len(monthly_data[monthly_data["异常第三方"] == 6]))
print(len(monthly_data[~monthly_data["异常第三方"].isna()]))
monthly_data
# In[39]:
# monthly_data[monthly_data["异常第三方"]==3].to_excel("异常第三方-3.xlsx")
commids = monthly_data[~monthly_data["异常第三方"].isna()].groupby(by=['commid']).count()
print(len(commids))
# In[40]:
monthly_data[monthly_data.around_price_ratio < 2]["around_price_ratio"].hist(bins=100)
# In[41]:
monthly_data["around_price"] = monthly_data["around_price"].apply(lambda x: int(x) if pd.notna(x) else x)
# In[68]:
data = monthly_data[~monthly_data["异常第三方"].isna()]
# data = monthly_data[monthly_data["异常第三方"] == 6]
# In[69]:
print(len(data))
# In[67]:
actions = []
for i, d in data.iterrows():
print(i)
# average_statistics_client.update_one({'s_date':d['s_date'],'city':d['city'],'region':d['region'], 'name':d['name']},{'$set':{d['source']:None}})
action = UpdateOne({'s_date': d['s_date'], 'city': d['city'], 'region': d['region'], 'name': d['name']},
{'$set': {d['source']: None}})
actions.append(action)
average_statistics_client.bulk_write(actions)
# rets1=monthly_data[(monthly_data["异常第三方"].isna()) & (monthly_data.web_count==1) & (monthly_data.source=="zhugezhaofang_esf_price")]
+17
View File
@@ -0,0 +1,17 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
dag_id="third_price_dag",
# 每月23日运行
schedule_interval="0 16 18 * *",
start_date=pendulum.datetime(2022, 18, 16, tz="Asia/Shanghai"),
catchup=True,
description="建行过滤第三方均价"
) as dag:
t1 = BashOperator(
task_id='third_price',
bash_command='python /opt/airflow/dags/repo/dags/jh/price/third_price.py',
)
t1