Files
airflow/dags/jh/price/third_price.py
T
shenjian@fangjia.com a2fa2be77a 优化
2023-03-21 11:07:07 +08:00

306 lines
9.4 KiB
Python

from datetime import datetime
from goojia_utils import MAIN_MONGO_CLIENT, dateutils
from sklearn.cluster import DBSCAN
import numpy as np
import pandas as pd
from pykalman import KalmanFilter
from pymongo import UpdateOne
import transbigdata as tbd
from sklearn.neighbors import NearestNeighbors
from sqlalchemy import create_engine
year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-2), '%Y%m'))
before_year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), years=-1, months=-1), '%Y%m'))
average_statistics_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "average_statistics")
average_statistics = pd.DataFrame(
list(average_statistics_client.find({"s_date": {"$gt": before_year_month}}).batch_size(50000)))
data = list()
keys = [key for key in average_statistics.columns if "esf_price" in key]
def assemble(item, data):
for website in keys:
if pd.notna(item[website]):
data.append({"city": item["city"], "region": item["region"], "name": item["name"], "s_date": item["s_date"],
"price": item[website], "source": website, "commid": item["seaweed_id"]})
average_statistics.apply(lambda x: assemble(x, data), axis=1)
data = pd.DataFrame(data)
# In[7]:
monthly_data = data[data.s_date == data.s_date.max()].copy()
monthly_data['region_mean'] = monthly_data.groupby(by=['city', 'region'])["price"].transform('mean').map(int)
monthly_data['region_min'] = monthly_data.groupby(by=['city', 'region'])["price"].transform(lambda x: x.quantile(0.01))
monthly_data['region_max'] = monthly_data.groupby(by=['city', 'region'])["price"].transform(lambda x: x.quantile(0.99))
# In[8]:
monthly_data.loc[
(monthly_data.region_min >= monthly_data.price) | (monthly_data.price >= monthly_data.region_max), "异常第三方"] = 1
# In[9]:
monthly_data['web_count'] = monthly_data.groupby(by=['city', 'region', "name"])["price"].transform('count')
monthly_data['comm_mean'] = monthly_data.groupby(by=['city', 'region', "name"])["price"].transform('mean').map(int)
# In[10]:
monthly_data.loc[(monthly_data.web_count >= 3) & (monthly_data["异常第三方"].isna()) & (
(monthly_data.price > monthly_data.comm_mean * 1.4) | (
(monthly_data.price < monthly_data.comm_mean * 0.6))), "异常第三方"] = 2
# In[11]:
def dbscan_denose(x):
db = DBSCAN(eps=x.max() / 10, min_samples=2).fit([[a] for a in x])
if (db.labels_ == -1).all() or len(x) < 3:
return np.zeros_like(db.labels_)
labels_ = db.labels_
labels_[np.where(labels_ > -1)] = 0
return db.labels_
# In[12]:
monthly_data["dbscan_denose"] = monthly_data.groupby(by=['city', 'region', 'name'])["price"].transform(
lambda x: dbscan_denose(x))
monthly_data.loc[(monthly_data["异常第三方"].isna()) & (monthly_data.dbscan_denose == -1), "异常第三方"] = 3
# In[13]:
monthly_data = monthly_data.sort_values(by=["city", "region", "name"])
# In[14]:
# community_base_price = oracle.scan_by_sql_to_data_frame(f"select commid, year_month, avg_price, calc_type,confidence, cityname from community_base_price_gj where year_month > {before_year_month}")
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
community_base_price = pd.read_sql(f"select gujia_bds_id as commid,gujiabds_clc_yrmo_val as year_month, gujia_comm_basic_prc as avg_price, urbn_nm as cityname,gujia_bds_clmtd as calc_type, gujia_bds_cfdrg as confidence from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val>{before_year_month} ", con=engine)
community_base_price = community_base_price.sort_values(by=['commid', 'year_month'], ascending=[True, True])
# In[15]:
city_prices = dict()
for entry in community_base_price.groupby(by="cityname")["avg_price"].mean().reset_index().to_dict(orient='records'):
city_prices[entry['cityname']] = int(entry['avg_price'])
def denose_kalman(stats):
stats = stats.reset_index(drop=True)
delta_t = np.pi * 2 / len(stats["avg_price"])
observations_data = stats["avg_price"]
var = stats["avg_price"].var()
initial_value_guess = stats[stats.confidence == stats.confidence.max()][['avg_price']][0:1]
if initial_value_guess.empty:
initial_value_guess = stats[['avg_price']][0:1]
transition_matrix = 1
cityname = stats.iloc[0, :]['cityname']
if cityname in city_prices:
transition_covariance = city_prices[cityname] * 0.15
else:
transition_covariance = 100
kf = KalmanFilter(
initial_state_mean=initial_value_guess,
initial_state_covariance=1,
observation_covariance=1,
transition_covariance=transition_covariance,
transition_matrices=1
)
pred_state, state_cov = kf.smooth(observations_data)
return int(pred_state[-1][0])
# In[16]:
kalman_stats = list()
i = 0
for commid, items in community_base_price.groupby(by="commid"):
i = i + 1
if i % 1000 == 0:
print(i)
kalman_stats.append({"commid": commid, "kalman_price": denose_kalman(items)})
kalman_stats = pd.DataFrame(kalman_stats)
# In[17]:
monthly_data = monthly_data.merge(kalman_stats, on="commid", how='left')
# In[18]:
monthly_data["kalman_price_ratio"] = monthly_data["price"] / monthly_data["kalman_price"]
# In[19]:
monthly_data.loc[(monthly_data.kalman_price_ratio >= 1.5) | (monthly_data.kalman_price_ratio < 0.5), "异常第三方"] = 4
# In[20]:
stat = community_base_price[community_base_price.year_month == community_base_price.year_month.max()]
# In[21]:
seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed")
seaweed = pd.DataFrame(
list(seaweed_client.find({"cat": "district", "status": 0}, {"_id": 1, "lat": 1, "lng": 1, "estate_type2": "1"})))
seaweed["lng"] = seaweed["lng"] / 10000000000.0
seaweed["lat"] = seaweed["lat"] / 10000000000.0
seaweed = seaweed.rename(columns={"_id": "commid"})
seaweed["commid"] = seaweed["commid"].apply(lambda x: str(x))
# In[22]:
stat = stat.merge(seaweed, on="commid", how="left")
# In[23]:
stat = stat.reset_index(drop=True)
stat = stat.dropna()
# In[24]:
# 计算经纬度距离
def get_distance(x, y):
return tbd.getdistance(x[0], x[1], y[0], y[1])
# In[25]:
stat_zhuzhai = stat[stat.estate_type2 == '普通住宅']
stat_zhuzhai = stat_zhuzhai.reset_index(drop=True)
stat_shangzhu = stat[stat.estate_type2 == '商住']
stat_shangzhu = stat_shangzhu.reset_index(drop=True)
# In[26]:
monthly_data = monthly_data.merge(stat[['lng', 'lat', 'estate_type2', 'commid']], on='commid', how='left')
# In[27]:
zhuzhai_model = NearestNeighbors(metric=get_distance, radius=1000)
zhuzhai_model.fit(stat_zhuzhai[['lng', 'lat']].values)
shangzhu_model = NearestNeighbors(metric=get_distance, radius=1000)
shangzhu_model.fit(stat_shangzhu[['lng', 'lat']].values)
# In[28]:
def calc(item):
if pd.isna(item['lng']):
return None
if item["estate_type2"] == "商住":
distances, indices = shangzhu_model.radius_neighbors(X=[[item['lng'], item['lat']]])
pois = stat_shangzhu.iloc[indices[0]].copy()
return pois["avg_price"].mean()
else:
distances, indices = zhuzhai_model.radius_neighbors(X=[[item['lng'], item['lat']]])
pois = stat_zhuzhai.iloc[indices[0]].copy()
return pois["avg_price"].mean()
# In[29]:
monthly_data['around_price'] = monthly_data.apply(lambda x: calc(x), axis=1)
# In[30]:
monthly_data["around_price_ratio"] = monthly_data["price"] / monthly_data["around_price"]
monthly_data.loc[(monthly_data.around_price_ratio >= 1.75) | (monthly_data.kalman_price_ratio < 0.5), "异常第三方"] = 5
# In[ ]:
# 去除诸葛找房独家数据
# In[ ]:
monthly_data.loc[(monthly_data["异常第三方"].isna()) & (monthly_data.web_count == 1) & (
monthly_data.source == "zhugezhaofang_esf_price"), "异常第三方"] = 6
# In[63]:
print(len(monthly_data[monthly_data["异常第三方"] == 1]))
print(len(monthly_data[monthly_data["异常第三方"] == 2]))
print(len(monthly_data[monthly_data["异常第三方"] == 3]))
print(len(monthly_data[monthly_data["异常第三方"] == 4]))
print(len(monthly_data[monthly_data["异常第三方"] == 5]))
print(len(monthly_data[monthly_data["异常第三方"] == 6]))
print(len(monthly_data[~monthly_data["异常第三方"].isna()]))
monthly_data
# In[39]:
# monthly_data[monthly_data["异常第三方"]==3].to_excel("异常第三方-3.xlsx")
commids = monthly_data[~monthly_data["异常第三方"].isna()].groupby(by=['commid']).count()
print(len(commids))
# In[40]:
monthly_data[monthly_data.around_price_ratio < 2]["around_price_ratio"].hist(bins=100)
# In[41]:
monthly_data["around_price"] = monthly_data["around_price"].apply(lambda x: int(x) if pd.notna(x) else x)
# In[68]:
data = monthly_data[~monthly_data["异常第三方"].isna()]
# data = monthly_data[monthly_data["异常第三方"] == 6]
# In[69]:
print(len(data))
# In[67]:
actions = []
for i, d in data.iterrows():
print(i)
# average_statistics_client.update_one({'s_date':d['s_date'],'city':d['city'],'region':d['region'], 'name':d['name']},{'$set':{d['source']:None}})
action = UpdateOne({'s_date': d['s_date'], 'city': d['city'], 'region': d['region'], 'name': d['name']},
{'$set': {d['source']: None}})
actions.append(action)
average_statistics_client.bulk_write(actions)
# rets1=monthly_data[(monthly_data["异常第三方"].isna()) & (monthly_data.web_count==1) & (monthly_data.source=="zhugezhaofang_esf_price")]