305 lines
9.2 KiB
Python
305 lines
9.2 KiB
Python
from datetime import datetime
|
|
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, pgsql
|
|
from sklearn.cluster import DBSCAN
|
|
import numpy as np
|
|
import pandas as pd
|
|
from pykalman import KalmanFilter
|
|
from pymongo import UpdateOne
|
|
import transbigdata as tbd
|
|
from sklearn.neighbors import NearestNeighbors
|
|
|
|
|
|
year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-2), '%Y%m'))
|
|
before_year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), years=-1, months=-1), '%Y%m'))
|
|
|
|
average_statistics_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "average_statistics")
|
|
|
|
average_statistics = pd.DataFrame(
|
|
list(average_statistics_client.find({"s_date": {"$gt": before_year_month}}).batch_size(50000)))
|
|
|
|
data = list()
|
|
keys = [key for key in average_statistics.columns if "esf_price" in key]
|
|
|
|
|
|
def assemble(item, data):
|
|
for website in keys:
|
|
if pd.notna(item[website]):
|
|
data.append({"city": item["city"], "region": item["region"], "name": item["name"], "s_date": item["s_date"],
|
|
"price": item[website], "source": website, "commid": item["seaweed_id"]})
|
|
|
|
|
|
average_statistics.apply(lambda x: assemble(x, data), axis=1)
|
|
data = pd.DataFrame(data)
|
|
|
|
# In[7]:
|
|
|
|
|
|
monthly_data = data[data.s_date == data.s_date.max()].copy()
|
|
monthly_data['region_mean'] = monthly_data.groupby(by=['city', 'region'])["price"].transform('mean').map(int)
|
|
monthly_data['region_min'] = monthly_data.groupby(by=['city', 'region'])["price"].transform(lambda x: x.quantile(0.01))
|
|
monthly_data['region_max'] = monthly_data.groupby(by=['city', 'region'])["price"].transform(lambda x: x.quantile(0.99))
|
|
|
|
# In[8]:
|
|
|
|
|
|
monthly_data.loc[
|
|
(monthly_data.region_min >= monthly_data.price) | (monthly_data.price >= monthly_data.region_max), "异常第三方"] = 1
|
|
|
|
# In[9]:
|
|
|
|
|
|
monthly_data['web_count'] = monthly_data.groupby(by=['city', 'region', "name"])["price"].transform('count')
|
|
monthly_data['comm_mean'] = monthly_data.groupby(by=['city', 'region', "name"])["price"].transform('mean').map(int)
|
|
|
|
# In[10]:
|
|
|
|
|
|
monthly_data.loc[(monthly_data.web_count >= 3) & (monthly_data["异常第三方"].isna()) & (
|
|
(monthly_data.price > monthly_data.comm_mean * 1.4) | (
|
|
(monthly_data.price < monthly_data.comm_mean * 0.6))), "异常第三方"] = 2
|
|
|
|
|
|
# In[11]:
|
|
|
|
|
|
def dbscan_denose(x):
|
|
db = DBSCAN(eps=x.max() / 10, min_samples=2).fit([[a] for a in x])
|
|
if (db.labels_ == -1).all() or len(x) < 3:
|
|
return np.zeros_like(db.labels_)
|
|
labels_ = db.labels_
|
|
labels_[np.where(labels_ > -1)] = 0
|
|
return db.labels_
|
|
|
|
|
|
# In[12]:
|
|
|
|
|
|
monthly_data["dbscan_denose"] = monthly_data.groupby(by=['city', 'region', 'name'])["price"].transform(
|
|
lambda x: dbscan_denose(x))
|
|
monthly_data.loc[(monthly_data["异常第三方"].isna()) & (monthly_data.dbscan_denose == -1), "异常第三方"] = 3
|
|
|
|
# In[13]:
|
|
|
|
|
|
monthly_data = monthly_data.sort_values(by=["city", "region", "name"])
|
|
|
|
# In[14]:
|
|
# community_base_price = oracle.scan_by_sql_to_data_frame(f"select commid, year_month, avg_price, calc_type,confidence, cityname from community_base_price_gj where year_month > {before_year_month}")
|
|
|
|
|
|
community_base_price = pgsql.scan_by_sql_to_data_frame(f"select gujia_bds_id as commid,gujiabds_clc_yrmo_val as year_month, gujia_comm_basic_prc as avg_price, urbn_nm as cityname,gujia_bds_clmtd as calc_type, gujia_bds_cfdrg as confidence from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val>{before_year_month} ")
|
|
community_base_price = community_base_price.sort_values(by=['commid', 'year_month'], ascending=[True, True])
|
|
|
|
# In[15]:
|
|
|
|
|
|
city_prices = dict()
|
|
for entry in community_base_price.groupby(by="cityname")["avg_price"].mean().reset_index().to_dict(orient='records'):
|
|
city_prices[entry['cityname']] = int(entry['avg_price'])
|
|
|
|
|
|
def denose_kalman(stats):
|
|
stats = stats.reset_index(drop=True)
|
|
delta_t = np.pi * 2 / len(stats["avg_price"])
|
|
observations_data = stats["avg_price"]
|
|
var = stats["avg_price"].var()
|
|
initial_value_guess = stats[stats.confidence == stats.confidence.max()][['avg_price']][0:1]
|
|
if initial_value_guess.empty:
|
|
initial_value_guess = stats[['avg_price']][0:1]
|
|
transition_matrix = 1
|
|
cityname = stats.iloc[0, :]['cityname']
|
|
if cityname in city_prices:
|
|
transition_covariance = city_prices[cityname] * 0.15
|
|
else:
|
|
transition_covariance = 100
|
|
kf = KalmanFilter(
|
|
initial_state_mean=initial_value_guess,
|
|
initial_state_covariance=1,
|
|
observation_covariance=1,
|
|
transition_covariance=transition_covariance,
|
|
transition_matrices=1
|
|
)
|
|
pred_state, state_cov = kf.smooth(observations_data)
|
|
return int(pred_state[-1][0])
|
|
|
|
|
|
# In[16]:
|
|
|
|
|
|
kalman_stats = list()
|
|
i = 0
|
|
for commid, items in community_base_price.groupby(by="commid"):
|
|
i = i + 1
|
|
if i % 1000 == 0:
|
|
print(i)
|
|
kalman_stats.append({"commid": commid, "kalman_price": denose_kalman(items)})
|
|
kalman_stats = pd.DataFrame(kalman_stats)
|
|
|
|
# In[17]:
|
|
|
|
|
|
monthly_data = monthly_data.merge(kalman_stats, on="commid", how='left')
|
|
|
|
# In[18]:
|
|
|
|
|
|
monthly_data["kalman_price_ratio"] = monthly_data["price"] / monthly_data["kalman_price"]
|
|
|
|
# In[19]:
|
|
|
|
|
|
monthly_data.loc[(monthly_data.kalman_price_ratio >= 1.5) | (monthly_data.kalman_price_ratio < 0.5), "异常第三方"] = 4
|
|
|
|
# In[20]:
|
|
|
|
|
|
stat = community_base_price[community_base_price.year_month == community_base_price.year_month.max()]
|
|
|
|
# In[21]:
|
|
|
|
|
|
seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed")
|
|
seaweed = pd.DataFrame(
|
|
list(seaweed_client.find({"cat": "district", "status": 0}, {"_id": 1, "lat": 1, "lng": 1, "estate_type2": "1"})))
|
|
seaweed["lng"] = seaweed["lng"] / 10000000000.0
|
|
seaweed["lat"] = seaweed["lat"] / 10000000000.0
|
|
seaweed = seaweed.rename(columns={"_id": "commid"})
|
|
seaweed["commid"] = seaweed["commid"].apply(lambda x: str(x))
|
|
|
|
# In[22]:
|
|
|
|
|
|
stat = stat.merge(seaweed, on="commid", how="left")
|
|
|
|
# In[23]:
|
|
|
|
|
|
stat = stat.reset_index(drop=True)
|
|
stat = stat.dropna()
|
|
|
|
# In[24]:
|
|
|
|
# 计算经纬度距离
|
|
def get_distance(x, y):
|
|
return tbd.getdistance(x[0], x[1], y[0], y[1])
|
|
|
|
|
|
# In[25]:
|
|
|
|
|
|
stat_zhuzhai = stat[stat.estate_type2 == '普通住宅']
|
|
stat_zhuzhai = stat_zhuzhai.reset_index(drop=True)
|
|
stat_shangzhu = stat[stat.estate_type2 == '商住']
|
|
stat_shangzhu = stat_shangzhu.reset_index(drop=True)
|
|
|
|
# In[26]:
|
|
|
|
|
|
monthly_data = monthly_data.merge(stat[['lng', 'lat', 'estate_type2', 'commid']], on='commid', how='left')
|
|
|
|
# In[27]:
|
|
|
|
|
|
zhuzhai_model = NearestNeighbors(metric=get_distance, radius=1000)
|
|
zhuzhai_model.fit(stat_zhuzhai[['lng', 'lat']].values)
|
|
shangzhu_model = NearestNeighbors(metric=get_distance, radius=1000)
|
|
shangzhu_model.fit(stat_shangzhu[['lng', 'lat']].values)
|
|
|
|
|
|
# In[28]:
|
|
|
|
|
|
def calc(item):
|
|
if pd.isna(item['lng']):
|
|
return None
|
|
if item["estate_type2"] == "商住":
|
|
distances, indices = shangzhu_model.radius_neighbors(X=[[item['lng'], item['lat']]])
|
|
pois = stat_shangzhu.iloc[indices[0]].copy()
|
|
return pois["avg_price"].mean()
|
|
else:
|
|
distances, indices = zhuzhai_model.radius_neighbors(X=[[item['lng'], item['lat']]])
|
|
pois = stat_zhuzhai.iloc[indices[0]].copy()
|
|
return pois["avg_price"].mean()
|
|
|
|
|
|
# In[29]:
|
|
|
|
|
|
monthly_data['around_price'] = monthly_data.apply(lambda x: calc(x), axis=1)
|
|
|
|
# In[30]:
|
|
|
|
|
|
monthly_data["around_price_ratio"] = monthly_data["price"] / monthly_data["around_price"]
|
|
monthly_data.loc[(monthly_data.around_price_ratio >= 1.75) | (monthly_data.kalman_price_ratio < 0.5), "异常第三方"] = 5
|
|
|
|
# In[ ]:
|
|
|
|
|
|
# 去除诸葛找房独家数据
|
|
|
|
|
|
# In[ ]:
|
|
|
|
|
|
monthly_data.loc[(monthly_data["异常第三方"].isna()) & (monthly_data.web_count == 1) & (
|
|
monthly_data.source == "zhugezhaofang_esf_price"), "异常第三方"] = 6
|
|
|
|
# In[63]:
|
|
|
|
|
|
print(len(monthly_data[monthly_data["异常第三方"] == 1]))
|
|
print(len(monthly_data[monthly_data["异常第三方"] == 2]))
|
|
print(len(monthly_data[monthly_data["异常第三方"] == 3]))
|
|
print(len(monthly_data[monthly_data["异常第三方"] == 4]))
|
|
print(len(monthly_data[monthly_data["异常第三方"] == 5]))
|
|
print(len(monthly_data[monthly_data["异常第三方"] == 6]))
|
|
print(len(monthly_data[~monthly_data["异常第三方"].isna()]))
|
|
monthly_data
|
|
|
|
# In[39]:
|
|
|
|
|
|
# monthly_data[monthly_data["异常第三方"]==3].to_excel("异常第三方-3.xlsx")
|
|
commids = monthly_data[~monthly_data["异常第三方"].isna()].groupby(by=['commid']).count()
|
|
print(len(commids))
|
|
|
|
# In[40]:
|
|
|
|
|
|
monthly_data[monthly_data.around_price_ratio < 2]["around_price_ratio"].hist(bins=100)
|
|
|
|
# In[41]:
|
|
|
|
|
|
monthly_data["around_price"] = monthly_data["around_price"].apply(lambda x: int(x) if pd.notna(x) else x)
|
|
|
|
# In[68]:
|
|
|
|
|
|
data = monthly_data[~monthly_data["异常第三方"].isna()]
|
|
# data = monthly_data[monthly_data["异常第三方"] == 6]
|
|
|
|
|
|
# In[69]:
|
|
|
|
|
|
print(len(data))
|
|
|
|
# In[67]:
|
|
actions = []
|
|
for i, d in data.iterrows():
|
|
print(i)
|
|
# average_statistics_client.update_one({'s_date':d['s_date'],'city':d['city'],'region':d['region'], 'name':d['name']},{'$set':{d['source']:None}})
|
|
action = UpdateOne({'s_date': d['s_date'], 'city': d['city'], 'region': d['region'], 'name': d['name']},
|
|
{'$set': {d['source']: None}})
|
|
actions.append(action)
|
|
|
|
average_statistics_client.bulk_write(actions)
|
|
# rets1=monthly_data[(monthly_data["异常第三方"].isna()) & (monthly_data.web_count==1) & (monthly_data.source=="zhugezhaofang_esf_price")]
|
|
|
|
|
|
|
|
|
|
|