from datetime import datetime from goojia_utils import MAIN_MONGO_CLIENT, dateutils, pgsql from sklearn.cluster import DBSCAN import numpy as np import pandas as pd from pykalman import KalmanFilter from pymongo import UpdateOne import transbigdata as tbd from sklearn.neighbors import NearestNeighbors year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-2), '%Y%m')) before_year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), years=-1, months=-1), '%Y%m')) average_statistics_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "average_statistics") average_statistics = pd.DataFrame( list(average_statistics_client.find({"s_date": {"$gt": before_year_month}}).batch_size(50000))) data = list() keys = [key for key in average_statistics.columns if "esf_price" in key] def assemble(item, data): for website in keys: if pd.notna(item[website]): data.append({"city": item["city"], "region": item["region"], "name": item["name"], "s_date": item["s_date"], "price": item[website], "source": website, "commid": item["seaweed_id"]}) average_statistics.apply(lambda x: assemble(x, data), axis=1) data = pd.DataFrame(data) # In[7]: monthly_data = data[data.s_date == data.s_date.max()].copy() monthly_data['region_mean'] = monthly_data.groupby(by=['city', 'region'])["price"].transform('mean').map(int) monthly_data['region_min'] = monthly_data.groupby(by=['city', 'region'])["price"].transform(lambda x: x.quantile(0.01)) monthly_data['region_max'] = monthly_data.groupby(by=['city', 'region'])["price"].transform(lambda x: x.quantile(0.99)) # In[8]: monthly_data.loc[ (monthly_data.region_min >= monthly_data.price) | (monthly_data.price >= monthly_data.region_max), "异常第三方"] = 1 # In[9]: monthly_data['web_count'] = monthly_data.groupby(by=['city', 'region', "name"])["price"].transform('count') monthly_data['comm_mean'] = monthly_data.groupby(by=['city', 'region', "name"])["price"].transform('mean').map(int) # In[10]: monthly_data.loc[(monthly_data.web_count >= 3) & (monthly_data["异常第三方"].isna()) & ( (monthly_data.price > monthly_data.comm_mean * 1.4) | ( (monthly_data.price < monthly_data.comm_mean * 0.6))), "异常第三方"] = 2 # In[11]: def dbscan_denose(x): db = DBSCAN(eps=x.max() / 10, min_samples=2).fit([[a] for a in x]) if (db.labels_ == -1).all() or len(x) < 3: return np.zeros_like(db.labels_) labels_ = db.labels_ labels_[np.where(labels_ > -1)] = 0 return db.labels_ # In[12]: monthly_data["dbscan_denose"] = monthly_data.groupby(by=['city', 'region', 'name'])["price"].transform( lambda x: dbscan_denose(x)) monthly_data.loc[(monthly_data["异常第三方"].isna()) & (monthly_data.dbscan_denose == -1), "异常第三方"] = 3 # In[13]: monthly_data = monthly_data.sort_values(by=["city", "region", "name"]) # In[14]: # community_base_price = oracle.scan_by_sql_to_data_frame(f"select commid, year_month, avg_price, calc_type,confidence, cityname from community_base_price_gj where year_month > {before_year_month}") community_base_price = pgsql.scan_by_sql_to_data_frame(f"select gujia_bds_id as commid,gujiabds_clc_yrmo_val as year_month, gujia_comm_basic_prc as avg_price, urbn_nm as cityname,gujia_bds_clmtd as calc_type, gujia_bds_cfdrg as confidence from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val>{before_year_month} ") community_base_price = community_base_price.sort_values(by=['commid', 'year_month'], ascending=[True, True]) # In[15]: city_prices = dict() for entry in community_base_price.groupby(by="cityname")["avg_price"].mean().reset_index().to_dict(orient='records'): city_prices[entry['cityname']] = int(entry['avg_price']) def denose_kalman(stats): stats = stats.reset_index(drop=True) delta_t = np.pi * 2 / len(stats["avg_price"]) observations_data = stats["avg_price"] var = stats["avg_price"].var() initial_value_guess = stats[stats.confidence == stats.confidence.max()][['avg_price']][0:1] if initial_value_guess.empty: initial_value_guess = stats[['avg_price']][0:1] transition_matrix = 1 cityname = stats.iloc[0, :]['cityname'] if cityname in city_prices: transition_covariance = city_prices[cityname] * 0.15 else: transition_covariance = 100 kf = KalmanFilter( initial_state_mean=initial_value_guess, initial_state_covariance=1, observation_covariance=1, transition_covariance=transition_covariance, transition_matrices=1 ) pred_state, state_cov = kf.smooth(observations_data) return int(pred_state[-1][0]) # In[16]: kalman_stats = list() i = 0 for commid, items in community_base_price.groupby(by="commid"): i = i + 1 if i % 1000 == 0: print(i) kalman_stats.append({"commid": commid, "kalman_price": denose_kalman(items)}) kalman_stats = pd.DataFrame(kalman_stats) # In[17]: monthly_data = monthly_data.merge(kalman_stats, on="commid", how='left') # In[18]: monthly_data["kalman_price_ratio"] = monthly_data["price"] / monthly_data["kalman_price"] # In[19]: monthly_data.loc[(monthly_data.kalman_price_ratio >= 1.5) | (monthly_data.kalman_price_ratio < 0.5), "异常第三方"] = 4 # In[20]: stat = community_base_price[community_base_price.year_month == community_base_price.year_month.max()] # In[21]: seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed") seaweed = pd.DataFrame( list(seaweed_client.find({"cat": "district", "status": 0}, {"_id": 1, "lat": 1, "lng": 1, "estate_type2": "1"}))) seaweed["lng"] = seaweed["lng"] / 10000000000.0 seaweed["lat"] = seaweed["lat"] / 10000000000.0 seaweed = seaweed.rename(columns={"_id": "commid"}) seaweed["commid"] = seaweed["commid"].apply(lambda x: str(x)) # In[22]: stat = stat.merge(seaweed, on="commid", how="left") # In[23]: stat = stat.reset_index(drop=True) stat = stat.dropna() # In[24]: # 计算经纬度距离 def get_distance(x, y): return tbd.getdistance(x[0], x[1], y[0], y[1]) # In[25]: stat_zhuzhai = stat[stat.estate_type2 == '普通住宅'] stat_zhuzhai = stat_zhuzhai.reset_index(drop=True) stat_shangzhu = stat[stat.estate_type2 == '商住'] stat_shangzhu = stat_shangzhu.reset_index(drop=True) # In[26]: monthly_data = monthly_data.merge(stat[['lng', 'lat', 'estate_type2', 'commid']], on='commid', how='left') # In[27]: zhuzhai_model = NearestNeighbors(metric=get_distance, radius=1000) zhuzhai_model.fit(stat_zhuzhai[['lng', 'lat']].values) shangzhu_model = NearestNeighbors(metric=get_distance, radius=1000) shangzhu_model.fit(stat_shangzhu[['lng', 'lat']].values) # In[28]: def calc(item): if pd.isna(item['lng']): return None if item["estate_type2"] == "商住": distances, indices = shangzhu_model.radius_neighbors(X=[[item['lng'], item['lat']]]) pois = stat_shangzhu.iloc[indices[0]].copy() return pois["avg_price"].mean() else: distances, indices = zhuzhai_model.radius_neighbors(X=[[item['lng'], item['lat']]]) pois = stat_zhuzhai.iloc[indices[0]].copy() return pois["avg_price"].mean() # In[29]: monthly_data['around_price'] = monthly_data.apply(lambda x: calc(x), axis=1) # In[30]: monthly_data["around_price_ratio"] = monthly_data["price"] / monthly_data["around_price"] monthly_data.loc[(monthly_data.around_price_ratio >= 1.75) | (monthly_data.kalman_price_ratio < 0.5), "异常第三方"] = 5 # In[ ]: # 去除诸葛找房独家数据 # In[ ]: monthly_data.loc[(monthly_data["异常第三方"].isna()) & (monthly_data.web_count == 1) & ( monthly_data.source == "zhugezhaofang_esf_price"), "异常第三方"] = 6 # In[63]: print(len(monthly_data[monthly_data["异常第三方"] == 1])) print(len(monthly_data[monthly_data["异常第三方"] == 2])) print(len(monthly_data[monthly_data["异常第三方"] == 3])) print(len(monthly_data[monthly_data["异常第三方"] == 4])) print(len(monthly_data[monthly_data["异常第三方"] == 5])) print(len(monthly_data[monthly_data["异常第三方"] == 6])) print(len(monthly_data[~monthly_data["异常第三方"].isna()])) monthly_data # In[39]: # monthly_data[monthly_data["异常第三方"]==3].to_excel("异常第三方-3.xlsx") commids = monthly_data[~monthly_data["异常第三方"].isna()].groupby(by=['commid']).count() print(len(commids)) # In[40]: monthly_data[monthly_data.around_price_ratio < 2]["around_price_ratio"].hist(bins=100) # In[41]: monthly_data["around_price"] = monthly_data["around_price"].apply(lambda x: int(x) if pd.notna(x) else x) # In[68]: data = monthly_data[~monthly_data["异常第三方"].isna()] # data = monthly_data[monthly_data["异常第三方"] == 6] # In[69]: print(len(data)) # In[67]: actions = [] for i, d in data.iterrows(): print(i) # average_statistics_client.update_one({'s_date':d['s_date'],'city':d['city'],'region':d['region'], 'name':d['name']},{'$set':{d['source']:None}}) action = UpdateOne({'s_date': d['s_date'], 'city': d['city'], 'region': d['region'], 'name': d['name']}, {'$set': {d['source']: None}}) actions.append(action) average_statistics_client.bulk_write(actions) # rets1=monthly_data[(monthly_data["异常第三方"].isna()) & (monthly_data.web_count==1) & (monthly_data.source=="zhugezhaofang_esf_price")]