6ce1075d29
修改: PriceSystem/feature/region_feature.py 修改: PriceSystem/rent_price/district_map0413.py 修改: PriceSystem/rent_test.py
424 lines
11 KiB
Python
424 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
Created on Tue Mar 27 13:23:56 2018
|
|
|
|
@author: luogan
|
|
"""
|
|
import os
|
|
|
|
##print( os.getcwd())
|
|
#from collections import Counter
|
|
import pymongo
|
|
#from pymongo import MongoClient
|
|
import numpy as np
|
|
import random
|
|
import collections
|
|
import pymongo
|
|
import sklearn.cluster
|
|
import pandas
|
|
import numpy
|
|
import sys, time
|
|
#import pp
|
|
import fangjia
|
|
import datetime
|
|
import re
|
|
import pandas
|
|
|
|
import dateutil
|
|
import time
|
|
import fangjia
|
|
import config
|
|
import datetime
|
|
import pytz
|
|
import pandas as pd
|
|
import calendar
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def ghhh(kkg): #555555555555555555555555555
|
|
kkg=kkg.fillna('无')
|
|
a=list()
|
|
for i in kkg.index:
|
|
b=list(kkg.loc[[i],:].values.flatten())
|
|
a.append(b)
|
|
return a
|
|
|
|
|
|
|
|
|
|
|
|
def f_del(qq,g):
|
|
|
|
if g!= None:
|
|
|
|
return qq
|
|
else:
|
|
del qq['region']
|
|
return qq
|
|
|
|
def sample_pool(ct,rl,timd,timm):
|
|
|
|
query = {"city":ct,"cat":"rent","region":rl,
|
|
"p_date":{"$gt":timd,"$lt":timm}}
|
|
query=f_del(query,rl)
|
|
|
|
fields = {"total_price":1,"area":1, "city":1,"region":1,'p_date':1,
|
|
"district_name":1,"rent_type":1,"fitment":1,"block":1}
|
|
#lt= seawater.count(query)
|
|
###print(lt)
|
|
#pos = list()
|
|
#for s in seawater.find(query,fields1).skip(10000).limit(lt):
|
|
|
|
print('query=',query)
|
|
return fangjia.rent_sample(query, fields)
|
|
|
|
def k_means(pp):
|
|
from sklearn.cluster import KMeans
|
|
pv=list(pp.dropna())
|
|
gf=numpy.array([pv]).T
|
|
estimator = KMeans(n_clusters=3)#构造聚类器
|
|
estimator.fit(gf)#聚类
|
|
label_pred = estimator.labels_ #获取聚类标签
|
|
aa=collections.Counter(label_pred)
|
|
v=pandas.Series(aa)
|
|
gg=list(v)
|
|
index_max=gg.index(max(gg))
|
|
#yy=np.array([label_pred]).T
|
|
#un=np.hstack((gf,yy))
|
|
#uv=pd.DataFrame(un,columns=['price','label'])
|
|
centroids = estimator.cluster_centers_ #获取聚类中心
|
|
#inertia = estimator.inertia_ # 获取聚类准则的总和
|
|
center=centroids[index_max][0]
|
|
del pv,gf,aa,v,gg,centroids
|
|
return (round(center,2))
|
|
|
|
|
|
|
|
|
|
def get_weekend():
|
|
|
|
#from dateutil.parser import parse
|
|
#bb=parse(tt)
|
|
now_time=datetime.datetime.now()
|
|
before_time = now_time-datetime.timedelta(days=14)
|
|
pacific = pytz.timezone('Asia/Shanghai')
|
|
return pacific.localize( before_time )
|
|
|
|
|
|
|
|
def disanganf_price(ct,rl,lk,timd,timm):
|
|
|
|
fields={'calc_value':1,'weekend':1,'district_name':1}
|
|
query = {"city":ct,"region":rl,
|
|
"district_name":{'$in':lk},'weekend':{'$gt':get_weekend()}}
|
|
#pos = list()
|
|
#数据转化为数组,数组的元素为字典
|
|
|
|
return fangjia.rent_disanganf_series(query, fields)
|
|
|
|
|
|
|
|
|
|
def rent_partial_sell(ct,rl,timd,timm):
|
|
|
|
rele0=sample_pool(ct,rl,timd,timm)
|
|
#print('rele0=',rele0)
|
|
|
|
if rele0.shape[0]>0:
|
|
rele1=rele0[['district_name','p_date','area','total_price']]
|
|
rele1['total_price']=rele1['total_price']/rele1['area']
|
|
#print('up_rele.shape[0]=',rele1.shape[0])
|
|
|
|
rele=rele0.drop_duplicates(['district_name'])
|
|
|
|
p1=rele['district_name'].dropna()
|
|
p2=list(set(p1))
|
|
|
|
###print('*****************************************************************')
|
|
dd=disanganf_price(ct,rl,p2,timd,timm)
|
|
###print('rele.shape[0]=',rele.shape[0])
|
|
|
|
if rele.shape[0]>10:
|
|
|
|
rele['total_price']=rele['total_price']/rele['area']
|
|
#rent_most=k_means(rele['total_price'])
|
|
rele['sell_rent_ratio']=1
|
|
rv=rele[['total_price','district_name','area','sell_rent_ratio']].dropna()
|
|
rv1=rv.values
|
|
ro=[]
|
|
sell_rent_ratio=[]
|
|
for k in range(len(rv1)):
|
|
namee=rv1[k][1]
|
|
pee1= rele1[rele1['district_name']==namee]
|
|
pee2=pee1.sort_values(by='p_date',ascending=False)
|
|
price=pee2['total_price'].iloc[0]
|
|
ro.append(price)
|
|
#price=rv1[k][0]
|
|
if namee in dd.index:
|
|
|
|
sell=dd[namee]
|
|
ppoo=sell/price
|
|
sell_rent_ratio.append(ppoo)
|
|
#rv1[k][3]=sell/price
|
|
###print('rele0=',rele)
|
|
rent_most=k_means(pandas.Series(ro))
|
|
#rele=pd.DataFrame(rv1,columns=rv.columns)
|
|
roseries=pandas.Series(sell_rent_ratio)
|
|
|
|
roseries=roseries[roseries>5]
|
|
|
|
#rele=rele[rele['sell_rent_ratio']>5]
|
|
##print('sell_rent_ratio=',list(rele['sell_rent_ratio'])[:5])
|
|
##print(rele)
|
|
##print('#######################')
|
|
##print(rele.shape[0])
|
|
if len(roseries)>10:
|
|
|
|
#rent_most=k_means(rele['total_price'])
|
|
|
|
##print('rele.shape[0]=',rele.shape[0])
|
|
#print('yyyyyyyyyyyyyyyyyyyyyy=',len(roseries))
|
|
rent_sell=k_means(roseries.dropna())
|
|
|
|
del rele
|
|
return rent_sell,rent_most
|
|
else:
|
|
del rele
|
|
return '没有区域租售比',rent_most
|
|
else:
|
|
del rele
|
|
return '没有区域租售比','没有区域租金数据'
|
|
else:
|
|
return '没有任何数据','没有任何数据'
|
|
|
|
|
|
def dff(i,j,timd,timm): #666666666666666666666666666666666666666666666666666666
|
|
reef=rent_partial_sell(i,j,timd,timm)
|
|
##print('$$$$$$$$$$')
|
|
if type(reef[0])!=str:
|
|
return reef
|
|
else:
|
|
##print('&&&&&&&&&&&&&')
|
|
return rent_partial_sell(i,None,timd,timm)
|
|
|
|
|
|
def add_eatype(ppd):
|
|
if 'estate_type2' not in ppd.columns:
|
|
ppd['estate_type2']='普通住宅'
|
|
|
|
return ppd
|
|
|
|
|
|
|
|
def timestamp_to_int(dd):
|
|
|
|
dd['week1']=1
|
|
|
|
for i in dd.index:
|
|
|
|
f1=dd['weekend'][i].strftime("%Y-%m-%d")
|
|
f2 = re.sub("\D", "", f1)
|
|
f3=int(f2)
|
|
dd['week1'][i]=f3
|
|
###print(dd)
|
|
return dd
|
|
|
|
'''
|
|
def before_month_lastday(ti):
|
|
|
|
today=dateutil.parser.parse(str(ti))
|
|
|
|
#first = datetime.date(day=1, month=today.month, year=today.year)
|
|
|
|
lastMonth = today - datetime.timedelta(days=35)
|
|
|
|
def plus(k):
|
|
if k<10:
|
|
return '0'+str(k)
|
|
else:
|
|
return str(k)
|
|
y=lastMonth.year
|
|
m=lastMonth.month
|
|
#d=lastMonth.day
|
|
day=calendar.monthrange(y,m)[1]
|
|
|
|
cc=str(y)+plus(m)+plus(day)
|
|
|
|
return int(cc)
|
|
'''
|
|
|
|
|
|
def before_month_lastday(ti,num):
|
|
from dateutil.parser import parse
|
|
today=dateutil.parser.parse(str(ti))
|
|
|
|
#first = datetime.date(day=1, month=today.month, year=today.year)
|
|
|
|
lastMonth = today - datetime.timedelta(days=num)
|
|
|
|
def plus(k):
|
|
if k<10:
|
|
return '0'+str(k)
|
|
else:
|
|
return str(k)
|
|
y=lastMonth.year
|
|
m=lastMonth.month
|
|
#d=lastMonth.day
|
|
day=calendar.monthrange(y,m)[1]
|
|
|
|
cc=str(y)+plus(m)+plus(day)
|
|
bb=parse(cc)
|
|
pacific = pytz.timezone('Asia/Shanghai')
|
|
return pacific.localize(bb)
|
|
|
|
|
|
def region_make_up_price(ct,rl,lk,ti):
|
|
|
|
lgg=fangjia.get_collection("fangjia",config.get_sqm_rentprice_db(), config.get_mongodb_host(),27017)
|
|
#save_lg=client136.fangjia_stat.test_lg5
|
|
|
|
cc=before_month_lastday(ti,32)
|
|
#print('ti=',ti)
|
|
#print('cc=',cc)
|
|
|
|
#stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017)
|
|
stat=lgg
|
|
query = {"city":ct,"region":rl,"cat":"region",'weekend':cc,'district_name':None}
|
|
|
|
pos1=list(stat.find(query).sort([("weekend",-1)]).limit(3))
|
|
|
|
pos=pandas.DataFrame(pos1)
|
|
|
|
#pos=timestamp_to_int(pos)
|
|
#pos=pos[pos['week1']>cc]
|
|
|
|
##print('5'*10)
|
|
##print('uuuuuuuuuuuuuuuu=',pos)
|
|
|
|
|
|
if pos.shape[0]>=1:
|
|
pos=pos.sort_values(by='weekend',ascending=False)
|
|
pp1=pos['avg_price'].iloc[0]
|
|
#print('pp1=',pp1)
|
|
if type(pp1)!=None and type(pp1)!=str:
|
|
return pp1
|
|
else:
|
|
return None
|
|
else:
|
|
return None
|
|
|
|
def return_operate(ppu,mm):
|
|
|
|
if ppu==None:
|
|
return 0
|
|
else:
|
|
return int(10000*(mm-ppu)/ppu)
|
|
|
|
'''
|
|
def before_month_lastday(ti):
|
|
|
|
today=dateutil.parser.parse(str(ti))
|
|
|
|
#first = datetime.date(day=1, month=today.month, year=today.year)
|
|
|
|
lastMonth = today - datetime.timedelta(days=31)
|
|
|
|
cc=str(lastMonth.year)+str(lastMonth.month)+str(lastMonth.day)
|
|
|
|
return int(cc)
|
|
'''
|
|
|
|
#dd1=sample_pool(ct,rl,timd,timm)
|
|
|
|
|
|
|
|
def save_region_feature(ct,rl,timd,timm):
|
|
#a=time.time()
|
|
region_feature=dff(ct,rl,timd,timm)
|
|
|
|
#print('region_feature=',region_feature)
|
|
|
|
mmup=region_make_up_price(ct,rl,None,timm)
|
|
#print('mmup_region=',mmup)
|
|
#print('this month region price',region_feature[1])
|
|
markup=return_operate(mmup,region_feature[1])
|
|
|
|
def c_date():
|
|
date1=datetime.datetime.now()
|
|
|
|
pacific = pytz.timezone('Asia/Shanghai')
|
|
return pacific.localize(date1)
|
|
|
|
|
|
|
|
|
|
def get_weekend(tt):
|
|
|
|
from dateutil.parser import parse
|
|
bb=parse(tt)
|
|
pacific = pytz.timezone('Asia/Shanghai')
|
|
return pacific.localize(bb)
|
|
|
|
|
|
def month_lastday(ti):
|
|
from dateutil.parser import parse
|
|
today=dateutil.parser.parse(str(ti))
|
|
|
|
#first = datetime.date(day=1, month=today.month, year=today.year)
|
|
|
|
lastMonth = today# - datetime.timedelta(days=35)
|
|
|
|
def plus(k):
|
|
if k<10:
|
|
return '0'+str(k)
|
|
else:
|
|
return str(k)
|
|
y=lastMonth.year
|
|
m=lastMonth.month
|
|
#d=lastMonth.day
|
|
day=calendar.monthrange(y,m)[1]
|
|
|
|
cc=str(y)+plus(m)+plus(day)
|
|
bb=parse(cc)
|
|
pacific = pytz.timezone('Asia/Shanghai')
|
|
return pacific.localize(bb)
|
|
|
|
timme=month_lastday(timm)
|
|
##print('region_timme=',timme)
|
|
|
|
|
|
#stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017)
|
|
##print("######################## ","ct=",ct,"rl=",rl,"makeup=",make_up)
|
|
|
|
|
|
#save_lg = fangjia.get_collection("fangjia_stat", "rent_1221", config.get_district_stat_host(), config.get_district_stat_port())
|
|
avg=region_feature[1]
|
|
date=c_date()
|
|
if type(avg)!=str:
|
|
avg=float(avg)
|
|
save_lg = fangjia.get_collection("fangjia",config.get_sqm_rentprice_db(), config.get_mongodb_host(),27017)
|
|
|
|
save_lg.replace_one(
|
|
|
|
{"city":ct,"region":rl,"weekend":timme,"cat":'region'},
|
|
|
|
{"city":ct,"region":rl,"district_name":None,"markup":markup,'ratio':region_feature[0],
|
|
|
|
"avg_price":region_feature[1], "c_date":date,"weekend":timme,"cat":'region'},True
|
|
)
|
|
|
|
##print(time.time()-a)
|
|
|
|
###print(len(disname),len(disname))
|
|
|
|
|
|
|
|
|