新文件: feature/__init__.py
新文件: feature/region_feature.py
This commit is contained in:
@@ -0,0 +1,410 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Tue Mar 27 13:23:56 2018
|
||||
|
||||
@author: luogan
|
||||
"""
|
||||
import os
|
||||
|
||||
##print( os.getcwd())
|
||||
#from collections import Counter
|
||||
import pymongo
|
||||
#from pymongo import MongoClient
|
||||
import numpy as np
|
||||
import random
|
||||
import collections
|
||||
import pymongo
|
||||
import sklearn.cluster
|
||||
import pandas
|
||||
import numpy
|
||||
import sys, time
|
||||
#import pp
|
||||
import fangjia
|
||||
import datetime
|
||||
import re
|
||||
import pandas
|
||||
|
||||
import dateutil
|
||||
import time
|
||||
import fangjia
|
||||
import config
|
||||
import datetime
|
||||
import pytz
|
||||
import pandas as pd
|
||||
import calendar
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def ghhh(kkg): #555555555555555555555555555
|
||||
kkg=kkg.fillna('无')
|
||||
a=list()
|
||||
for i in kkg.index:
|
||||
b=list(kkg.loc[[i],:].values.flatten())
|
||||
a.append(b)
|
||||
return a
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def f_del(qq,g):
|
||||
|
||||
if g!= None:
|
||||
|
||||
return qq
|
||||
else:
|
||||
del qq['region']
|
||||
return qq
|
||||
|
||||
def sample_pool(ct,rl,timd,timm):
|
||||
|
||||
query = {"city":ct,"cat":"rent","region":rl,
|
||||
"p_date":{"$gt":timd,"$lt":timm}}
|
||||
#query=f_del(query,rl)
|
||||
|
||||
fields = {"total_price":1,"area":1, "city":1,"region":1,'p_date':1,
|
||||
"district_name":1,"rent_type":1,"fitment":1,"block":1}
|
||||
#lt= seawater.count(query)
|
||||
###print(lt)
|
||||
#pos = list()
|
||||
#for s in seawater.find(query,fields1).skip(10000).limit(lt):
|
||||
return fangjia.rent_sample(query, fields)
|
||||
|
||||
def k_means(pp):
|
||||
from sklearn.cluster import KMeans
|
||||
pv=list(pp.dropna())
|
||||
gf=numpy.array([pv]).T
|
||||
estimator = KMeans(n_clusters=3)#构造聚类器
|
||||
estimator.fit(gf)#聚类
|
||||
label_pred = estimator.labels_ #获取聚类标签
|
||||
aa=collections.Counter(label_pred)
|
||||
v=pandas.Series(aa)
|
||||
gg=list(v)
|
||||
index_max=gg.index(max(gg))
|
||||
#yy=np.array([label_pred]).T
|
||||
#un=np.hstack((gf,yy))
|
||||
#uv=pd.DataFrame(un,columns=['price','label'])
|
||||
centroids = estimator.cluster_centers_ #获取聚类中心
|
||||
#inertia = estimator.inertia_ # 获取聚类准则的总和
|
||||
center=centroids[index_max][0]
|
||||
del pv,gf,aa,v,gg,centroids
|
||||
return (round(center,2))
|
||||
|
||||
|
||||
|
||||
|
||||
def get_weekend():
|
||||
|
||||
#from dateutil.parser import parse
|
||||
#bb=parse(tt)
|
||||
now_time=datetime.datetime.now()
|
||||
before_time = now_time-datetime.timedelta(days=14)
|
||||
pacific = pytz.timezone('Asia/Shanghai')
|
||||
return pacific.localize( before_time )
|
||||
|
||||
|
||||
|
||||
def disanganf_price(ct,rl,lk,timd,timm):
|
||||
|
||||
fields={'calc_value':1,'weekend':1,'district_name':1}
|
||||
query = {"city":ct,"region":rl,
|
||||
"district_name":{'$in':lk},'weekend':{'$gt':get_weekend()}}
|
||||
#pos = list()
|
||||
#数据转化为数组,数组的元素为字典
|
||||
|
||||
return fangjia.rent_disanganf_series(query, fields)
|
||||
|
||||
|
||||
|
||||
|
||||
def rent_partial_sell(ct,rl,timd,timm):
|
||||
|
||||
rele0=sample_pool(ct,rl,timd,timm)
|
||||
##print(rele0)
|
||||
rele1=rele0[['district_name','p_date','area','total_price']]
|
||||
rele1['total_price']=rele1['total_price']/rele1['area']
|
||||
#print('up_rele.shape[0]=',rele1.shape[0])
|
||||
|
||||
rele=rele0.drop_duplicates(['district_name'])
|
||||
|
||||
p1=rele['district_name'].dropna()
|
||||
p2=list(set(p1))
|
||||
|
||||
###print('*****************************************************************')
|
||||
dd=disanganf_price(ct,rl,p2,timd,timm)
|
||||
###print('rele.shape[0]=',rele.shape[0])
|
||||
|
||||
if rele.shape[0]>10:
|
||||
|
||||
rele['total_price']=rele['total_price']/rele['area']
|
||||
#rent_most=k_means(rele['total_price'])
|
||||
rele['sell_rent_ratio']=1
|
||||
rv=rele[['total_price','district_name','area','sell_rent_ratio']].dropna()
|
||||
rv1=rv.values
|
||||
ro=[]
|
||||
sell_rent_ratio=[]
|
||||
for k in range(len(rv1)):
|
||||
namee=rv1[k][1]
|
||||
pee1= rele1[rele1['district_name']==namee]
|
||||
pee2=pee1.sort_values(by='p_date',ascending=False)
|
||||
price=pee2['total_price'].iloc[0]
|
||||
ro.append(price)
|
||||
#price=rv1[k][0]
|
||||
if namee in dd.index:
|
||||
|
||||
sell=dd[namee]
|
||||
ppoo=sell/price
|
||||
sell_rent_ratio.append(ppoo)
|
||||
#rv1[k][3]=sell/price
|
||||
###print('rele0=',rele)
|
||||
rent_most=k_means(pandas.Series(ro))
|
||||
#rele=pd.DataFrame(rv1,columns=rv.columns)
|
||||
roseries=pandas.Series(sell_rent_ratio)
|
||||
|
||||
roseries=roseries[roseries>5]
|
||||
|
||||
#rele=rele[rele['sell_rent_ratio']>5]
|
||||
##print('sell_rent_ratio=',list(rele['sell_rent_ratio'])[:5])
|
||||
##print(rele)
|
||||
##print('#######################')
|
||||
##print(rele.shape[0])
|
||||
if len(roseries)>10:
|
||||
|
||||
#rent_most=k_means(rele['total_price'])
|
||||
|
||||
##print('rele.shape[0]=',rele.shape[0])
|
||||
#print('yyyyyyyyyyyyyyyyyyyyyy=',len(roseries))
|
||||
rent_sell=k_means(roseries.dropna())
|
||||
|
||||
del rele
|
||||
return rent_sell,rent_most
|
||||
else:
|
||||
del rele
|
||||
return '没有区域租售比',rent_most
|
||||
else:
|
||||
del rele
|
||||
return '没有区域租售比','没有区域租金数据'
|
||||
|
||||
def dff(i,j,timd,timm): #666666666666666666666666666666666666666666666666666666
|
||||
reef=rent_partial_sell(i,j,timd,timm)
|
||||
##print('$$$$$$$$$$')
|
||||
if type(reef[0])!=str:
|
||||
return reef
|
||||
else:
|
||||
##print('&&&&&&&&&&&&&')
|
||||
return rent_partial_sell(i,None,timd,timm)
|
||||
|
||||
|
||||
def add_eatype(ppd):
|
||||
if 'estate_type2' not in ppd.columns:
|
||||
ppd['estate_type2']='普通住宅'
|
||||
|
||||
return ppd
|
||||
|
||||
|
||||
|
||||
def timestamp_to_int(dd):
|
||||
|
||||
dd['week1']=1
|
||||
|
||||
for i in dd.index:
|
||||
|
||||
f1=dd['weekend'][i].strftime("%Y-%m-%d")
|
||||
f2 = re.sub("\D", "", f1)
|
||||
f3=int(f2)
|
||||
dd['week1'][i]=f3
|
||||
###print(dd)
|
||||
return dd
|
||||
|
||||
'''
|
||||
def before_month_lastday(ti):
|
||||
|
||||
today=dateutil.parser.parse(str(ti))
|
||||
|
||||
#first = datetime.date(day=1, month=today.month, year=today.year)
|
||||
|
||||
lastMonth = today - datetime.timedelta(days=35)
|
||||
|
||||
def plus(k):
|
||||
if k<10:
|
||||
return '0'+str(k)
|
||||
else:
|
||||
return str(k)
|
||||
y=lastMonth.year
|
||||
m=lastMonth.month
|
||||
#d=lastMonth.day
|
||||
day=calendar.monthrange(y,m)[1]
|
||||
|
||||
cc=str(y)+plus(m)+plus(day)
|
||||
|
||||
return int(cc)
|
||||
'''
|
||||
|
||||
|
||||
def before_month_lastday(ti,num):
|
||||
from dateutil.parser import parse
|
||||
today=dateutil.parser.parse(str(ti))
|
||||
|
||||
#first = datetime.date(day=1, month=today.month, year=today.year)
|
||||
|
||||
lastMonth = today - datetime.timedelta(days=num)
|
||||
|
||||
def plus(k):
|
||||
if k<10:
|
||||
return '0'+str(k)
|
||||
else:
|
||||
return str(k)
|
||||
y=lastMonth.year
|
||||
m=lastMonth.month
|
||||
#d=lastMonth.day
|
||||
day=calendar.monthrange(y,m)[1]
|
||||
|
||||
cc=str(y)+plus(m)+plus(day)
|
||||
bb=parse(cc)
|
||||
pacific = pytz.timezone('Asia/Shanghai')
|
||||
return pacific.localize(bb)
|
||||
|
||||
|
||||
def region_make_up_price(ct,rl,lk,ti):
|
||||
|
||||
lgg=fangjia.get_collection("fangjia_stat",config.get_sqm_rentprice_db(), config.get_mongodb_host(),27017)
|
||||
#save_lg=client136.fangjia_stat.test_lg5
|
||||
|
||||
cc=before_month_lastday(ti,32)
|
||||
#print('ti=',ti)
|
||||
#print('cc=',cc)
|
||||
|
||||
#stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017)
|
||||
stat=lgg
|
||||
query = {"city":ct,"region":rl,"cat":"region",'weekend':cc,'district_name':None}
|
||||
|
||||
pos1=list(stat.find(query).sort([("weekend",-1)]).limit(3))
|
||||
|
||||
pos=pandas.DataFrame(pos1)
|
||||
|
||||
#pos=timestamp_to_int(pos)
|
||||
#pos=pos[pos['week1']>cc]
|
||||
|
||||
##print('5'*10)
|
||||
##print('uuuuuuuuuuuuuuuu=',pos)
|
||||
|
||||
|
||||
if pos.shape[0]>=1:
|
||||
pos=pos.sort_values(by='weekend',ascending=False)
|
||||
pp1=pos['avg_price'].iloc[0]
|
||||
#print('pp1=',pp1)
|
||||
if type(pp1)!=None and type(pp1)!=str:
|
||||
return pp1
|
||||
else:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
|
||||
def return_operate(ppu,mm):
|
||||
|
||||
if ppu==None:
|
||||
return 0
|
||||
else:
|
||||
return round(10000*(mm-ppu)/ppu,2)
|
||||
|
||||
'''
|
||||
def before_month_lastday(ti):
|
||||
|
||||
today=dateutil.parser.parse(str(ti))
|
||||
|
||||
#first = datetime.date(day=1, month=today.month, year=today.year)
|
||||
|
||||
lastMonth = today - datetime.timedelta(days=31)
|
||||
|
||||
cc=str(lastMonth.year)+str(lastMonth.month)+str(lastMonth.day)
|
||||
|
||||
return int(cc)
|
||||
'''
|
||||
|
||||
#dd1=sample_pool(ct,rl,timd,timm)
|
||||
|
||||
|
||||
|
||||
def save_region_feature(ct,rl,timd,timm):
|
||||
a=time.time()
|
||||
region_feature=dff(ct,rl,timd,timm)
|
||||
|
||||
#print('region_feature=',region_feature)
|
||||
|
||||
mmup=region_make_up_price(ct,rl,None,timm)
|
||||
#print('mmup_region=',mmup)
|
||||
#print('this month region price',region_feature[1])
|
||||
make_up=return_operate(mmup,region_feature[1])
|
||||
date=datetime.datetime.now()
|
||||
|
||||
|
||||
|
||||
|
||||
def get_weekend(tt):
|
||||
|
||||
from dateutil.parser import parse
|
||||
bb=parse(tt)
|
||||
pacific = pytz.timezone('Asia/Shanghai')
|
||||
return pacific.localize(bb)
|
||||
|
||||
|
||||
def month_lastday(ti):
|
||||
from dateutil.parser import parse
|
||||
today=dateutil.parser.parse(str(ti))
|
||||
|
||||
#first = datetime.date(day=1, month=today.month, year=today.year)
|
||||
|
||||
lastMonth = today# - datetime.timedelta(days=35)
|
||||
|
||||
def plus(k):
|
||||
if k<10:
|
||||
return '0'+str(k)
|
||||
else:
|
||||
return str(k)
|
||||
y=lastMonth.year
|
||||
m=lastMonth.month
|
||||
#d=lastMonth.day
|
||||
day=calendar.monthrange(y,m)[1]
|
||||
|
||||
cc=str(y)+plus(m)+plus(day)
|
||||
bb=parse(cc)
|
||||
pacific = pytz.timezone('Asia/Shanghai')
|
||||
return pacific.localize(bb)
|
||||
|
||||
timme=month_lastday(timm)
|
||||
##print('region_timme=',timme)
|
||||
|
||||
|
||||
#stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017)
|
||||
##print("######################## ","ct=",ct,"rl=",rl,"makeup=",make_up)
|
||||
|
||||
|
||||
#save_lg = fangjia.get_collection("fangjia_stat", "rent_1221", config.get_district_stat_host(), config.get_district_stat_port())
|
||||
avg=region_feature[1]
|
||||
if type(avg)!=str:
|
||||
avg=float(avg)
|
||||
save_lg = fangjia.get_collection("fangjia_stat",config.get_sqm_rentprice_db(), config.get_mongodb_host(),27017)
|
||||
|
||||
save_lg.replace_one(
|
||||
|
||||
{"city":ct,"region":rl,"weekend":timme},
|
||||
|
||||
{"city":ct,"region":rl,"district_name":None,"make_up":make_up,'ratio':region_feature[0],
|
||||
|
||||
"avg_price":region_feature[1], "c_date":date,"weekend":timme,"cat":'region'},True
|
||||
)
|
||||
|
||||
##print(time.time()-a)
|
||||
|
||||
###print(len(disname),len(disname))
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user