This commit is contained in:
wangwenyan@fangjia.com
2018-12-05 18:10:13 +08:00
parent 6c21676583
commit 04eaf142f2
2 changed files with 306 additions and 271 deletions
+304 -269
View File
@@ -13,12 +13,13 @@ import pymongo
#from pymongo import MongoClient
import numpy as np
import random
import collections
import pymongo
import collections
import pymongo
import sklearn.cluster
import pandas
import numpy
import sys, time
import sys
import time
#import pp
import fangjia
import datetime
@@ -29,21 +30,66 @@ import dateutil
import time
import fangjia
import config
import datetime
import pytz
import datetime
import pytz
import pandas as pd
import calendar
def ghhh(kkg): #555555555555555555555555555
kkg=kkg.fillna('无')
a=list()
import pandas as pd
from elasticsearch import Elasticsearch, helpers
client_DB_es = Elasticsearch('192.168.0.206', port=9200, timeout=1000, http_auth=(
'developer', '5e8edd851d2fdfbd7415232c67367cc3'))
ES_db_index = "seawater-2018,seawater-2017,seawater-2016"
def findHouse_region(ct, rl, timd, timm, fields):
# local_switch and select_content are not applied in this function
# query = {"city": ct, "cat": "rent", "region": rl, # 'rent_type':'整租',
# "district_name": {"$in": lk},
# "p_date": {"$gt": timd, "$lt": timm}}
city = ct
region = rl
# district_name = lk[0]
# name_pack = findquery_statement['name_pack']
query_list =client_DB_es.search(
index=ES_db_index,
scroll='20m',
size=100,
doc_type="_doc",
_source = fields,
body={
"query":
{"bool":
{"must":
[{"term": {"cat": "rent"}},
{"term": {"city": city}},
{"term": {"region": region}},
# {"term": {"district_name": district_name}},
#{"terms": {"district_name": lk}},
# {"terms": {"district_name": ["新泾三村", "新泾四村"]}},
{"range": {"p_date": {
"gte": timd, "lte": timm}}}
], "must_not": [], "should": []
}
}, "from": 0, "size": 100, "sort": [], "aggs": {}}
)
query_list_raw = query_list['hits']['hits']
query_pack = []
for singlequery in query_list_raw:
query_pack.append(singlequery['_source'])
return pd.DataFrame(query_pack)
def ghhh(kkg): # 555555555555555555555555555
kkg = kkg.fillna('无')
a = list()
for i in kkg.index:
b=list(kkg.loc[[i],:].values.flatten())
b = list(kkg.loc[[i], :].values.flatten())
a.append(b)
return a
return a
'''
@@ -58,180 +104,178 @@ def f_del(qq,g):
'''
def sample_pool(ct,rl,timd,timm):
query = {"city":ct,"cat":"rent","region":rl,
"p_date":{"$gt":timd,"$lt":timm}}
#query=f_del(query,rl)
fields = {"total_price":1,"area":1, "city":1,"region":1,'p_date':1,
"district_name":1,"rent_type":1,"fitment":1,"block":1}
def sample_pool(ct, rl, timd, timm):
query = {"city": ct, "cat": "rent", "region": rl,
"p_date": {"$gt": timd, "$lt": timm}}
# query=f_del(query,rl)
# fields = {"total_price":1,"area":1, "city":1,"region":1,'p_date':1,
# "district_name":1,"rent_type":1,"fitment":1,"block":1}
fields = ["total_price", "area", "city", "region", "p_date",
"district_name", "rent_type", "fitment", "block"]
#lt= seawater.count(query)
####print(lt)
# print(lt)
#pos = list()
#for s in seawater.find(query,fields1).skip(10000).limit(lt):
#print('query=',query)
return fangjia.rent_sample(query, fields)
# for s in seawater.find(query,fields1).skip(10000).limit(lt):
# print('query=',query)
# return fangjia.rent_sample(query, fields)
return findHouse_region(ct, rl, timd, timm, fields)
def k_means(pp):
from sklearn.cluster import KMeans
pv=list(pp.dropna())
gf=numpy.array([pv]).T
estimator = KMeans(n_clusters=3)#构造聚类器
estimator.fit(gf)#聚类
label_pred = estimator.labels_ #获取聚类标签
aa=collections.Counter(label_pred)
v=pandas.Series(aa)
gg=list(v)
index_max=gg.index(max(gg))
#yy=np.array([label_pred]).T
#un=np.hstack((gf,yy))
#uv=pd.DataFrame(un,columns=['price','label'])
centroids = estimator.cluster_centers_ #获取聚类中心
#inertia = estimator.inertia_ # 获取聚类准则的总和
center=centroids[index_max][0]
del pv,gf,aa,v,gg,centroids
return (round(center,2))
pv = list(pp.dropna())
gf = numpy.array([pv]).T
estimator = KMeans(n_clusters=3) # 构造聚类器
estimator.fit(gf) # 聚类
label_pred = estimator.labels_ # 获取聚类标签
aa = collections.Counter(label_pred)
v = pandas.Series(aa)
gg = list(v)
index_max = gg.index(max(gg))
# yy=np.array([label_pred]).T
# un=np.hstack((gf,yy))
# uv=pd.DataFrame(un,columns=['price','label'])
centroids = estimator.cluster_centers_ # 获取聚类中心
# inertia = estimator.inertia_ # 获取聚类准则的总和
center = centroids[index_max][0]
del pv, gf, aa, v, gg, centroids
return (round(center, 2))
def get_weekend():
#from dateutil.parser import parse
#bb=parse(tt)
now_time=datetime.datetime.now()
before_time = now_time-datetime.timedelta(days=21)
# bb=parse(tt)
now_time = datetime.datetime.now()
before_time = now_time - datetime.timedelta(days=21)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize( before_time )
return pacific.localize(before_time)
def disanganf_price(ct, rl, lk, timd, timm):
def disanganf_price(ct,rl,lk,timd,timm):
fields={'calc_value':1,'weekend':1,'district_name':1}
query = {"city":ct,"region":rl,
"district_name":{'$in':lk},'weekend':{'$gt':get_weekend()}}
fields = {'calc_value': 1, 'weekend': 1, 'district_name': 1}
query = {"city": ct, "region": rl,
"district_name": {'$in': lk}, 'weekend': {'$gt': get_weekend()}}
#pos = list()
#数据转化为数组,数组的元素为字典
# 数据转化为数组,数组的元素为字典
return fangjia.rent_disanganf_series(query, fields)
def rent_partial_sell(ct, rl, timd, timm):
def rent_partial_sell(ct,rl,timd,timm):
rele0=sample_pool(ct,rl,timd,timm)
##print('rele0=',rele0)
if rele0.shape[0]>0:
rele1=rele0[['district_name','p_date','area','total_price']]
rele1['total_price']=rele1['total_price']/rele1['area']
##print('up_rele.shape[0]=',rele1.shape[0])
rele=rele0.drop_duplicates(['district_name'])
p1=rele['district_name'].dropna()
p2=list(set(p1))
#print('p2=',p2)
####print('*****************************************************************')
dd=disanganf_price(ct,rl,p2,timd,timm)
##print('rele.shape[0]=',rele.shape[0])
#print('dd=',dd)
if rele.shape[0]>10:
rele['total_price']=rele['total_price']/rele['area']
#rent_most=k_means(rele['total_price'])
rele['sell_rent_ratio']=1
rv=rele[['total_price','district_name','area','sell_rent_ratio']].dropna()
rv1=rv.values
ro=[]
sell_rent_ratio=[]
rele0 = sample_pool(ct, rl, timd, timm)
# print('rele0=',rele0)
if rele0.shape[0] > 0:
rele1 = rele0[['district_name', 'p_date', 'area', 'total_price']]
rele1['total_price'] = rele1['total_price'] / rele1['area']
# print('up_rele.shape[0]=',rele1.shape[0])
rele = rele0.drop_duplicates(['district_name'])
p1 = rele['district_name'].dropna()
p2 = list(set(p1))
# print('p2=',p2)
# print('*****************************************************************')
dd = disanganf_price(ct, rl, p2, timd, timm)
# print('rele.shape[0]=',rele.shape[0])
# print('dd=',dd)
if rele.shape[0] > 10:
rele['total_price'] = rele['total_price'] / rele['area']
# rent_most=k_means(rele['total_price'])
rele['sell_rent_ratio'] = 1
rv = rele[['total_price', 'district_name',
'area', 'sell_rent_ratio']].dropna()
rv1 = rv.values
ro = []
sell_rent_ratio = []
for k in range(len(rv1)):
namee=rv1[k][1]
pee1= rele1[rele1['district_name']==namee]
pee2=pee1.sort_values(by='p_date',ascending=False)
price=pee2['total_price'].iloc[0]
namee = rv1[k][1]
pee1 = rele1[rele1['district_name'] == namee]
pee2 = pee1.sort_values(by='p_date', ascending=False)
price = pee2['total_price'].iloc[0]
ro.append(price)
#price=rv1[k][0]
# price=rv1[k][0]
if namee in dd.index:
#print(namee)
sell=dd[namee]
ppoo=sell/price
# print(namee)
sell = dd[namee]
ppoo = sell / price
sell_rent_ratio.append(ppoo)
#rv1[k][3]=sell/price
##print('rele0=',rele)
rent_most=k_means(pandas.Series(ro))
#rele=pd.DataFrame(rv1,columns=rv.columns)
roseries=pandas.Series(sell_rent_ratio)
roseries=roseries[roseries>5]
#rele=rele[rele['sell_rent_ratio']>5]
###print('sell_rent_ratio=',list(rele['sell_rent_ratio'])[:5])
###print(rele)
###print('#######################')
###print(rele.shape[0])
#print('len(roseries)=',len(roseries))
#print(roseries)
if len(roseries)>10:
#rent_most=k_means(rele['total_price'])
###print('rele.shape[0]=',rele.shape[0])
##print('yyyyyyyyyyyyyyyyyyyyyy=',len(roseries))
rent_sell=k_means(roseries.dropna())
# rv1[k][3]=sell/price
# print('rele0=',rele)
rent_most = k_means(pandas.Series(ro))
# rele=pd.DataFrame(rv1,columns=rv.columns)
roseries = pandas.Series(sell_rent_ratio)
roseries = roseries[roseries > 5]
# rele=rele[rele['sell_rent_ratio']>5]
# print('sell_rent_ratio=',list(rele['sell_rent_ratio'])[:5])
# print(rele)
# print('#######################')
# print(rele.shape[0])
# print('len(roseries)=',len(roseries))
# print(roseries)
if len(roseries) > 10:
# rent_most=k_means(rele['total_price'])
# print('rele.shape[0]=',rele.shape[0])
# print('yyyyyyyyyyyyyyyyyyyyyy=',len(roseries))
rent_sell = k_means(roseries.dropna())
del rele
return rent_sell,rent_most
return rent_sell, rent_most
else:
del rele
return -1,rent_most
return -1, rent_most
else:
del rele
return -1,-1
return -1, -1
else:
return -1,-1
return -1, -1
def dff(i,j,timd,timm): #666666666666666666666666666666666666666666666666666666
reef=rent_partial_sell(i,j,timd,timm)
###print('$$$$$$$$$$')
if reef[0]>0:
def dff(i, j, timd, timm): # 666666666666666666666666666666666666666666666666666666
reef = rent_partial_sell(i, j, timd, timm)
# print('$$$$$$$$$$')
if reef[0] > 0:
return reef
else:
###print('&&&&&&&&&&&&&')
return rent_partial_sell(i,None,timd,timm)
# print('&&&&&&&&&&&&&')
return rent_partial_sell(i, None, timd, timm)
def add_eatype(ppd):
if 'estate_type2' not in ppd.columns:
ppd['estate_type2']='普通住宅'
ppd['estate_type2'] = '普通住宅'
return ppd
def timestamp_to_int(dd):
dd['week1']=1
for i in dd.index:
f1=dd['weekend'][i].strftime("%Y-%m-%d")
dd['week1'] = 1
for i in dd.index:
f1 = dd['weekend'][i].strftime("%Y-%m-%d")
f2 = re.sub("\D", "", f1)
f3=int(f2)
dd['week1'][i]=f3
####print(dd)
f3 = int(f2)
dd['week1'][i] = f3
# print(dd)
return dd
'''
@@ -259,72 +303,74 @@ def before_month_lastday(ti):
'''
def before_month_lastday(ti,num):
def before_month_lastday(ti, num):
from dateutil.parser import parse
today=dateutil.parser.parse(str(ti))
today = dateutil.parser.parse(str(ti))
#first = datetime.date(day=1, month=today.month, year=today.year)
lastMonth = today - datetime.timedelta(days=num)
def plus(k):
if k<10:
return '0'+str(k)
if k < 10:
return '0' + str(k)
else:
return str(k)
y=lastMonth.year
m=lastMonth.month
#d=lastMonth.day
day=calendar.monthrange(y,m)[1]
y = lastMonth.year
m = lastMonth.month
# d=lastMonth.day
day = calendar.monthrange(y, m)[1]
cc=str(y)+plus(m)+plus(day)
bb=parse(cc)
cc = str(y) + plus(m) + plus(day)
bb = parse(cc)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(bb)
def region_make_up_price(ct,rl,lk,ti):
lgg=fangjia.get_collection("fangjia",config.get_sqm_rentprice_db(), config.get_mongodb_host(),27017)
#save_lg=client136.fangjia_stat.test_lg5
cc=before_month_lastday(ti,32)
##print('ti=',ti)
##print('cc=',cc)
def region_make_up_price(ct, rl, lk, ti):
lgg = fangjia.get_collection(
"fangjia", config.get_sqm_rentprice_db(), config.get_mongodb_host(), 27017)
# save_lg=client136.fangjia_stat.test_lg5
cc = before_month_lastday(ti, 32)
# print('ti=',ti)
# print('cc=',cc)
#stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017)
stat=lgg
query = {"city":ct,"region":rl,"cat":"region",'weekend':cc,'district_name':None}
pos1=list(stat.find(query).sort([("weekend",-1)]).limit(3))
pos=pandas.DataFrame(pos1)
#pos=timestamp_to_int(pos)
#pos=pos[pos['week1']>cc]
###print('5'*10)
###print('uuuuuuuuuuuuuuuu=',pos)
if pos.shape[0]>=1:
pos=pos.sort_values(by='weekend',ascending=False)
pp1=pos['avg_price'].iloc[0]
##print('pp1=',pp1)
if type(pp1)!=None and type(pp1)!=str:
stat = lgg
query = {"city": ct, "region": rl, "cat": "region",
'weekend': cc, 'district_name': None}
pos1 = list(stat.find(query).sort([("weekend", -1)]).limit(3))
pos = pandas.DataFrame(pos1)
# pos=timestamp_to_int(pos)
# pos=pos[pos['week1']>cc]
# print('5'*10)
# print('uuuuuuuuuuuuuuuu=',pos)
if pos.shape[0] >= 1:
pos = pos.sort_values(by='weekend', ascending=False)
pp1 = pos['avg_price'].iloc[0]
# print('pp1=',pp1)
if type(pp1) != None and type(pp1) != str:
return pp1
else:
return None
return None
else:
return None
def return_operate(ppu,mm):
if ppu==None:
def return_operate(ppu, mm):
if ppu == None:
return 0
else:
return int(10000*(mm-ppu)/ppu)
return int(10000 * (mm - ppu) / ppu)
'''
def before_month_lastday(ti):
@@ -338,106 +384,95 @@ def before_month_lastday(ti):
return int(cc)
'''
#dd1=sample_pool(ct,rl,timd,timm)
# dd1=sample_pool(ct,rl,timd,timm)
def save_region_feature(ct, rl, timd, timm):
# a=time.time()
region_feature = dff(ct, rl, timd, timm)
def save_region_feature(ct,rl,timd,timm):
#a=time.time()
region_feature=dff(ct,rl,timd,timm)
##print('region_feature=',region_feature)
mmup=region_make_up_price(ct,rl,None,timm)
##print('mmup_region=',mmup)
# print('region_feature=',region_feature)
mmup = region_make_up_price(ct, rl, None, timm)
# print('mmup_region=',mmup)
#print('this month region price',region_feature[1])
markup=return_operate(mmup,region_feature[1])
markup = return_operate(mmup, region_feature[1])
def c_date():
date1=datetime.datetime.now()
date1 = datetime.datetime.now()
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(date1)
def get_weekend(tt):
from dateutil.parser import parse
bb=parse(tt)
bb = parse(tt)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(bb)
def month_lastday(ti):
from dateutil.parser import parse
today=dateutil.parser.parse(str(ti))
today = dateutil.parser.parse(str(ti))
#first = datetime.date(day=1, month=today.month, year=today.year)
lastMonth = today# - datetime.timedelta(days=35)
lastMonth = today # - datetime.timedelta(days=35)
def plus(k):
if k<10:
return '0'+str(k)
if k < 10:
return '0' + str(k)
else:
return str(k)
y=lastMonth.year
m=lastMonth.month
#d=lastMonth.day
day=calendar.monthrange(y,m)[1]
y = lastMonth.year
m = lastMonth.month
# d=lastMonth.day
day = calendar.monthrange(y, m)[1]
cc=str(y)+plus(m)+plus(day)
bb=parse(cc)
cc = str(y) + plus(m) + plus(day)
bb = parse(cc)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(bb)
timme=month_lastday(timm)
###print('region_timme=',timme)
timme = month_lastday(timm)
# print('region_timme=',timme)
#stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017)
###print("######################## ","ct=",ct,"rl=",rl,"makeup=",make_up)
# print("######################## ","ct=",ct,"rl=",rl,"makeup=",make_up)
#save_lg = fangjia.get_collection("fangjia_stat", "rent_1221", config.get_district_stat_host(), config.get_district_stat_port())
date=c_date()
temp = fangjia.get_collection("fangjia",config.get_rent_temp_db(), '192.168.0.136',27017)
date = c_date()
temp = fangjia.get_collection(
"fangjia", config.get_rent_temp_db(), '192.168.0.136', 27017)
temp.replace_one(
{"city":ct,"region":rl,"weekend":timme,"cat":'region'},
{"city":ct,"region":rl,"district_name":None,"markup":markup,'ratio':region_feature[0],
"avg_price":region_feature[1], "c_date":date,"weekend":timme,"cat":'region'},True
)
avg=region_feature[1]
if avg>0:
avg=float(avg)
save_lg = fangjia.get_collection("fangjia",config.get_sqm_rentprice_db(), config.get_mongodb_host(),27017)
{"city": ct, "region": rl, "weekend": timme, "cat": 'region'},
{"city": ct, "region": rl, "district_name": None, "markup": markup, 'ratio': region_feature[0],
"avg_price": region_feature[1], "c_date": date, "weekend": timme, "cat": 'region'}, True
)
avg = region_feature[1]
if avg > 0:
avg = float(avg)
save_lg = fangjia.get_collection(
"fangjia", config.get_sqm_rentprice_db(), config.get_mongodb_host(), 27017)
save_lg.replace_one(
{"city":ct,"region":rl,"weekend":timme,"cat":'region'},
{"city":ct,"region":rl,"district_name":None,"markup":markup,'ratio':region_feature[0],
"avg_price":region_feature[1], "c_date":date,"weekend":timme,"cat":'region'},True
)
###print(time.time()-a)
####print(len(disname),len(disname))
{"city": ct, "region": rl, "weekend": timme, "cat": 'region'},
{"city": ct, "region": rl, "district_name": None, "markup": markup, 'ratio': region_feature[0],
"avg_price": region_feature[1], "c_date": date, "weekend": timme, "cat": 'region'}, True
)
# print(time.time()-a)
# print(len(disname),len(disname))
+2 -2
View File
@@ -22,8 +22,8 @@ rl='闵行'
lk='上海康城'#上海闵行闵浦新苑一村普陀曹杨一村上海北京东城左安漪园
#黄浦伯爵居静安泰安里长宁新泾北苑上海奉贤海岸景苑闵行陇兴别墅阳江雅苑养生城二期
timm=20180430
timd=20170531
timm=20181130
timd=20171130
a=time.time()
rentPrice.save_to_mongo(ct,rl,lk,timd,timm)