This commit is contained in:
wangwenyan@fangjia.com
2018-12-05 18:10:13 +08:00
parent 6c21676583
commit 04eaf142f2
2 changed files with 306 additions and 271 deletions
+304 -269
View File
@@ -13,12 +13,13 @@ import pymongo
#from pymongo import MongoClient #from pymongo import MongoClient
import numpy as np import numpy as np
import random import random
import collections import collections
import pymongo import pymongo
import sklearn.cluster import sklearn.cluster
import pandas import pandas
import numpy import numpy
import sys, time import sys
import time
#import pp #import pp
import fangjia import fangjia
import datetime import datetime
@@ -29,21 +30,66 @@ import dateutil
import time import time
import fangjia import fangjia
import config import config
import datetime import datetime
import pytz import pytz
import pandas as pd import pandas as pd
import calendar import calendar
def ghhh(kkg): #555555555555555555555555555 import pandas as pd
kkg=kkg.fillna('无') from elasticsearch import Elasticsearch, helpers
a=list() client_DB_es = Elasticsearch('192.168.0.206', port=9200, timeout=1000, http_auth=(
'developer', '5e8edd851d2fdfbd7415232c67367cc3'))
ES_db_index = "seawater-2018,seawater-2017,seawater-2016"
def findHouse_region(ct, rl, timd, timm, fields):
# local_switch and select_content are not applied in this function
# query = {"city": ct, "cat": "rent", "region": rl, # 'rent_type':'整租',
# "district_name": {"$in": lk},
# "p_date": {"$gt": timd, "$lt": timm}}
city = ct
region = rl
# district_name = lk[0]
# name_pack = findquery_statement['name_pack']
query_list =client_DB_es.search(
index=ES_db_index,
scroll='20m',
size=100,
doc_type="_doc",
_source = fields,
body={
"query":
{"bool":
{"must":
[{"term": {"cat": "rent"}},
{"term": {"city": city}},
{"term": {"region": region}},
# {"term": {"district_name": district_name}},
#{"terms": {"district_name": lk}},
# {"terms": {"district_name": ["新泾三村", "新泾四村"]}},
{"range": {"p_date": {
"gte": timd, "lte": timm}}}
], "must_not": [], "should": []
}
}, "from": 0, "size": 100, "sort": [], "aggs": {}}
)
query_list_raw = query_list['hits']['hits']
query_pack = []
for singlequery in query_list_raw:
query_pack.append(singlequery['_source'])
return pd.DataFrame(query_pack)
def ghhh(kkg): # 555555555555555555555555555
kkg = kkg.fillna('无')
a = list()
for i in kkg.index: for i in kkg.index:
b=list(kkg.loc[[i],:].values.flatten()) b = list(kkg.loc[[i], :].values.flatten())
a.append(b) a.append(b)
return a return a
''' '''
@@ -58,180 +104,178 @@ def f_del(qq,g):
''' '''
def sample_pool(ct,rl,timd,timm):
query = {"city":ct,"cat":"rent","region":rl, def sample_pool(ct, rl, timd, timm):
"p_date":{"$gt":timd,"$lt":timm}}
#query=f_del(query,rl) query = {"city": ct, "cat": "rent", "region": rl,
"p_date": {"$gt": timd, "$lt": timm}}
fields = {"total_price":1,"area":1, "city":1,"region":1,'p_date':1, # query=f_del(query,rl)
"district_name":1,"rent_type":1,"fitment":1,"block":1}
# fields = {"total_price":1,"area":1, "city":1,"region":1,'p_date':1,
# "district_name":1,"rent_type":1,"fitment":1,"block":1}
fields = ["total_price", "area", "city", "region", "p_date",
"district_name", "rent_type", "fitment", "block"]
#lt= seawater.count(query) #lt= seawater.count(query)
####print(lt) # print(lt)
#pos = list() #pos = list()
#for s in seawater.find(query,fields1).skip(10000).limit(lt): # for s in seawater.find(query,fields1).skip(10000).limit(lt):
#print('query=',query) # print('query=',query)
return fangjia.rent_sample(query, fields) # return fangjia.rent_sample(query, fields)
return findHouse_region(ct, rl, timd, timm, fields)
def k_means(pp): def k_means(pp):
from sklearn.cluster import KMeans from sklearn.cluster import KMeans
pv=list(pp.dropna()) pv = list(pp.dropna())
gf=numpy.array([pv]).T gf = numpy.array([pv]).T
estimator = KMeans(n_clusters=3)#构造聚类器 estimator = KMeans(n_clusters=3) # 构造聚类器
estimator.fit(gf)#聚类 estimator.fit(gf) # 聚类
label_pred = estimator.labels_ #获取聚类标签 label_pred = estimator.labels_ # 获取聚类标签
aa=collections.Counter(label_pred) aa = collections.Counter(label_pred)
v=pandas.Series(aa) v = pandas.Series(aa)
gg=list(v) gg = list(v)
index_max=gg.index(max(gg)) index_max = gg.index(max(gg))
#yy=np.array([label_pred]).T # yy=np.array([label_pred]).T
#un=np.hstack((gf,yy)) # un=np.hstack((gf,yy))
#uv=pd.DataFrame(un,columns=['price','label']) # uv=pd.DataFrame(un,columns=['price','label'])
centroids = estimator.cluster_centers_ #获取聚类中心 centroids = estimator.cluster_centers_ # 获取聚类中心
#inertia = estimator.inertia_ # 获取聚类准则的总和 # inertia = estimator.inertia_ # 获取聚类准则的总和
center=centroids[index_max][0] center = centroids[index_max][0]
del pv,gf,aa,v,gg,centroids del pv, gf, aa, v, gg, centroids
return (round(center,2)) return (round(center, 2))
def get_weekend(): def get_weekend():
#from dateutil.parser import parse #from dateutil.parser import parse
#bb=parse(tt) # bb=parse(tt)
now_time=datetime.datetime.now() now_time = datetime.datetime.now()
before_time = now_time-datetime.timedelta(days=21) before_time = now_time - datetime.timedelta(days=21)
pacific = pytz.timezone('Asia/Shanghai') pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize( before_time ) return pacific.localize(before_time)
def disanganf_price(ct, rl, lk, timd, timm):
def disanganf_price(ct,rl,lk,timd,timm): fields = {'calc_value': 1, 'weekend': 1, 'district_name': 1}
query = {"city": ct, "region": rl,
fields={'calc_value':1,'weekend':1,'district_name':1} "district_name": {'$in': lk}, 'weekend': {'$gt': get_weekend()}}
query = {"city":ct,"region":rl,
"district_name":{'$in':lk},'weekend':{'$gt':get_weekend()}}
#pos = list() #pos = list()
#数据转化为数组,数组的元素为字典 # 数据转化为数组,数组的元素为字典
return fangjia.rent_disanganf_series(query, fields) return fangjia.rent_disanganf_series(query, fields)
def rent_partial_sell(ct, rl, timd, timm):
def rent_partial_sell(ct,rl,timd,timm): rele0 = sample_pool(ct, rl, timd, timm)
# print('rele0=',rele0)
rele0=sample_pool(ct,rl,timd,timm)
##print('rele0=',rele0) if rele0.shape[0] > 0:
if rele0.shape[0]>0: rele1 = rele0[['district_name', 'p_date', 'area', 'total_price']]
rele1['total_price'] = rele1['total_price'] / rele1['area']
# print('up_rele.shape[0]=',rele1.shape[0])
rele1=rele0[['district_name','p_date','area','total_price']]
rele1['total_price']=rele1['total_price']/rele1['area'] rele = rele0.drop_duplicates(['district_name'])
##print('up_rele.shape[0]=',rele1.shape[0]) p1 = rele['district_name'].dropna()
p2 = list(set(p1))
# print('p2=',p2)
rele=rele0.drop_duplicates(['district_name']) # print('*****************************************************************')
p1=rele['district_name'].dropna() dd = disanganf_price(ct, rl, p2, timd, timm)
p2=list(set(p1)) # print('rele.shape[0]=',rele.shape[0])
#print('p2=',p2)
####print('*****************************************************************') # print('dd=',dd)
dd=disanganf_price(ct,rl,p2,timd,timm)
##print('rele.shape[0]=',rele.shape[0]) if rele.shape[0] > 10:
#print('dd=',dd) rele['total_price'] = rele['total_price'] / rele['area']
# rent_most=k_means(rele['total_price'])
if rele.shape[0]>10: rele['sell_rent_ratio'] = 1
rv = rele[['total_price', 'district_name',
rele['total_price']=rele['total_price']/rele['area'] 'area', 'sell_rent_ratio']].dropna()
#rent_most=k_means(rele['total_price']) rv1 = rv.values
rele['sell_rent_ratio']=1 ro = []
rv=rele[['total_price','district_name','area','sell_rent_ratio']].dropna() sell_rent_ratio = []
rv1=rv.values
ro=[]
sell_rent_ratio=[]
for k in range(len(rv1)): for k in range(len(rv1)):
namee=rv1[k][1] namee = rv1[k][1]
pee1= rele1[rele1['district_name']==namee] pee1 = rele1[rele1['district_name'] == namee]
pee2=pee1.sort_values(by='p_date',ascending=False) pee2 = pee1.sort_values(by='p_date', ascending=False)
price=pee2['total_price'].iloc[0] price = pee2['total_price'].iloc[0]
ro.append(price) ro.append(price)
#price=rv1[k][0] # price=rv1[k][0]
if namee in dd.index: if namee in dd.index:
#print(namee) # print(namee)
sell=dd[namee] sell = dd[namee]
ppoo=sell/price ppoo = sell / price
sell_rent_ratio.append(ppoo) sell_rent_ratio.append(ppoo)
#rv1[k][3]=sell/price # rv1[k][3]=sell/price
##print('rele0=',rele) # print('rele0=',rele)
rent_most=k_means(pandas.Series(ro)) rent_most = k_means(pandas.Series(ro))
#rele=pd.DataFrame(rv1,columns=rv.columns) # rele=pd.DataFrame(rv1,columns=rv.columns)
roseries=pandas.Series(sell_rent_ratio) roseries = pandas.Series(sell_rent_ratio)
roseries=roseries[roseries>5] roseries = roseries[roseries > 5]
#rele=rele[rele['sell_rent_ratio']>5] # rele=rele[rele['sell_rent_ratio']>5]
###print('sell_rent_ratio=',list(rele['sell_rent_ratio'])[:5]) # print('sell_rent_ratio=',list(rele['sell_rent_ratio'])[:5])
###print(rele) # print(rele)
###print('#######################') # print('#######################')
###print(rele.shape[0]) # print(rele.shape[0])
#print('len(roseries)=',len(roseries)) # print('len(roseries)=',len(roseries))
#print(roseries) # print(roseries)
if len(roseries)>10: if len(roseries) > 10:
#rent_most=k_means(rele['total_price']) # rent_most=k_means(rele['total_price'])
###print('rele.shape[0]=',rele.shape[0]) # print('rele.shape[0]=',rele.shape[0])
##print('yyyyyyyyyyyyyyyyyyyyyy=',len(roseries)) # print('yyyyyyyyyyyyyyyyyyyyyy=',len(roseries))
rent_sell=k_means(roseries.dropna()) rent_sell = k_means(roseries.dropna())
del rele del rele
return rent_sell,rent_most return rent_sell, rent_most
else: else:
del rele del rele
return -1,rent_most return -1, rent_most
else: else:
del rele del rele
return -1,-1 return -1, -1
else: else:
return -1,-1 return -1, -1
def dff(i,j,timd,timm): #666666666666666666666666666666666666666666666666666666
reef=rent_partial_sell(i,j,timd,timm) def dff(i, j, timd, timm): # 666666666666666666666666666666666666666666666666666666
###print('$$$$$$$$$$') reef = rent_partial_sell(i, j, timd, timm)
# print('$$$$$$$$$$')
if reef[0]>0:
if reef[0] > 0:
return reef return reef
else: else:
###print('&&&&&&&&&&&&&') # print('&&&&&&&&&&&&&')
return rent_partial_sell(i,None,timd,timm) return rent_partial_sell(i, None, timd, timm)
def add_eatype(ppd): def add_eatype(ppd):
if 'estate_type2' not in ppd.columns: if 'estate_type2' not in ppd.columns:
ppd['estate_type2']='普通住宅' ppd['estate_type2'] = '普通住宅'
return ppd return ppd
def timestamp_to_int(dd): def timestamp_to_int(dd):
dd['week1']=1 dd['week1'] = 1
for i in dd.index: for i in dd.index:
f1=dd['weekend'][i].strftime("%Y-%m-%d") f1 = dd['weekend'][i].strftime("%Y-%m-%d")
f2 = re.sub("\D", "", f1) f2 = re.sub("\D", "", f1)
f3=int(f2) f3 = int(f2)
dd['week1'][i]=f3 dd['week1'][i] = f3
####print(dd) # print(dd)
return dd return dd
''' '''
@@ -259,72 +303,74 @@ def before_month_lastday(ti):
''' '''
def before_month_lastday(ti,num): def before_month_lastday(ti, num):
from dateutil.parser import parse from dateutil.parser import parse
today=dateutil.parser.parse(str(ti)) today = dateutil.parser.parse(str(ti))
#first = datetime.date(day=1, month=today.month, year=today.year) #first = datetime.date(day=1, month=today.month, year=today.year)
lastMonth = today - datetime.timedelta(days=num) lastMonth = today - datetime.timedelta(days=num)
def plus(k): def plus(k):
if k<10: if k < 10:
return '0'+str(k) return '0' + str(k)
else: else:
return str(k) return str(k)
y=lastMonth.year y = lastMonth.year
m=lastMonth.month m = lastMonth.month
#d=lastMonth.day # d=lastMonth.day
day=calendar.monthrange(y,m)[1] day = calendar.monthrange(y, m)[1]
cc=str(y)+plus(m)+plus(day) cc = str(y) + plus(m) + plus(day)
bb=parse(cc) bb = parse(cc)
pacific = pytz.timezone('Asia/Shanghai') pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(bb) return pacific.localize(bb)
def region_make_up_price(ct,rl,lk,ti): def region_make_up_price(ct, rl, lk, ti):
lgg=fangjia.get_collection("fangjia",config.get_sqm_rentprice_db(), config.get_mongodb_host(),27017) lgg = fangjia.get_collection(
#save_lg=client136.fangjia_stat.test_lg5 "fangjia", config.get_sqm_rentprice_db(), config.get_mongodb_host(), 27017)
# save_lg=client136.fangjia_stat.test_lg5
cc=before_month_lastday(ti,32)
##print('ti=',ti) cc = before_month_lastday(ti, 32)
##print('cc=',cc) # print('ti=',ti)
# print('cc=',cc)
#stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) #stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017)
stat=lgg stat = lgg
query = {"city":ct,"region":rl,"cat":"region",'weekend':cc,'district_name':None} query = {"city": ct, "region": rl, "cat": "region",
'weekend': cc, 'district_name': None}
pos1=list(stat.find(query).sort([("weekend",-1)]).limit(3))
pos1 = list(stat.find(query).sort([("weekend", -1)]).limit(3))
pos=pandas.DataFrame(pos1)
pos = pandas.DataFrame(pos1)
#pos=timestamp_to_int(pos)
#pos=pos[pos['week1']>cc] # pos=timestamp_to_int(pos)
# pos=pos[pos['week1']>cc]
###print('5'*10)
###print('uuuuuuuuuuuuuuuu=',pos) # print('5'*10)
# print('uuuuuuuuuuuuuuuu=',pos)
if pos.shape[0]>=1: if pos.shape[0] >= 1:
pos=pos.sort_values(by='weekend',ascending=False) pos = pos.sort_values(by='weekend', ascending=False)
pp1=pos['avg_price'].iloc[0] pp1 = pos['avg_price'].iloc[0]
##print('pp1=',pp1) # print('pp1=',pp1)
if type(pp1)!=None and type(pp1)!=str: if type(pp1) != None and type(pp1) != str:
return pp1 return pp1
else: else:
return None return None
else: else:
return None return None
def return_operate(ppu,mm):
def return_operate(ppu, mm):
if ppu==None:
if ppu == None:
return 0 return 0
else: else:
return int(10000*(mm-ppu)/ppu) return int(10000 * (mm - ppu) / ppu)
''' '''
def before_month_lastday(ti): def before_month_lastday(ti):
@@ -338,106 +384,95 @@ def before_month_lastday(ti):
return int(cc) return int(cc)
''' '''
#dd1=sample_pool(ct,rl,timd,timm) # dd1=sample_pool(ct,rl,timd,timm)
def save_region_feature(ct, rl, timd, timm):
# a=time.time()
region_feature = dff(ct, rl, timd, timm)
def save_region_feature(ct,rl,timd,timm): # print('region_feature=',region_feature)
#a=time.time()
region_feature=dff(ct,rl,timd,timm) mmup = region_make_up_price(ct, rl, None, timm)
# print('mmup_region=',mmup)
##print('region_feature=',region_feature)
mmup=region_make_up_price(ct,rl,None,timm)
##print('mmup_region=',mmup)
#print('this month region price',region_feature[1]) #print('this month region price',region_feature[1])
markup=return_operate(mmup,region_feature[1]) markup = return_operate(mmup, region_feature[1])
def c_date(): def c_date():
date1=datetime.datetime.now() date1 = datetime.datetime.now()
pacific = pytz.timezone('Asia/Shanghai') pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(date1) return pacific.localize(date1)
def get_weekend(tt): def get_weekend(tt):
from dateutil.parser import parse from dateutil.parser import parse
bb=parse(tt) bb = parse(tt)
pacific = pytz.timezone('Asia/Shanghai') pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(bb) return pacific.localize(bb)
def month_lastday(ti): def month_lastday(ti):
from dateutil.parser import parse from dateutil.parser import parse
today=dateutil.parser.parse(str(ti)) today = dateutil.parser.parse(str(ti))
#first = datetime.date(day=1, month=today.month, year=today.year) #first = datetime.date(day=1, month=today.month, year=today.year)
lastMonth = today# - datetime.timedelta(days=35) lastMonth = today # - datetime.timedelta(days=35)
def plus(k): def plus(k):
if k<10: if k < 10:
return '0'+str(k) return '0' + str(k)
else: else:
return str(k) return str(k)
y=lastMonth.year y = lastMonth.year
m=lastMonth.month m = lastMonth.month
#d=lastMonth.day # d=lastMonth.day
day=calendar.monthrange(y,m)[1] day = calendar.monthrange(y, m)[1]
cc=str(y)+plus(m)+plus(day) cc = str(y) + plus(m) + plus(day)
bb=parse(cc) bb = parse(cc)
pacific = pytz.timezone('Asia/Shanghai') pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(bb) return pacific.localize(bb)
timme=month_lastday(timm) timme = month_lastday(timm)
###print('region_timme=',timme) # print('region_timme=',timme)
#stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) #stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017)
###print("######################## ","ct=",ct,"rl=",rl,"makeup=",make_up) # print("######################## ","ct=",ct,"rl=",rl,"makeup=",make_up)
#save_lg = fangjia.get_collection("fangjia_stat", "rent_1221", config.get_district_stat_host(), config.get_district_stat_port()) #save_lg = fangjia.get_collection("fangjia_stat", "rent_1221", config.get_district_stat_host(), config.get_district_stat_port())
date=c_date() date = c_date()
temp = fangjia.get_collection("fangjia",config.get_rent_temp_db(), '192.168.0.136',27017) temp = fangjia.get_collection(
"fangjia", config.get_rent_temp_db(), '192.168.0.136', 27017)
temp.replace_one( temp.replace_one(
{"city":ct,"region":rl,"weekend":timme,"cat":'region'},
{"city":ct,"region":rl,"district_name":None,"markup":markup,'ratio':region_feature[0],
"avg_price":region_feature[1], "c_date":date,"weekend":timme,"cat":'region'},True
)
avg=region_feature[1] {"city": ct, "region": rl, "weekend": timme, "cat": 'region'},
if avg>0: {"city": ct, "region": rl, "district_name": None, "markup": markup, 'ratio': region_feature[0],
avg=float(avg)
save_lg = fangjia.get_collection("fangjia",config.get_sqm_rentprice_db(), config.get_mongodb_host(),27017) "avg_price": region_feature[1], "c_date": date, "weekend": timme, "cat": 'region'}, True
)
avg = region_feature[1]
if avg > 0:
avg = float(avg)
save_lg = fangjia.get_collection(
"fangjia", config.get_sqm_rentprice_db(), config.get_mongodb_host(), 27017)
save_lg.replace_one( save_lg.replace_one(
{"city":ct,"region":rl,"weekend":timme,"cat":'region'},
{"city":ct,"region":rl,"district_name":None,"markup":markup,'ratio':region_feature[0],
"avg_price":region_feature[1], "c_date":date,"weekend":timme,"cat":'region'},True
)
###print(time.time()-a)
####print(len(disname),len(disname)) {"city": ct, "region": rl, "weekend": timme, "cat": 'region'},
{"city": ct, "region": rl, "district_name": None, "markup": markup, 'ratio': region_feature[0],
"avg_price": region_feature[1], "c_date": date, "weekend": timme, "cat": 'region'}, True
)
# print(time.time()-a)
# print(len(disname),len(disname))
+2 -2
View File
@@ -22,8 +22,8 @@ rl='闵行'
lk='上海康城'#上海闵行闵浦新苑一村普陀曹杨一村上海北京东城左安漪园 lk='上海康城'#上海闵行闵浦新苑一村普陀曹杨一村上海北京东城左安漪园
#黄浦伯爵居静安泰安里长宁新泾北苑上海奉贤海岸景苑闵行陇兴别墅阳江雅苑养生城二期 #黄浦伯爵居静安泰安里长宁新泾北苑上海奉贤海岸景苑闵行陇兴别墅阳江雅苑养生城二期
timm=20180430 timm=20181130
timd=20170531 timd=20171130
a=time.time() a=time.time()
rentPrice.save_to_mongo(ct,rl,lk,timd,timm) rentPrice.save_to_mongo(ct,rl,lk,timd,timm)