修改: PriceSystem/config/__init__.py

修改:     PriceSystem/fangjia/__init__.py
	修改:     PriceSystem/feature/region_feature.py
	修改:     PriceSystem/rent_price/district_map0413.py
	修改:     PriceSystem/rent_test.py
This commit is contained in:
罗干
2018-05-07 17:41:05 +08:00
parent 6ce1075d29
commit 58968908de
5 changed files with 206 additions and 160 deletions
+4 -1
View File
@@ -6,10 +6,10 @@ def get_worker_num():
return 20
def get_redis_host():
return "192.168.0.144"
def get_district_stat_host():
return "avgprice.mongo.db.fangjia.com"
@@ -72,6 +72,9 @@ def get_rent_calc_cmd_queue_name():
def get_sqm_rentprice_db():
return "sqmetre_rent"
def get_rent_temp_db():
return "temp_rent"
def get_lg_mongo():
return 'luogan.mongo.com'
+47 -24
View File
@@ -408,40 +408,34 @@ def rent_disanganf_series(query, fields):
pos=list(seawater105.find(query,fields))
#print(pos)
print('pos=',pos)
data1=pandas.DataFrame(pos)
if data1.shape[0]>3:
one_date=list(set(data1['weekend']))[0]
data=data1[data1['weekend']==one_date]
d0=list(data['calc_value'])
d1=list(data['district_name'])
d2=pandas.Series(d0,index=d1)
return d2
else:
return pandas.DataFrame()
one_date=list(set(data1['weekend']))[1]
#two_date=list(set(data1['weekend']))[1]
data=data1[data1['weekend']==one_date]
d0=list(data['calc_value'])
d1=list(data['district_name'])
d2=pandas.Series(d0,index=d1)
return d2
except:
# print('flag2')
pos=list(seawater105.find(query,fields))
data1=pandas.DataFrame(pos)
print('data1=',data1)
if data1.shape[0]>3:
one_date=list(set(data1['weekend']))[0]
data=data1[data1['weekend']==one_date]
d0=list(data['calc_value'])
d1=list(data['district_name'])
d2=pandas.Series(d0,index=d1)
return d2
else:
return pandas.DataFrame()
one_date=list(set(data1['weekend']))[1]
data=data1[data1['weekend']==one_date]
d0=list(data['calc_value'])
d1=list(data['district_name'])
d2=pandas.Series(d0,index=d1)
return d2
raise Exception("样本查询失败!")
def rent_region_feature(query, fields):
rf = get_collection("fangjia", config.get_sqm_rentprice_db(), 'mongo.db.fangjia.com', 27017)
rf = get_collection("fangjia", config.get_rent_temp_db(), '192.168.0.136', 27017)
#client182 = pymongo.MongoClient('192.168.10.182',27017)
#db182 = client182.fangjia_stat.rent_1221
for err_count in range(0, 2):
@@ -466,6 +460,35 @@ def rent_region_feature(query, fields):
else:
return ()
"""
def rent_find_disanganf_price(query, fields=None):
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# dsf = db1.average_statistics
# s = time.time()
seawater105 = get_collection("fangjia", "district_stat",'mongo.db.fangjia.com', 27017)
for err_count in range(0, 2):
try:
# print('flag3')
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("yang ben cha xun shi bai1")
+67 -47
View File
@@ -7,7 +7,7 @@ Created on Tue Mar 27 13:23:56 2018
"""
import os
##print( os.getcwd())
###print( os.getcwd())
#from collections import Counter
import pymongo
#from pymongo import MongoClient
@@ -35,13 +35,6 @@ import pandas as pd
import calendar
def ghhh(kkg): #555555555555555555555555555
kkg=kkg.fillna('无')
a=list()
@@ -53,7 +46,7 @@ def ghhh(kkg): #555555555555555555555555555
'''
def f_del(qq,g):
if g!= None:
@@ -62,21 +55,23 @@ def f_del(qq,g):
else:
del qq['region']
return qq
'''
def sample_pool(ct,rl,timd,timm):
query = {"city":ct,"cat":"rent","region":rl,
"p_date":{"$gt":timd,"$lt":timm}}
query=f_del(query,rl)
#query=f_del(query,rl)
fields = {"total_price":1,"area":1, "city":1,"region":1,'p_date':1,
"district_name":1,"rent_type":1,"fitment":1,"block":1}
#lt= seawater.count(query)
###print(lt)
####print(lt)
#pos = list()
#for s in seawater.find(query,fields1).skip(10000).limit(lt):
print('query=',query)
#print('query=',query)
return fangjia.rent_sample(query, fields)
def k_means(pp):
@@ -107,7 +102,7 @@ def get_weekend():
#from dateutil.parser import parse
#bb=parse(tt)
now_time=datetime.datetime.now()
before_time = now_time-datetime.timedelta(days=14)
before_time = now_time-datetime.timedelta(days=21)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize( before_time )
@@ -129,21 +124,25 @@ def disanganf_price(ct,rl,lk,timd,timm):
def rent_partial_sell(ct,rl,timd,timm):
rele0=sample_pool(ct,rl,timd,timm)
#print('rele0=',rele0)
##print('rele0=',rele0)
if rele0.shape[0]>0:
rele1=rele0[['district_name','p_date','area','total_price']]
rele1['total_price']=rele1['total_price']/rele1['area']
#print('up_rele.shape[0]=',rele1.shape[0])
##print('up_rele.shape[0]=',rele1.shape[0])
rele=rele0.drop_duplicates(['district_name'])
p1=rele['district_name'].dropna()
p2=list(set(p1))
###print('*****************************************************************')
#print('p2=',p2)
####print('*****************************************************************')
dd=disanganf_price(ct,rl,p2,timd,timm)
###print('rele.shape[0]=',rele.shape[0])
##print('rele.shape[0]=',rele.shape[0])
#print('dd=',dd)
if rele.shape[0]>10:
@@ -162,12 +161,12 @@ def rent_partial_sell(ct,rl,timd,timm):
ro.append(price)
#price=rv1[k][0]
if namee in dd.index:
#print(namee)
sell=dd[namee]
ppoo=sell/price
sell_rent_ratio.append(ppoo)
#rv1[k][3]=sell/price
###print('rele0=',rele)
##print('rele0=',rele)
rent_most=k_means(pandas.Series(ro))
#rele=pd.DataFrame(rv1,columns=rv.columns)
roseries=pandas.Series(sell_rent_ratio)
@@ -175,37 +174,42 @@ def rent_partial_sell(ct,rl,timd,timm):
roseries=roseries[roseries>5]
#rele=rele[rele['sell_rent_ratio']>5]
##print('sell_rent_ratio=',list(rele['sell_rent_ratio'])[:5])
##print(rele)
##print('#######################')
##print(rele.shape[0])
###print('sell_rent_ratio=',list(rele['sell_rent_ratio'])[:5])
###print(rele)
###print('#######################')
###print(rele.shape[0])
#print('len(roseries)=',len(roseries))
#print(roseries)
if len(roseries)>10:
#rent_most=k_means(rele['total_price'])
##print('rele.shape[0]=',rele.shape[0])
#print('yyyyyyyyyyyyyyyyyyyyyy=',len(roseries))
###print('rele.shape[0]=',rele.shape[0])
##print('yyyyyyyyyyyyyyyyyyyyyy=',len(roseries))
rent_sell=k_means(roseries.dropna())
del rele
return rent_sell,rent_most
else:
del rele
return '没有区域租售比',rent_most
return -1,rent_most
else:
del rele
return '没有区域租售比','没有区域租金数据'
return -1,-1
else:
return '没有任何数据','没有任何数据'
return -1,-1
def dff(i,j,timd,timm): #666666666666666666666666666666666666666666666666666666
reef=rent_partial_sell(i,j,timd,timm)
##print('$$$$$$$$$$')
if type(reef[0])!=str:
###print('$$$$$$$$$$')
if reef[0]>0:
return reef
else:
##print('&&&&&&&&&&&&&')
###print('&&&&&&&&&&&&&')
return rent_partial_sell(i,None,timd,timm)
@@ -227,7 +231,7 @@ def timestamp_to_int(dd):
f2 = re.sub("\D", "", f1)
f3=int(f2)
dd['week1'][i]=f3
###print(dd)
####print(dd)
return dd
'''
@@ -285,8 +289,8 @@ def region_make_up_price(ct,rl,lk,ti):
#save_lg=client136.fangjia_stat.test_lg5
cc=before_month_lastday(ti,32)
#print('ti=',ti)
#print('cc=',cc)
##print('ti=',ti)
##print('cc=',cc)
#stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017)
stat=lgg
@@ -299,14 +303,14 @@ def region_make_up_price(ct,rl,lk,ti):
#pos=timestamp_to_int(pos)
#pos=pos[pos['week1']>cc]
##print('5'*10)
##print('uuuuuuuuuuuuuuuu=',pos)
###print('5'*10)
###print('uuuuuuuuuuuuuuuu=',pos)
if pos.shape[0]>=1:
pos=pos.sort_values(by='weekend',ascending=False)
pp1=pos['avg_price'].iloc[0]
#print('pp1=',pp1)
##print('pp1=',pp1)
if type(pp1)!=None and type(pp1)!=str:
return pp1
else:
@@ -343,10 +347,10 @@ def save_region_feature(ct,rl,timd,timm):
#a=time.time()
region_feature=dff(ct,rl,timd,timm)
#print('region_feature=',region_feature)
##print('region_feature=',region_feature)
mmup=region_make_up_price(ct,rl,None,timm)
#print('mmup_region=',mmup)
##print('mmup_region=',mmup)
#print('this month region price',region_feature[1])
markup=return_operate(mmup,region_feature[1])
@@ -391,17 +395,33 @@ def save_region_feature(ct,rl,timd,timm):
return pacific.localize(bb)
timme=month_lastday(timm)
##print('region_timme=',timme)
###print('region_timme=',timme)
#stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017)
##print("######################## ","ct=",ct,"rl=",rl,"makeup=",make_up)
###print("######################## ","ct=",ct,"rl=",rl,"makeup=",make_up)
#save_lg = fangjia.get_collection("fangjia_stat", "rent_1221", config.get_district_stat_host(), config.get_district_stat_port())
avg=region_feature[1]
date=c_date()
if type(avg)!=str:
temp = fangjia.get_collection("fangjia",config.get_rent_temp_db(), '192.168.0.136',27017)
temp.replace_one(
{"city":ct,"region":rl,"weekend":timme,"cat":'region'},
{"city":ct,"region":rl,"district_name":None,"markup":markup,'ratio':region_feature[0],
"avg_price":region_feature[1], "c_date":date,"weekend":timme,"cat":'region'},True
)
avg=region_feature[1]
if avg>0:
avg=float(avg)
save_lg = fangjia.get_collection("fangjia",config.get_sqm_rentprice_db(), config.get_mongodb_host(),27017)
@@ -414,9 +434,9 @@ def save_region_feature(ct,rl,timd,timm):
"avg_price":region_feature[1], "c_date":date,"weekend":timme,"cat":'region'},True
)
##print(time.time()-a)
###print(time.time()-a)
###print(len(disname),len(disname))
####print(len(disname),len(disname))
+85 -85
View File
@@ -38,7 +38,7 @@ import pytz
def common_rent_price(ct,rl,p,timd,timm,region_feature,mongodb):
'''
def f_del(qq,g):
if g!= None:
@@ -47,12 +47,15 @@ def common_rent_price(ct,rl,p,timd,timm,region_feature,mongodb):
else:
del qq['region']
return qq
'''
def add_rent_type(ppd):
if 'rent_type' not in ppd.columns:
ppd['rent_type']='整租'
return ppd
'''
def sample_pool(ct,rl,timd,timm):
query = {"city":ct,"cat":"rent","region":rl,
"p_date":{"$gt":timd,"$lt":timm}}
@@ -69,6 +72,8 @@ def common_rent_price(ct,rl,p,timd,timm,region_feature,mongodb):
return data[['district_name','total_price','area']]
else:
return numpy.array([])
'''
def disanganf_price(ct,rl,lk,timd,timm):
#client1 = pymongo.MongoClient('192.168.0.136',27017)
@@ -927,116 +932,111 @@ def save_to_mongo(ct,rl,lk,timd,timm):
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(bb)
'''
def region_feature(ct,rl,weekend):
def region_feature(ct,rl):
#client182 = pymongo.MongoClient('192.168.10.182',27017)
#db182 = client182.fangjia_stat.rent_1221
query1 = {"city":ct,"region":rl,'weekend':before_month_lastday(timm,0)}
#a1=list(db182.find(query1))
#a2=pd.DataFrame(a1)
a2=fangjia.rent_region_feature(query1, None)
a2=fangjia.rent_region_feature(query1,None)
if a2.shape[0]>0:
ratio=a2['ratio'].iloc[0]
price=a2['avg_price'].iloc[0]
return (ratio,price)
else:
return ()
def add_region_fea(ct,rl,weekend):
def add_region_fea(ct,rl):
#timm=20180131
#timd=20170303
d1=region_feature(ct,rl,None)
#print('d1=',d1)
d1=region_feature(ct,rl)
print('d1=',d1)
if len(d1)>0:
return d1
else:
feature.region_feature.save_region_feature(ct,rl,timd,timm)
return region_feature(ct,rl,None)
return region_feature(ct,rl)
#a=time.time()
lf=region_district_list(ct,rl,lk) #3333333333333333333333333333333333333333333333
#disname=list(lf['name'])
#print('a_time=',time.time()-a)
###print('len(lf)',len(lf))
#b=time.time()
lf=add_alias(lf) #4444444444444444444444444444444444444444444444444444444
lf=add_eatype(lf)
lf['estate_type2']=lf['estate_type2'].fillna('普通住宅')
p=ghhh(lf[['name','alias','estate_type2']])[0] #555555555555555555555555555
region_feature=add_region_fea(ct,rl,None)
print('region_feature=',region_feature)
#print('b_time=',time.time()-b)
def get_weekend(tt):
from dateutil.parser import parse
bb=parse(tt)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(bb)
#timme=dateutil.parser.parse(str(timm))
timme=before_month_lastday(timm,0)
##print('timme=',timme)
###print(len(disname),len(disname))
###print('region_feature',region_feature)
##print('pos=',pos)
#c=time.time()
if len(region_feature)>0:
mm=common_rent_price(ct,rl,p,timd,timm,region_feature,None)
#print('c_time=',time.time()-c)
##print(ct,rl,p[0],mm)
lf=region_district_list(ct,rl,lk)
if lf.shape[0]>0:
lf=add_alias(lf)
lf=add_eatype(lf)
lf['estate_type2']=lf['estate_type2'].fillna('普通住宅')
#d=time.time()
mmup=dis_make_up_price(ct,rl,p[0],timm)
#print('mmup=',mmup)
#print('mm=',mm)
make_up=return_operate(mmup,mm)
#print('d_time=',time.time()-d)
###print('p=',p)
p=ghhh(lf[['name','alias','estate_type2']])[0]
def c_date():
date1=datetime.datetime.now()
region_feature=add_region_fea(ct,rl)
print('region_feature=',region_feature)
#print('b_time=',time.time()-b)
def get_weekend(tt):
from dateutil.parser import parse
bb=parse(tt)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(date1)
date=c_date()
#weekend=20161231
print("######################## ","ct=",ct,"rl=",rl,"lk=",p[0],"mm=",mm,make_up)
"""
client1 = pymongo.MongoClient('192.168.0.136',27017)
db1 = client1.fangjia_stat
stat = db1.test_lg55
return pacific.localize(bb)
#timme=dateutil.parser.parse(str(timm))
timme=before_month_lastday(timm,0)
#stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017)
save_lg.save({"city":ct,"region":rl,"district_name":p[0],"make_up":make_up,
"avg_price":mm, "c_date":date,"weekend":timme,"cat":'district'})
save_lg=client136.fangjia_stat.rent_1221
"""
if type(mm)!=str and mm!=None:
mm=float(mm)
#get_sqm_rentprice_db()
save_lg=lg_db
save_lg.replace_one(
{"city":ct,"region":rl,"district_name":p[0],"weekend":timme},
{"city":ct,"region":rl,"district_name":p[0],"markup":make_up,
"avg_price":mm, "c_date":date,"weekend":timme,"cat":'district'},True
)
##print('timme=',timme)
###print(len(disname),len(disname))
###print('region_feature',region_feature)
##print('pos=',pos)
#c=time.time()
if len(region_feature)>0 and region_feature[0]>0 and region_feature[1]>0:
mm=common_rent_price(ct,rl,p,timd,timm,region_feature,None)
#print('c_time=',time.time()-c)
##print(ct,rl,p[0],mm)
#d=time.time()
mmup=dis_make_up_price(ct,rl,p[0],timm)
#print('mmup=',mmup)
#print('mm=',mm)
make_up=return_operate(mmup,mm)
#print('d_time=',time.time()-d)
###print('p=',p)
def c_date():
date1=datetime.datetime.now()
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(date1)
date=c_date()
#weekend=20161231
print("######################## ","ct=",ct,"rl=",rl,"lk=",p[0],"mm=",mm,make_up)
"""
client1 = pymongo.MongoClient('192.168.0.136',27017)
db1 = client1.fangjia_stat
stat = db1.test_lg55
#stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017)
save_lg.save({"city":ct,"region":rl,"district_name":p[0],"make_up":make_up,
"avg_price":mm, "c_date":date,"weekend":timme,"cat":'district'})
save_lg=client136.fangjia_stat.rent_1221
"""
if type(mm)!=str and mm!=None:
mm=float(mm)
#get_sqm_rentprice_db()
save_lg=lg_db
save_lg.replace_one(
{"city":ct,"region":rl,"district_name":p[0],"weekend":timme},
{"city":ct,"region":rl,"district_name":p[0],"markup":make_up,
"avg_price":mm, "c_date":date,"weekend":timme,"cat":'district'},True
)
+3 -3
View File
@@ -16,10 +16,10 @@ def get_weekend(tt):
weekend = get_weekend('20180515')
ct='琼中'
ct='上海'
rl='琼中'
lk='大美山雨墅'#上海闵行闵浦新苑一村普陀曹杨一村上海北京东城左安漪园
rl='闵行'
lk='上海康城'#上海闵行闵浦新苑一村普陀曹杨一村上海北京东城左安漪园
#黄浦伯爵居静安泰安里长宁新泾北苑上海奉贤海岸景苑闵行陇兴别墅阳江雅苑养生城二期
timm=20180430