From 58968908deb34ed813095b97552ba8ad98f8fc15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BD=97=E5=B9=B2?= Date: Mon, 7 May 2018 17:41:05 +0800 Subject: [PATCH] =?UTF-8?q?=09=E4=BF=AE=E6=94=B9=EF=BC=9A=20=20=20=20=20Pr?= =?UTF-8?q?iceSystem/config/=5F=5Finit=5F=5F.py=20=09=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=EF=BC=9A=20=20=20=20=20PriceSystem/fangjia/=5F=5Finit=5F=5F.py?= =?UTF-8?q?=20=09=E4=BF=AE=E6=94=B9=EF=BC=9A=20=20=20=20=20PriceSystem/fea?= =?UTF-8?q?ture/region=5Ffeature.py=20=09=E4=BF=AE=E6=94=B9=EF=BC=9A=20=20?= =?UTF-8?q?=20=20=20PriceSystem/rent=5Fprice/district=5Fmap0413.py=20=09?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=EF=BC=9A=20=20=20=20=20PriceSystem/rent=5Fte?= =?UTF-8?q?st.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PriceSystem/config/__init__.py | 5 +- PriceSystem/fangjia/__init__.py | 71 ++++++--- PriceSystem/feature/region_feature.py | 114 ++++++++------ PriceSystem/rent_price/district_map0413.py | 170 ++++++++++----------- PriceSystem/rent_test.py | 6 +- 5 files changed, 206 insertions(+), 160 deletions(-) diff --git a/PriceSystem/config/__init__.py b/PriceSystem/config/__init__.py index 6047b0c..ad6b538 100644 --- a/PriceSystem/config/__init__.py +++ b/PriceSystem/config/__init__.py @@ -6,10 +6,10 @@ def get_worker_num(): return 20 + def get_redis_host(): return "192.168.0.144" - def get_district_stat_host(): return "avgprice.mongo.db.fangjia.com" @@ -72,6 +72,9 @@ def get_rent_calc_cmd_queue_name(): def get_sqm_rentprice_db(): return "sqmetre_rent" +def get_rent_temp_db(): + return "temp_rent" + def get_lg_mongo(): return 'luogan.mongo.com' diff --git a/PriceSystem/fangjia/__init__.py b/PriceSystem/fangjia/__init__.py index 34b39f6..3e6c831 100644 --- a/PriceSystem/fangjia/__init__.py +++ b/PriceSystem/fangjia/__init__.py @@ -408,40 +408,34 @@ def rent_disanganf_series(query, fields): pos=list(seawater105.find(query,fields)) - #print(pos) + print('pos=',pos) data1=pandas.DataFrame(pos) - if data1.shape[0]>3: - one_date=list(set(data1['weekend']))[0] - - - data=data1[data1['weekend']==one_date] - d0=list(data['calc_value']) - d1=list(data['district_name']) - d2=pandas.Series(d0,index=d1) - return d2 - else: - return pandas.DataFrame() + one_date=list(set(data1['weekend']))[1] + #two_date=list(set(data1['weekend']))[1] + data=data1[data1['weekend']==one_date] + + + d0=list(data['calc_value']) + d1=list(data['district_name']) + d2=pandas.Series(d0,index=d1) + return d2 except: # print('flag2') pos=list(seawater105.find(query,fields)) data1=pandas.DataFrame(pos) print('data1=',data1) - if data1.shape[0]>3: - one_date=list(set(data1['weekend']))[0] - - - data=data1[data1['weekend']==one_date] - d0=list(data['calc_value']) - d1=list(data['district_name']) - d2=pandas.Series(d0,index=d1) - return d2 - else: - return pandas.DataFrame() + one_date=list(set(data1['weekend']))[1] + data=data1[data1['weekend']==one_date] + d0=list(data['calc_value']) + d1=list(data['district_name']) + d2=pandas.Series(d0,index=d1) + return d2 + raise Exception("样本查询失败!") def rent_region_feature(query, fields): - rf = get_collection("fangjia", config.get_sqm_rentprice_db(), 'mongo.db.fangjia.com', 27017) + rf = get_collection("fangjia", config.get_rent_temp_db(), '192.168.0.136', 27017) #client182 = pymongo.MongoClient('192.168.10.182',27017) #db182 = client182.fangjia_stat.rent_1221 for err_count in range(0, 2): @@ -466,6 +460,35 @@ def rent_region_feature(query, fields): else: return () """ + +def rent_find_disanganf_price(query, fields=None): + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # dsf = db1.average_statistics + # s = time.time() + seawater105 = get_collection("fangjia", "district_stat",'mongo.db.fangjia.com', 27017) + for err_count in range(0, 2): + + try: + # print('flag3') + lt = seawater105.count(query) + + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + # print("==================>", time.time() - s) + return data + + except: + + # err_count = err_count + 1 + lt = seawater105.count(query) + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + return data + + raise Exception("yang ben cha xun shi bai1") diff --git a/PriceSystem/feature/region_feature.py b/PriceSystem/feature/region_feature.py index 8f5e899..7a8e1e5 100644 --- a/PriceSystem/feature/region_feature.py +++ b/PriceSystem/feature/region_feature.py @@ -7,7 +7,7 @@ Created on Tue Mar 27 13:23:56 2018 """ import os -##print( os.getcwd()) +###print( os.getcwd()) #from collections import Counter import pymongo #from pymongo import MongoClient @@ -35,13 +35,6 @@ import pandas as pd import calendar - - - - - - - def ghhh(kkg): #555555555555555555555555555 kkg=kkg.fillna('无') a=list() @@ -53,7 +46,7 @@ def ghhh(kkg): #555555555555555555555555555 - +''' def f_del(qq,g): if g!= None: @@ -62,21 +55,23 @@ def f_del(qq,g): else: del qq['region'] return qq + +''' def sample_pool(ct,rl,timd,timm): query = {"city":ct,"cat":"rent","region":rl, "p_date":{"$gt":timd,"$lt":timm}} - query=f_del(query,rl) + #query=f_del(query,rl) fields = {"total_price":1,"area":1, "city":1,"region":1,'p_date':1, "district_name":1,"rent_type":1,"fitment":1,"block":1} #lt= seawater.count(query) - ###print(lt) + ####print(lt) #pos = list() #for s in seawater.find(query,fields1).skip(10000).limit(lt): - print('query=',query) + #print('query=',query) return fangjia.rent_sample(query, fields) def k_means(pp): @@ -107,7 +102,7 @@ def get_weekend(): #from dateutil.parser import parse #bb=parse(tt) now_time=datetime.datetime.now() - before_time = now_time-datetime.timedelta(days=14) + before_time = now_time-datetime.timedelta(days=21) pacific = pytz.timezone('Asia/Shanghai') return pacific.localize( before_time ) @@ -129,21 +124,25 @@ def disanganf_price(ct,rl,lk,timd,timm): def rent_partial_sell(ct,rl,timd,timm): rele0=sample_pool(ct,rl,timd,timm) - #print('rele0=',rele0) + ##print('rele0=',rele0) if rele0.shape[0]>0: + + rele1=rele0[['district_name','p_date','area','total_price']] rele1['total_price']=rele1['total_price']/rele1['area'] - #print('up_rele.shape[0]=',rele1.shape[0]) + ##print('up_rele.shape[0]=',rele1.shape[0]) + rele=rele0.drop_duplicates(['district_name']) - p1=rele['district_name'].dropna() p2=list(set(p1)) - - ###print('*****************************************************************') + #print('p2=',p2) + ####print('*****************************************************************') dd=disanganf_price(ct,rl,p2,timd,timm) - ###print('rele.shape[0]=',rele.shape[0]) + ##print('rele.shape[0]=',rele.shape[0]) + + #print('dd=',dd) if rele.shape[0]>10: @@ -162,12 +161,12 @@ def rent_partial_sell(ct,rl,timd,timm): ro.append(price) #price=rv1[k][0] if namee in dd.index: - + #print(namee) sell=dd[namee] ppoo=sell/price sell_rent_ratio.append(ppoo) #rv1[k][3]=sell/price - ###print('rele0=',rele) + ##print('rele0=',rele) rent_most=k_means(pandas.Series(ro)) #rele=pd.DataFrame(rv1,columns=rv.columns) roseries=pandas.Series(sell_rent_ratio) @@ -175,37 +174,42 @@ def rent_partial_sell(ct,rl,timd,timm): roseries=roseries[roseries>5] #rele=rele[rele['sell_rent_ratio']>5] - ##print('sell_rent_ratio=',list(rele['sell_rent_ratio'])[:5]) - ##print(rele) - ##print('#######################') - ##print(rele.shape[0]) + ###print('sell_rent_ratio=',list(rele['sell_rent_ratio'])[:5]) + ###print(rele) + ###print('#######################') + ###print(rele.shape[0]) + + #print('len(roseries)=',len(roseries)) + + #print(roseries) if len(roseries)>10: #rent_most=k_means(rele['total_price']) - ##print('rele.shape[0]=',rele.shape[0]) - #print('yyyyyyyyyyyyyyyyyyyyyy=',len(roseries)) + ###print('rele.shape[0]=',rele.shape[0]) + ##print('yyyyyyyyyyyyyyyyyyyyyy=',len(roseries)) rent_sell=k_means(roseries.dropna()) del rele return rent_sell,rent_most else: del rele - return '没有区域租售比',rent_most + return -1,rent_most else: del rele - return '没有区域租售比','没有区域租金数据' + return -1,-1 else: - return '没有任何数据','没有任何数据' + return -1,-1 def dff(i,j,timd,timm): #666666666666666666666666666666666666666666666666666666 reef=rent_partial_sell(i,j,timd,timm) - ##print('$$$$$$$$$$') - if type(reef[0])!=str: + ###print('$$$$$$$$$$') + + if reef[0]>0: return reef else: - ##print('&&&&&&&&&&&&&') + ###print('&&&&&&&&&&&&&') return rent_partial_sell(i,None,timd,timm) @@ -227,7 +231,7 @@ def timestamp_to_int(dd): f2 = re.sub("\D", "", f1) f3=int(f2) dd['week1'][i]=f3 - ###print(dd) + ####print(dd) return dd ''' @@ -285,8 +289,8 @@ def region_make_up_price(ct,rl,lk,ti): #save_lg=client136.fangjia_stat.test_lg5 cc=before_month_lastday(ti,32) - #print('ti=',ti) - #print('cc=',cc) + ##print('ti=',ti) + ##print('cc=',cc) #stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) stat=lgg @@ -299,14 +303,14 @@ def region_make_up_price(ct,rl,lk,ti): #pos=timestamp_to_int(pos) #pos=pos[pos['week1']>cc] - ##print('5'*10) - ##print('uuuuuuuuuuuuuuuu=',pos) + ###print('5'*10) + ###print('uuuuuuuuuuuuuuuu=',pos) if pos.shape[0]>=1: pos=pos.sort_values(by='weekend',ascending=False) pp1=pos['avg_price'].iloc[0] - #print('pp1=',pp1) + ##print('pp1=',pp1) if type(pp1)!=None and type(pp1)!=str: return pp1 else: @@ -343,10 +347,10 @@ def save_region_feature(ct,rl,timd,timm): #a=time.time() region_feature=dff(ct,rl,timd,timm) - #print('region_feature=',region_feature) + ##print('region_feature=',region_feature) mmup=region_make_up_price(ct,rl,None,timm) - #print('mmup_region=',mmup) + ##print('mmup_region=',mmup) #print('this month region price',region_feature[1]) markup=return_operate(mmup,region_feature[1]) @@ -391,17 +395,33 @@ def save_region_feature(ct,rl,timd,timm): return pacific.localize(bb) timme=month_lastday(timm) - ##print('region_timme=',timme) + ###print('region_timme=',timme) #stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) - ##print("######################## ","ct=",ct,"rl=",rl,"makeup=",make_up) + ###print("######################## ","ct=",ct,"rl=",rl,"makeup=",make_up) #save_lg = fangjia.get_collection("fangjia_stat", "rent_1221", config.get_district_stat_host(), config.get_district_stat_port()) - avg=region_feature[1] + + + date=c_date() - if type(avg)!=str: + + temp = fangjia.get_collection("fangjia",config.get_rent_temp_db(), '192.168.0.136',27017) + + temp.replace_one( + + {"city":ct,"region":rl,"weekend":timme,"cat":'region'}, + + {"city":ct,"region":rl,"district_name":None,"markup":markup,'ratio':region_feature[0], + + "avg_price":region_feature[1], "c_date":date,"weekend":timme,"cat":'region'},True + ) + + avg=region_feature[1] + + if avg>0: avg=float(avg) save_lg = fangjia.get_collection("fangjia",config.get_sqm_rentprice_db(), config.get_mongodb_host(),27017) @@ -414,9 +434,9 @@ def save_region_feature(ct,rl,timd,timm): "avg_price":region_feature[1], "c_date":date,"weekend":timme,"cat":'region'},True ) - ##print(time.time()-a) + ###print(time.time()-a) -###print(len(disname),len(disname)) +####print(len(disname),len(disname)) diff --git a/PriceSystem/rent_price/district_map0413.py b/PriceSystem/rent_price/district_map0413.py index 1d63478..5df1d04 100644 --- a/PriceSystem/rent_price/district_map0413.py +++ b/PriceSystem/rent_price/district_map0413.py @@ -38,7 +38,7 @@ import pytz def common_rent_price(ct,rl,p,timd,timm,region_feature,mongodb): - + ''' def f_del(qq,g): if g!= None: @@ -47,12 +47,15 @@ def common_rent_price(ct,rl,p,timd,timm,region_feature,mongodb): else: del qq['region'] return qq + ''' def add_rent_type(ppd): if 'rent_type' not in ppd.columns: ppd['rent_type']='整租' return ppd - + + + ''' def sample_pool(ct,rl,timd,timm): query = {"city":ct,"cat":"rent","region":rl, "p_date":{"$gt":timd,"$lt":timm}} @@ -69,6 +72,8 @@ def common_rent_price(ct,rl,p,timd,timm,region_feature,mongodb): return data[['district_name','total_price','area']] else: return numpy.array([]) + ''' + def disanganf_price(ct,rl,lk,timd,timm): #client1 = pymongo.MongoClient('192.168.0.136',27017) @@ -927,116 +932,111 @@ def save_to_mongo(ct,rl,lk,timd,timm): pacific = pytz.timezone('Asia/Shanghai') return pacific.localize(bb) ''' - def region_feature(ct,rl,weekend): + def region_feature(ct,rl): #client182 = pymongo.MongoClient('192.168.10.182',27017) #db182 = client182.fangjia_stat.rent_1221 query1 = {"city":ct,"region":rl,'weekend':before_month_lastday(timm,0)} #a1=list(db182.find(query1)) #a2=pd.DataFrame(a1) - a2=fangjia.rent_region_feature(query1, None) + a2=fangjia.rent_region_feature(query1,None) if a2.shape[0]>0: ratio=a2['ratio'].iloc[0] price=a2['avg_price'].iloc[0] return (ratio,price) else: return () - def add_region_fea(ct,rl,weekend): + def add_region_fea(ct,rl): #timm=20180131 #timd=20170303 - d1=region_feature(ct,rl,None) - #print('d1=',d1) + d1=region_feature(ct,rl) + print('d1=',d1) if len(d1)>0: return d1 else: feature.region_feature.save_region_feature(ct,rl,timd,timm) - return region_feature(ct,rl,None) + return region_feature(ct,rl) #a=time.time() - lf=region_district_list(ct,rl,lk) #3333333333333333333333333333333333333333333333 - #disname=list(lf['name']) - - #print('a_time=',time.time()-a) - - ###print('len(lf)',len(lf)) - #b=time.time() - lf=add_alias(lf) #4444444444444444444444444444444444444444444444444444444 - lf=add_eatype(lf) - lf['estate_type2']=lf['estate_type2'].fillna('普通住宅') - - p=ghhh(lf[['name','alias','estate_type2']])[0] #555555555555555555555555555 - - - - region_feature=add_region_fea(ct,rl,None) - - print('region_feature=',region_feature) - #print('b_time=',time.time()-b) - - - - def get_weekend(tt): - - from dateutil.parser import parse - bb=parse(tt) - pacific = pytz.timezone('Asia/Shanghai') - return pacific.localize(bb) - #timme=dateutil.parser.parse(str(timm)) - timme=before_month_lastday(timm,0) - - ##print('timme=',timme) - ###print(len(disname),len(disname)) - ###print('region_feature',region_feature) - ##print('pos=',pos) - #c=time.time() - - if len(region_feature)>0: - mm=common_rent_price(ct,rl,p,timd,timm,region_feature,None) - #print('c_time=',time.time()-c) - ##print(ct,rl,p[0],mm) + lf=region_district_list(ct,rl,lk) + if lf.shape[0]>0: + lf=add_alias(lf) + lf=add_eatype(lf) + lf['estate_type2']=lf['estate_type2'].fillna('普通住宅') - #d=time.time() - mmup=dis_make_up_price(ct,rl,p[0],timm) - #print('mmup=',mmup) - #print('mm=',mm) - make_up=return_operate(mmup,mm) - #print('d_time=',time.time()-d) - ###print('p=',p) + p=ghhh(lf[['name','alias','estate_type2']])[0] - def c_date(): - date1=datetime.datetime.now() - + + + region_feature=add_region_fea(ct,rl) + + print('region_feature=',region_feature) + #print('b_time=',time.time()-b) + + + + def get_weekend(tt): + + from dateutil.parser import parse + bb=parse(tt) pacific = pytz.timezone('Asia/Shanghai') - return pacific.localize(date1) - date=c_date() - #weekend=20161231 - print("######################## ","ct=",ct,"rl=",rl,"lk=",p[0],"mm=",mm,make_up) - """ - client1 = pymongo.MongoClient('192.168.0.136',27017) - db1 = client1.fangjia_stat - stat = db1.test_lg55 + return pacific.localize(bb) + #timme=dateutil.parser.parse(str(timm)) + timme=before_month_lastday(timm,0) - - #stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) - save_lg.save({"city":ct,"region":rl,"district_name":p[0],"make_up":make_up, - "avg_price":mm, "c_date":date,"weekend":timme,"cat":'district'}) - save_lg=client136.fangjia_stat.rent_1221 - """ - if type(mm)!=str and mm!=None: - - mm=float(mm) - #get_sqm_rentprice_db() - save_lg=lg_db - save_lg.replace_one( - - {"city":ct,"region":rl,"district_name":p[0],"weekend":timme}, - - {"city":ct,"region":rl,"district_name":p[0],"markup":make_up, - "avg_price":mm, "c_date":date,"weekend":timme,"cat":'district'},True - ) + ##print('timme=',timme) + ###print(len(disname),len(disname)) + ###print('region_feature',region_feature) + ##print('pos=',pos) + #c=time.time() + if len(region_feature)>0 and region_feature[0]>0 and region_feature[1]>0: + mm=common_rent_price(ct,rl,p,timd,timm,region_feature,None) + #print('c_time=',time.time()-c) + ##print(ct,rl,p[0],mm) + + #d=time.time() + mmup=dis_make_up_price(ct,rl,p[0],timm) + #print('mmup=',mmup) + #print('mm=',mm) + make_up=return_operate(mmup,mm) + #print('d_time=',time.time()-d) + ###print('p=',p) + + def c_date(): + date1=datetime.datetime.now() + + pacific = pytz.timezone('Asia/Shanghai') + return pacific.localize(date1) + date=c_date() + #weekend=20161231 + print("######################## ","ct=",ct,"rl=",rl,"lk=",p[0],"mm=",mm,make_up) + """ + client1 = pymongo.MongoClient('192.168.0.136',27017) + db1 = client1.fangjia_stat + stat = db1.test_lg55 + + + #stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) + save_lg.save({"city":ct,"region":rl,"district_name":p[0],"make_up":make_up, + "avg_price":mm, "c_date":date,"weekend":timme,"cat":'district'}) + save_lg=client136.fangjia_stat.rent_1221 + """ + if type(mm)!=str and mm!=None: + + mm=float(mm) + #get_sqm_rentprice_db() + save_lg=lg_db + save_lg.replace_one( + + {"city":ct,"region":rl,"district_name":p[0],"weekend":timme}, + + {"city":ct,"region":rl,"district_name":p[0],"markup":make_up, + "avg_price":mm, "c_date":date,"weekend":timme,"cat":'district'},True + ) + diff --git a/PriceSystem/rent_test.py b/PriceSystem/rent_test.py index 34360e7..3eb3733 100644 --- a/PriceSystem/rent_test.py +++ b/PriceSystem/rent_test.py @@ -16,10 +16,10 @@ def get_weekend(tt): weekend = get_weekend('20180515') -ct='琼中' +ct='上海' -rl='琼中' -lk='大美山雨墅'#上海闵行闵浦新苑一村普陀曹杨一村上海北京东城左安漪园 +rl='闵行' +lk='上海康城'#上海闵行闵浦新苑一村普陀曹杨一村上海北京东城左安漪园 #黄浦伯爵居静安泰安里长宁新泾北苑上海奉贤海岸景苑闵行陇兴别墅阳江雅苑养生城二期 timm=20180430