From 04eaf142f205a6b60bc456579762c962835ff114 Mon Sep 17 00:00:00 2001 From: "wangwenyan@fangjia.com" Date: Wed, 5 Dec 2018 18:10:13 +0800 Subject: [PATCH] fix bug --- PriceSystem/feature/region_feature.py | 573 ++++++++++++++------------ PriceSystem/rent_test.py | 4 +- 2 files changed, 306 insertions(+), 271 deletions(-) diff --git a/PriceSystem/feature/region_feature.py b/PriceSystem/feature/region_feature.py index 7a8e1e5..7ab12ef 100644 --- a/PriceSystem/feature/region_feature.py +++ b/PriceSystem/feature/region_feature.py @@ -13,12 +13,13 @@ import pymongo #from pymongo import MongoClient import numpy as np import random -import collections -import pymongo +import collections +import pymongo import sklearn.cluster import pandas import numpy -import sys, time +import sys +import time #import pp import fangjia import datetime @@ -29,21 +30,66 @@ import dateutil import time import fangjia import config -import datetime -import pytz +import datetime +import pytz import pandas as pd import calendar -def ghhh(kkg): #555555555555555555555555555 - kkg=kkg.fillna('无') - a=list() +import pandas as pd +from elasticsearch import Elasticsearch, helpers +client_DB_es = Elasticsearch('192.168.0.206', port=9200, timeout=1000, http_auth=( + 'developer', '5e8edd851d2fdfbd7415232c67367cc3')) +ES_db_index = "seawater-2018,seawater-2017,seawater-2016" + + +def findHouse_region(ct, rl, timd, timm, fields): + # local_switch and select_content are not applied in this function + + # query = {"city": ct, "cat": "rent", "region": rl, # 'rent_type':'整租', + # "district_name": {"$in": lk}, + # "p_date": {"$gt": timd, "$lt": timm}} + city = ct + region = rl + # district_name = lk[0] + # name_pack = findquery_statement['name_pack'] + query_list =client_DB_es.search( + index=ES_db_index, + scroll='20m', + size=100, + doc_type="_doc", + _source = fields, + body={ + "query": + {"bool": + {"must": + [{"term": {"cat": "rent"}}, + {"term": {"city": city}}, + {"term": {"region": region}}, + # {"term": {"district_name": district_name}}, + #{"terms": {"district_name": lk}}, + # {"terms": {"district_name": ["新泾三村", "新泾四村"]}}, + {"range": {"p_date": { + "gte": timd, "lte": timm}}} + ], "must_not": [], "should": [] + } + }, "from": 0, "size": 100, "sort": [], "aggs": {}} + ) + + query_list_raw = query_list['hits']['hits'] + query_pack = [] + for singlequery in query_list_raw: + query_pack.append(singlequery['_source']) + return pd.DataFrame(query_pack) + + +def ghhh(kkg): # 555555555555555555555555555 + kkg = kkg.fillna('无') + a = list() for i in kkg.index: - b=list(kkg.loc[[i],:].values.flatten()) + b = list(kkg.loc[[i], :].values.flatten()) a.append(b) - return a - - + return a ''' @@ -58,180 +104,178 @@ def f_del(qq,g): ''' -def sample_pool(ct,rl,timd,timm): - query = {"city":ct,"cat":"rent","region":rl, - "p_date":{"$gt":timd,"$lt":timm}} - #query=f_del(query,rl) - - fields = {"total_price":1,"area":1, "city":1,"region":1,'p_date':1, - "district_name":1,"rent_type":1,"fitment":1,"block":1} +def sample_pool(ct, rl, timd, timm): + + query = {"city": ct, "cat": "rent", "region": rl, + "p_date": {"$gt": timd, "$lt": timm}} + # query=f_del(query,rl) + + # fields = {"total_price":1,"area":1, "city":1,"region":1,'p_date':1, + # "district_name":1,"rent_type":1,"fitment":1,"block":1} + fields = ["total_price", "area", "city", "region", "p_date", + "district_name", "rent_type", "fitment", "block"] #lt= seawater.count(query) - ####print(lt) + # print(lt) #pos = list() - #for s in seawater.find(query,fields1).skip(10000).limit(lt): - - #print('query=',query) - return fangjia.rent_sample(query, fields) - + # for s in seawater.find(query,fields1).skip(10000).limit(lt): + + # print('query=',query) + # return fangjia.rent_sample(query, fields) + return findHouse_region(ct, rl, timd, timm, fields) + + def k_means(pp): from sklearn.cluster import KMeans - pv=list(pp.dropna()) - gf=numpy.array([pv]).T - estimator = KMeans(n_clusters=3)#构造聚类器 - estimator.fit(gf)#聚类 - label_pred = estimator.labels_ #获取聚类标签 - aa=collections.Counter(label_pred) - v=pandas.Series(aa) - gg=list(v) - index_max=gg.index(max(gg)) - #yy=np.array([label_pred]).T - #un=np.hstack((gf,yy)) - #uv=pd.DataFrame(un,columns=['price','label']) - centroids = estimator.cluster_centers_ #获取聚类中心 - #inertia = estimator.inertia_ # 获取聚类准则的总和 - center=centroids[index_max][0] - del pv,gf,aa,v,gg,centroids - return (round(center,2)) - - + pv = list(pp.dropna()) + gf = numpy.array([pv]).T + estimator = KMeans(n_clusters=3) # 构造聚类器 + estimator.fit(gf) # 聚类 + label_pred = estimator.labels_ # 获取聚类标签 + aa = collections.Counter(label_pred) + v = pandas.Series(aa) + gg = list(v) + index_max = gg.index(max(gg)) + # yy=np.array([label_pred]).T + # un=np.hstack((gf,yy)) + # uv=pd.DataFrame(un,columns=['price','label']) + centroids = estimator.cluster_centers_ # 获取聚类中心 + # inertia = estimator.inertia_ # 获取聚类准则的总和 + center = centroids[index_max][0] + del pv, gf, aa, v, gg, centroids + return (round(center, 2)) def get_weekend(): - + #from dateutil.parser import parse - #bb=parse(tt) - now_time=datetime.datetime.now() - before_time = now_time-datetime.timedelta(days=21) + # bb=parse(tt) + now_time = datetime.datetime.now() + before_time = now_time - datetime.timedelta(days=21) pacific = pytz.timezone('Asia/Shanghai') - return pacific.localize( before_time ) + return pacific.localize(before_time) +def disanganf_price(ct, rl, lk, timd, timm): -def disanganf_price(ct,rl,lk,timd,timm): - - fields={'calc_value':1,'weekend':1,'district_name':1} - query = {"city":ct,"region":rl, - "district_name":{'$in':lk},'weekend':{'$gt':get_weekend()}} + fields = {'calc_value': 1, 'weekend': 1, 'district_name': 1} + query = {"city": ct, "region": rl, + "district_name": {'$in': lk}, 'weekend': {'$gt': get_weekend()}} #pos = list() -#数据转化为数组,数组的元素为字典 - +# 数据转化为数组,数组的元素为字典 + return fangjia.rent_disanganf_series(query, fields) - +def rent_partial_sell(ct, rl, timd, timm): -def rent_partial_sell(ct,rl,timd,timm): - - rele0=sample_pool(ct,rl,timd,timm) - ##print('rele0=',rele0) - - if rele0.shape[0]>0: - - - rele1=rele0[['district_name','p_date','area','total_price']] - rele1['total_price']=rele1['total_price']/rele1['area'] - ##print('up_rele.shape[0]=',rele1.shape[0]) - - - rele=rele0.drop_duplicates(['district_name']) - p1=rele['district_name'].dropna() - p2=list(set(p1)) - #print('p2=',p2) - ####print('*****************************************************************') - dd=disanganf_price(ct,rl,p2,timd,timm) - ##print('rele.shape[0]=',rele.shape[0]) - - #print('dd=',dd) - - if rele.shape[0]>10: - - rele['total_price']=rele['total_price']/rele['area'] - #rent_most=k_means(rele['total_price']) - rele['sell_rent_ratio']=1 - rv=rele[['total_price','district_name','area','sell_rent_ratio']].dropna() - rv1=rv.values - ro=[] - sell_rent_ratio=[] + rele0 = sample_pool(ct, rl, timd, timm) + # print('rele0=',rele0) + + if rele0.shape[0] > 0: + + rele1 = rele0[['district_name', 'p_date', 'area', 'total_price']] + rele1['total_price'] = rele1['total_price'] / rele1['area'] + # print('up_rele.shape[0]=',rele1.shape[0]) + + rele = rele0.drop_duplicates(['district_name']) + p1 = rele['district_name'].dropna() + p2 = list(set(p1)) + # print('p2=',p2) + # print('*****************************************************************') + dd = disanganf_price(ct, rl, p2, timd, timm) + # print('rele.shape[0]=',rele.shape[0]) + + # print('dd=',dd) + + if rele.shape[0] > 10: + + rele['total_price'] = rele['total_price'] / rele['area'] + # rent_most=k_means(rele['total_price']) + rele['sell_rent_ratio'] = 1 + rv = rele[['total_price', 'district_name', + 'area', 'sell_rent_ratio']].dropna() + rv1 = rv.values + ro = [] + sell_rent_ratio = [] for k in range(len(rv1)): - namee=rv1[k][1] - pee1= rele1[rele1['district_name']==namee] - pee2=pee1.sort_values(by='p_date',ascending=False) - price=pee2['total_price'].iloc[0] + namee = rv1[k][1] + pee1 = rele1[rele1['district_name'] == namee] + pee2 = pee1.sort_values(by='p_date', ascending=False) + price = pee2['total_price'].iloc[0] ro.append(price) - #price=rv1[k][0] + # price=rv1[k][0] if namee in dd.index: - #print(namee) - sell=dd[namee] - ppoo=sell/price + # print(namee) + sell = dd[namee] + ppoo = sell / price sell_rent_ratio.append(ppoo) - #rv1[k][3]=sell/price - ##print('rele0=',rele) - rent_most=k_means(pandas.Series(ro)) - #rele=pd.DataFrame(rv1,columns=rv.columns) - roseries=pandas.Series(sell_rent_ratio) - - roseries=roseries[roseries>5] - - #rele=rele[rele['sell_rent_ratio']>5] - ###print('sell_rent_ratio=',list(rele['sell_rent_ratio'])[:5]) - ###print(rele) - ###print('#######################') - ###print(rele.shape[0]) - - #print('len(roseries)=',len(roseries)) - - #print(roseries) - if len(roseries)>10: - - #rent_most=k_means(rele['total_price']) - - ###print('rele.shape[0]=',rele.shape[0]) - ##print('yyyyyyyyyyyyyyyyyyyyyy=',len(roseries)) - rent_sell=k_means(roseries.dropna()) - + # rv1[k][3]=sell/price + # print('rele0=',rele) + rent_most = k_means(pandas.Series(ro)) + # rele=pd.DataFrame(rv1,columns=rv.columns) + roseries = pandas.Series(sell_rent_ratio) + + roseries = roseries[roseries > 5] + + # rele=rele[rele['sell_rent_ratio']>5] + # print('sell_rent_ratio=',list(rele['sell_rent_ratio'])[:5]) + # print(rele) + # print('#######################') + # print(rele.shape[0]) + + # print('len(roseries)=',len(roseries)) + + # print(roseries) + if len(roseries) > 10: + + # rent_most=k_means(rele['total_price']) + + # print('rele.shape[0]=',rele.shape[0]) + # print('yyyyyyyyyyyyyyyyyyyyyy=',len(roseries)) + rent_sell = k_means(roseries.dropna()) + del rele - return rent_sell,rent_most + return rent_sell, rent_most else: del rele - return -1,rent_most + return -1, rent_most else: del rele - return -1,-1 + return -1, -1 else: - return -1,-1 - + return -1, -1 -def dff(i,j,timd,timm): #666666666666666666666666666666666666666666666666666666 - reef=rent_partial_sell(i,j,timd,timm) - ###print('$$$$$$$$$$') - - if reef[0]>0: + +def dff(i, j, timd, timm): # 666666666666666666666666666666666666666666666666666666 + reef = rent_partial_sell(i, j, timd, timm) + # print('$$$$$$$$$$') + + if reef[0] > 0: return reef else: - ###print('&&&&&&&&&&&&&') - return rent_partial_sell(i,None,timd,timm) - - + # print('&&&&&&&&&&&&&') + return rent_partial_sell(i, None, timd, timm) + + def add_eatype(ppd): if 'estate_type2' not in ppd.columns: - ppd['estate_type2']='普通住宅' + ppd['estate_type2'] = '普通住宅' return ppd - def timestamp_to_int(dd): - - dd['week1']=1 - - for i in dd.index: - - f1=dd['weekend'][i].strftime("%Y-%m-%d") + + dd['week1'] = 1 + + for i in dd.index: + + f1 = dd['weekend'][i].strftime("%Y-%m-%d") f2 = re.sub("\D", "", f1) - f3=int(f2) - dd['week1'][i]=f3 - ####print(dd) + f3 = int(f2) + dd['week1'][i] = f3 + # print(dd) return dd ''' @@ -259,72 +303,74 @@ def before_month_lastday(ti): ''' -def before_month_lastday(ti,num): +def before_month_lastday(ti, num): from dateutil.parser import parse - today=dateutil.parser.parse(str(ti)) - + today = dateutil.parser.parse(str(ti)) + #first = datetime.date(day=1, month=today.month, year=today.year) - + lastMonth = today - datetime.timedelta(days=num) - + def plus(k): - if k<10: - return '0'+str(k) + if k < 10: + return '0' + str(k) else: return str(k) - y=lastMonth.year - m=lastMonth.month - #d=lastMonth.day - day=calendar.monthrange(y,m)[1] + y = lastMonth.year + m = lastMonth.month + # d=lastMonth.day + day = calendar.monthrange(y, m)[1] - cc=str(y)+plus(m)+plus(day) - bb=parse(cc) + cc = str(y) + plus(m) + plus(day) + bb = parse(cc) pacific = pytz.timezone('Asia/Shanghai') return pacific.localize(bb) -def region_make_up_price(ct,rl,lk,ti): - - lgg=fangjia.get_collection("fangjia",config.get_sqm_rentprice_db(), config.get_mongodb_host(),27017) - #save_lg=client136.fangjia_stat.test_lg5 - - cc=before_month_lastday(ti,32) - ##print('ti=',ti) - ##print('cc=',cc) - +def region_make_up_price(ct, rl, lk, ti): + + lgg = fangjia.get_collection( + "fangjia", config.get_sqm_rentprice_db(), config.get_mongodb_host(), 27017) + # save_lg=client136.fangjia_stat.test_lg5 + + cc = before_month_lastday(ti, 32) + # print('ti=',ti) + # print('cc=',cc) + #stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) - stat=lgg - query = {"city":ct,"region":rl,"cat":"region",'weekend':cc,'district_name':None} - - pos1=list(stat.find(query).sort([("weekend",-1)]).limit(3)) - - pos=pandas.DataFrame(pos1) - - #pos=timestamp_to_int(pos) - #pos=pos[pos['week1']>cc] - - ###print('5'*10) - ###print('uuuuuuuuuuuuuuuu=',pos) - - - if pos.shape[0]>=1: - pos=pos.sort_values(by='weekend',ascending=False) - pp1=pos['avg_price'].iloc[0] - ##print('pp1=',pp1) - if type(pp1)!=None and type(pp1)!=str: + stat = lgg + query = {"city": ct, "region": rl, "cat": "region", + 'weekend': cc, 'district_name': None} + + pos1 = list(stat.find(query).sort([("weekend", -1)]).limit(3)) + + pos = pandas.DataFrame(pos1) + + # pos=timestamp_to_int(pos) + # pos=pos[pos['week1']>cc] + + # print('5'*10) + # print('uuuuuuuuuuuuuuuu=',pos) + + if pos.shape[0] >= 1: + pos = pos.sort_values(by='weekend', ascending=False) + pp1 = pos['avg_price'].iloc[0] + # print('pp1=',pp1) + if type(pp1) != None and type(pp1) != str: return pp1 else: - return None + return None else: return None - -def return_operate(ppu,mm): - - if ppu==None: + + +def return_operate(ppu, mm): + + if ppu == None: return 0 else: - return int(10000*(mm-ppu)/ppu) - + return int(10000 * (mm - ppu) / ppu) + ''' def before_month_lastday(ti): @@ -338,106 +384,95 @@ def before_month_lastday(ti): return int(cc) ''' - -#dd1=sample_pool(ct,rl,timd,timm) + +# dd1=sample_pool(ct,rl,timd,timm) +def save_region_feature(ct, rl, timd, timm): + # a=time.time() + region_feature = dff(ct, rl, timd, timm) -def save_region_feature(ct,rl,timd,timm): - #a=time.time() - region_feature=dff(ct,rl,timd,timm) - - ##print('region_feature=',region_feature) - - mmup=region_make_up_price(ct,rl,None,timm) - ##print('mmup_region=',mmup) + # print('region_feature=',region_feature) + + mmup = region_make_up_price(ct, rl, None, timm) + # print('mmup_region=',mmup) #print('this month region price',region_feature[1]) - markup=return_operate(mmup,region_feature[1]) - + markup = return_operate(mmup, region_feature[1]) + def c_date(): - date1=datetime.datetime.now() - + date1 = datetime.datetime.now() + pacific = pytz.timezone('Asia/Shanghai') return pacific.localize(date1) - - - - + def get_weekend(tt): - + from dateutil.parser import parse - bb=parse(tt) + bb = parse(tt) pacific = pytz.timezone('Asia/Shanghai') return pacific.localize(bb) - - + def month_lastday(ti): from dateutil.parser import parse - today=dateutil.parser.parse(str(ti)) - + today = dateutil.parser.parse(str(ti)) + #first = datetime.date(day=1, month=today.month, year=today.year) - - lastMonth = today# - datetime.timedelta(days=35) - + + lastMonth = today # - datetime.timedelta(days=35) + def plus(k): - if k<10: - return '0'+str(k) + if k < 10: + return '0' + str(k) else: return str(k) - y=lastMonth.year - m=lastMonth.month - #d=lastMonth.day - day=calendar.monthrange(y,m)[1] + y = lastMonth.year + m = lastMonth.month + # d=lastMonth.day + day = calendar.monthrange(y, m)[1] - cc=str(y)+plus(m)+plus(day) - bb=parse(cc) + cc = str(y) + plus(m) + plus(day) + bb = parse(cc) pacific = pytz.timezone('Asia/Shanghai') return pacific.localize(bb) - - timme=month_lastday(timm) - ###print('region_timme=',timme) - - + + timme = month_lastday(timm) + # print('region_timme=',timme) + #stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) - ###print("######################## ","ct=",ct,"rl=",rl,"makeup=",make_up) - - + # print("######################## ","ct=",ct,"rl=",rl,"makeup=",make_up) + #save_lg = fangjia.get_collection("fangjia_stat", "rent_1221", config.get_district_stat_host(), config.get_district_stat_port()) - - - date=c_date() - - temp = fangjia.get_collection("fangjia",config.get_rent_temp_db(), '192.168.0.136',27017) - + date = c_date() + + temp = fangjia.get_collection( + "fangjia", config.get_rent_temp_db(), '192.168.0.136', 27017) + temp.replace_one( - - {"city":ct,"region":rl,"weekend":timme,"cat":'region'}, - - {"city":ct,"region":rl,"district_name":None,"markup":markup,'ratio':region_feature[0], - - "avg_price":region_feature[1], "c_date":date,"weekend":timme,"cat":'region'},True - ) - avg=region_feature[1] - - if avg>0: - avg=float(avg) - save_lg = fangjia.get_collection("fangjia",config.get_sqm_rentprice_db(), config.get_mongodb_host(),27017) - + {"city": ct, "region": rl, "weekend": timme, "cat": 'region'}, + + {"city": ct, "region": rl, "district_name": None, "markup": markup, 'ratio': region_feature[0], + + "avg_price": region_feature[1], "c_date": date, "weekend": timme, "cat": 'region'}, True + ) + + avg = region_feature[1] + + if avg > 0: + avg = float(avg) + save_lg = fangjia.get_collection( + "fangjia", config.get_sqm_rentprice_db(), config.get_mongodb_host(), 27017) + save_lg.replace_one( - - {"city":ct,"region":rl,"weekend":timme,"cat":'region'}, - - {"city":ct,"region":rl,"district_name":None,"markup":markup,'ratio':region_feature[0], - - "avg_price":region_feature[1], "c_date":date,"weekend":timme,"cat":'region'},True - ) - - ###print(time.time()-a) -####print(len(disname),len(disname)) + {"city": ct, "region": rl, "weekend": timme, "cat": 'region'}, + {"city": ct, "region": rl, "district_name": None, "markup": markup, 'ratio': region_feature[0], - - + "avg_price": region_feature[1], "c_date": date, "weekend": timme, "cat": 'region'}, True + ) + + # print(time.time()-a) + +# print(len(disname),len(disname)) diff --git a/PriceSystem/rent_test.py b/PriceSystem/rent_test.py index 3eb3733..a61e235 100644 --- a/PriceSystem/rent_test.py +++ b/PriceSystem/rent_test.py @@ -22,8 +22,8 @@ rl='闵行' lk='上海康城'#上海闵行闵浦新苑一村普陀曹杨一村上海北京东城左安漪园 #黄浦伯爵居静安泰安里长宁新泾北苑上海奉贤海岸景苑闵行陇兴别墅阳江雅苑养生城二期 -timm=20180430 -timd=20170531 +timm=20181130 +timd=20171130 a=time.time() rentPrice.save_to_mongo(ct,rl,lk,timd,timm)