From cd712cc6f8066d5f9706c2b9f26fc01196b72b67 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Thu, 8 Mar 2018 11:46:52 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PriceSystem/test.py | 4 +- PriceSystem/zhuge/zhuge_consist.py | 618 +++++++++++++---------------- 2 files changed, 276 insertions(+), 346 deletions(-) diff --git a/PriceSystem/test.py b/PriceSystem/test.py index 4202f01..72af330 100644 --- a/PriceSystem/test.py +++ b/PriceSystem/test.py @@ -6,8 +6,8 @@ import sell_price.zhuge0118A as sellPrice s = time.time() ct = '上海' -rl = '闵行' -lk = '上海康城' +rl = '嘉定' +lk = '莱茵北郡' weekend = datetime.datetime.strptime("2018-03-11", "%Y-%m-%d") sellPrice.out_put_price(ct, rl, lk, weekend) diff --git a/PriceSystem/zhuge/zhuge_consist.py b/PriceSystem/zhuge/zhuge_consist.py index aadbfdf..aa7b9bc 100644 --- a/PriceSystem/zhuge/zhuge_consist.py +++ b/PriceSystem/zhuge/zhuge_consist.py @@ -11,23 +11,11 @@ import numpy import collections import re from fangjia import timer -@timer -def url_lable(dmi1): - - company_list=['lianjia','zhuge','5i5j','iwjw','fangdd','centanet', 'cityhouse','baixing','ganji','anjuke','58'] - ##有Bug - for p in company_list: - for k in dmi1.index: - ff=dmi1['url'][k] - if type(ff)==str: - if p in dmi1['url'][k]: - dmi1.loc[k,'url']=p - return dmi1 def url_to_root(url): - company_list = ['lianjia', 'zhuge', '5i5j', 'iwjw', 'fangdd', 'centanet', 'cityhouse', 'baixing', 'ganji', - 'anjuke', '58'] + company_list = ['lianjia', 'zhuge', '5i5j', 'iwjw', 'fangdd', 'centanet', 'cityhouse', 'baixing', 'ganji', + 'anjuke', '58.com'] for c in company_list: if c in url: return c @@ -35,370 +23,334 @@ def url_to_root(url): return url -# @timer -# def url_lable(dmi1): -# -# house_list = dmi1.to_dict('records') -# for h in house_list: -# h["url"] = url_to_root(h["url"]) -# dmi1 = pandas.DataFrame(house_list) -# return dmi1 +@timer +def url_lable(dmi1): + house_list = dmi1.to_dict('records') + for h in house_list: + h["url"] = url_to_root(h["url"]) + dmi1 = pandas.DataFrame(house_list) + return dmi1 + - def drop_rubbish(dd1): - - for i in dd1.index: - pp=dd1['url'][i] - - # print(pp) - - if type(pp)==str: - - if len(pp)>12 : - dd1=dd1.drop(i) - return dd1 - + for i in dd1.index: + pp = dd1['url'][i] -#dd=drop_rubbish(dd1) + # print(pp) + + if type(pp) == str: + + if len(pp) > 12: + dd1 = dd1.drop(i) + return dd1 + + +# dd=drop_rubbish(dd1) @timer def zhuge_out(ddd): - - #print('我在诸葛文件内',ddd) - - def k_means(pp,clus): + # print('我在诸葛文件内',ddd) + + def k_means(pp, clus): from sklearn.cluster import KMeans - pv=list(pp) - if len(set(pv))>clus: - gf=numpy.array([pv]).T - estimator = KMeans(n_clusters=clus)#构造聚类器 - estimator.fit(gf)#聚类 - label_pred = estimator.labels_ #获取聚类标签 - - aa=collections.Counter(label_pred) - - #print('aa=',aa) - v=pandas.Series(aa) - gg=list(v) - index_max=gg.index(max(gg)) - - centroids = estimator.cluster_centers_ #获取聚类中心 - #inertia = estimator.inertia_ # 获取聚类准则的总和 - center=centroids[index_max][0] + pv = list(pp) + if len(set(pv)) > clus: + gf = numpy.array([pv]).T + estimator = KMeans(n_clusters=clus) # 构造聚类器 + estimator.fit(gf) # 聚类 + label_pred = estimator.labels_ # 获取聚类标签 + + aa = collections.Counter(label_pred) + + # print('aa=',aa) + v = pandas.Series(aa) + gg = list(v) + index_max = gg.index(max(gg)) + + centroids = estimator.cluster_centers_ # 获取聚类中心 + # inertia = estimator.inertia_ # 获取聚类准则的总和 + center = centroids[index_max][0] return int(center) else: return int(pp.mean()) - + def spilit_number_from_string(dd): - + for pl in dd.index: - bte=dd['build_date'][pl] - if bte!=None: - dd['build_date'][pl]=re.sub("\D", "", str(bte)) - - most_build=Counter(dd['build_date']).most_common(1) - dd['build_date']=dd['build_date'].fillna(most_build[0][0]) - - return dd[dd['build_date']==most_build[0][0]] + bte = dd['build_date'][pl] + if bte != None: + dd['build_date'][pl] = re.sub("\D", "", str(bte)) + + most_build = Counter(dd['build_date']).most_common(1) + dd['build_date'] = dd['build_date'].fillna(most_build[0][0]) + + return dd[dd['build_date'] == most_build[0][0]] def lower_dimention(dd): - - url_set=list( set(dd['url']) ) - - url_price=pandas.Series() - - index_list=[] + + url_set = list(set(dd['url'])) + + url_price = pandas.Series() + + index_list = [] for i in url_set: - dd1=dd[dd['url']==i] - if dd1.shape[0]<=2: + dd1 = dd[dd['url'] == i] + if dd1.shape[0] <= 2: index_list.extend(list(dd1.index)) else: - most_price=k_means(dd1['total_price'],3)*1.0 - - url_price[i]=round(most_price,2) - + most_price = k_means(dd1['total_price'], 3) * 1.0 + + url_price[i] = round(most_price, 2) + for j in index_list: - url_price[dd['url'][j]]=dd['total_price'][j] - + url_price[dd['url'][j]] = dd['total_price'][j] + return url_price - - - def k_means_ratio(pp,clus): + + def k_means_ratio(pp, clus): from sklearn.cluster import KMeans - pv=list(pp) - if len(set(pv))>3: - gf=numpy.array([pv]).T - estimator = KMeans(n_clusters=clus)#构造聚类器 - estimator.fit(gf)#聚类 - label_pred = estimator.labels_ #获取聚类标签 - - aa=collections.Counter(label_pred) - - #print('aa=',aa) - v=pandas.Series(aa) - v=v.sort_values(ascending=False) - - ratio=v.iloc[0]/v.iloc[1] - - return round(ratio,2) + pv = list(pp) + if len(set(pv)) > 3: + gf = numpy.array([pv]).T + estimator = KMeans(n_clusters=clus) # 构造聚类器 + estimator.fit(gf) # 聚类 + label_pred = estimator.labels_ # 获取聚类标签 + + aa = collections.Counter(label_pred) + + # print('aa=',aa) + v = pandas.Series(aa) + v = v.sort_values(ascending=False) + + ratio = v.iloc[0] / v.iloc[1] + + return round(ratio, 2) else: return 1 - - - def popo(dmi,columns): - - most=Counter(dmi[columns]).most_common(1) - #print(dmi) - dmi2=dmi[dmi[columns]==most[0][0]] - #print('dmi2=',dmi2) - - dmiy=lower_dimention(dmi2) - #print(dmiy) - ratio2=(dmiy.max()-dmiy.min())/dmiy.mean() - ww1=round(k_means( dmiy ,3 ),2) - ww2=round(k_means(dmi2['area'].dropna(),3),2) - ww3=round(k_means(dmi2['room'].dropna(),3),2) - ww4=list(dmiy.index) - - #ww5=round(k_means_ratio(dmi['total_price'].dropna(),3),2) - ww5=round(k_means_ratio(dmiy,3),2) - #print('dmi2=',dmi2) - - if (ratio2<0.15 and len(dmiy)>1) or (ratio2>1.2 and len(dmiy)>1) : - - return [round(ww1,2),round(ww2,2),round(ww3,2),round(100,2),ww4,ww5] - - #print('area_filter',dd) + + def popo(dmi, columns): + + most = Counter(dmi[columns]).most_common(1) + # print(dmi) + dmi2 = dmi[dmi[columns] == most[0][0]] + # print('dmi2=',dmi2) + + dmiy = lower_dimention(dmi2) + # print(dmiy) + ratio2 = (dmiy.max() - dmiy.min()) / dmiy.mean() + ww1 = round(k_means(dmiy, 3), 2) + ww2 = round(k_means(dmi2['area'].dropna(), 3), 2) + ww3 = round(k_means(dmi2['room'].dropna(), 3), 2) + ww4 = list(dmiy.index) + + # ww5=round(k_means_ratio(dmi['total_price'].dropna(),3),2) + ww5 = round(k_means_ratio(dmiy, 3), 2) + # print('dmi2=',dmi2) + + if (ratio2 < 0.15 and len(dmiy) > 1) or (ratio2 > 1.2 and len(dmiy) > 1): + return [round(ww1, 2), round(ww2, 2), round(ww3, 2), round(100, 2), ww4, ww5] + + # print('area_filter',dd) """ else: return [round(ww1,2),round(ww2,2),round(ww3,2),round(len(dmiy),2),ww4,ww5] """ - - - - - def room_define(dd,j): - - pdd=numpy.array([[None,None,None,None,None,None], - [None,None,None,None,None,None], - [None,None,None,None,None,None]]) - - - d4=dd[dd['room']==j] - - #print('room=',j,'的样本个数为',d4.shape[0]) - - pdd[0]=popo(d4,'room') - - most_area=Counter(d4['area']).most_common(1) - - - - if most_area[0][1]>=2: - - d4area=d4[d4['area']==most_area[0][0]] - - pdd[1]=popo(d4area,'area') - - most_floor=Counter(d4area['floor']).most_common(1) - - if most_floor[0][1]>=2: - d4floor=d4area[d4area['floor']==most_floor[0][0]] - #print('d4floor=',d4floor) - pdd[2]=popo(d4floor,'floor') - - return ( pdd ) - - - def dout(room_pair,dd): - - bbb=[] - #print('room_pair=',room_pair) - #print('dd=',dd) + + def room_define(dd, j): + + pdd = numpy.array([[None, None, None, None, None, None], + [None, None, None, None, None, None], + [None, None, None, None, None, None]]) + + d4 = dd[dd['room'] == j] + + # print('room=',j,'的样本个数为',d4.shape[0]) + + pdd[0] = popo(d4, 'room') + + most_area = Counter(d4['area']).most_common(1) + + if most_area[0][1] >= 2: + + d4area = d4[d4['area'] == most_area[0][0]] + + pdd[1] = popo(d4area, 'area') + + most_floor = Counter(d4area['floor']).most_common(1) + + if most_floor[0][1] >= 2: + d4floor = d4area[d4area['floor'] == most_floor[0][0]] + # print('d4floor=',d4floor) + pdd[2] = popo(d4floor, 'floor') + + return (pdd) + + def dout(room_pair, dd): + + bbb = [] + # print('room_pair=',room_pair) + # print('dd=',dd) for j in room_pair.index: - - kgg=pandas.DataFrame(room_define(dd,j), - columns=['total_price','area','room','credit','url_list','choose_index']) - kgg['avg_price']=kgg['total_price']/kgg['area'] - - #print('kgg=',kgg) - - bbb.append(kgg) + kgg = pandas.DataFrame(room_define(dd, j), + columns=['total_price', 'area', 'room', 'credit', 'url_list', 'choose_index']) + kgg['avg_price'] = kgg['total_price'] / kgg['area'] + + # print('kgg=',kgg) + + bbb.append(kgg) return bbb - - - - - - - - def dsf_average_price(ct,rl,lk,timd,timm): - - client = pymongo.MongoClient('192.168.0.136',27017) #'192.168.0.65',27777 + + def dsf_average_price(ct, rl, lk, timd, timm): + + client = pymongo.MongoClient('192.168.0.136', 27017) # '192.168.0.65',27777 db = client.fangjia seawater = db.average_statistics - - timd1=timd//100 - timm1=timm//100 - - query = {"city":ct,"region":rl, - "name":{"$in":lk}, - "s_date":{"$gt":timd1,"$lt":timm1}} - #lt= seawater.count(query) - #print(lt) + + timd1 = timd // 100 + timm1 = timm // 100 + + query = {"city": ct, "region": rl, + "name": {"$in": lk}, + "s_date": {"$gt": timd1, "$lt": timm1}} + # lt= seawater.count(query) + # print(lt) pos = list(seawater.find(query).limit(3000)) - - data=pandas.DataFrame(pos) + + data = pandas.DataFrame(pos) del pos return data - - - dsf_name_list={'soufang_new_price','soufang_esf_price', - 'anjuke_new_price','anjuke_esf_price', - 'sina_new_price','sina_esf_price', - 'ganji_esf_price','cric_new_price', - 'cric_esf_price', - 'lianjia_esf_price','zhugezhaofang_esf_price' - - 'fanggugu_esf_price','yungujia_esf_price' - } - - - - - - - company_list=['lianjia','zhuge','5i5j','iwjw','fangdd','centanet', - 'cityhouse','baixing','ganji','anjuke','58'] - - company_name=['链家','诸葛找房','我爱我家','爱屋吉屋','房多多','中原', - '城市房产','百姓','赶集','安居客','58同城'] - company_values=[10, 10, 5, 5, 3, 2, - 1 , 1, 1, 1, 1] - - #companypp=[] + + dsf_name_list = {'soufang_new_price', 'soufang_esf_price', + 'anjuke_new_price', 'anjuke_esf_price', + 'sina_new_price', 'sina_esf_price', + 'ganji_esf_price', 'cric_new_price', + 'cric_esf_price', + 'lianjia_esf_price', 'zhugezhaofang_esf_price' + + 'fanggugu_esf_price', 'yungujia_esf_price' + } + + company_list = ['lianjia', 'zhuge', '5i5j', 'iwjw', 'fangdd', 'centanet', + 'cityhouse', 'baixing', 'ganji', 'anjuke', '58'] + + company_name = ['链家', '诸葛找房', '我爱我家', '爱屋吉屋', '房多多', '中原', + '城市房产', '百姓', '赶集', '安居客', '58同城'] + company_values = [10, 10, 5, 5, 3, 2, + 1, 1, 1, 1, 1] + + # companypp=[] import numpy as np - - a=np.array([company_name,company_values]) - - company_pandas=pandas.DataFrame(a.T,columns=['name','values'],index=company_list) - name_pair=pandas.Series(company_list,index=company_name) - - + + a = np.array([company_name, company_values]) + + company_pandas = pandas.DataFrame(a.T, columns=['name', 'values'], index=company_list) + name_pair = pandas.Series(company_list, index=company_name) + def add_floor(ppd): if 'floor' not in ppd.columns: - ppd['floor']= 6 + ppd['floor'] = 6 return ppd - - #ddd=url_lable(company_list,ddd) - - - @timer - def opfm(ddo,fm): - floor_column=ddo[fm].dropna() - if len(floor_column)>0: - - #print('floor_column=',floor_column) - most_floor=Counter(floor_column).most_common(1) - #print('floor=',ddo['floor']) - ddo[fm]=ddo[fm].fillna(most_floor[0][0]) + # ddd=url_lable(company_list,ddd) + + @timer + def opfm(ddo, fm): + + floor_column = ddo[fm].dropna() + if len(floor_column) > 0: + + # print('floor_column=',floor_column) + most_floor = Counter(floor_column).most_common(1) + # print('floor=',ddo['floor']) + ddo[fm] = ddo[fm].fillna(most_floor[0][0]) return ddo else: - ddo[fm]=ddo[fm].fillna(2) + ddo[fm] = ddo[fm].fillna(2) return ddo - - - - - + @timer def export_pp(ddd): - - ddo=ddd - #ddo=ddo.drop(0) - #print('ddo=',ddo) - ddo=add_floor(ddo) - #print('ddo=',ddo) - - if ddo.shape[0]>1: - - - ddo=opfm(ddo,'floor') - ddo=opfm(ddo,'room') - - - ddo['area']=(ddo['area']//2)*2+1 - - #dd1=url_lable(company_list,ddo) - - #dd=drop_rubbish(dd1) - - #print(Counter(dd['url'])) - dd=ddo - mr=Counter(dd['room']) - room_pair=pandas.Series(mr) - room_pair1=room_pair.sort_values(ascending=False) - room_pair=room_pair1.iloc[:3] - opt=dout(room_pair,dd) + ddo = ddd + # ddo=ddo.drop(0) + # print('ddo=',ddo) + ddo = add_floor(ddo) + # print('ddo=',ddo) + + if ddo.shape[0] > 1: + + ddo = opfm(ddo, 'floor') + ddo = opfm(ddo, 'room') + + ddo['area'] = (ddo['area'] // 2) * 2 + 1 + + # dd1=url_lable(company_list,ddo) + + # dd=drop_rubbish(dd1) + + # print(Counter(dd['url'])) + dd = ddo + mr = Counter(dd['room']) + room_pair = pandas.Series(mr) + room_pair1 = room_pair.sort_values(ascending=False) + room_pair = room_pair1.iloc[:3] + opt = dout(room_pair, dd) return opt else: return pandas.DataFrame() - - opt=export_pp(ddd) - - #print('opt=',opt) - - def k_means22(pp,clus): + + opt = export_pp(ddd) + + # print('opt=',opt) + + def k_means22(pp, clus): from sklearn.cluster import KMeans - pv=list(pp) - if len(set(pv))>1: - gf=numpy.array([pv]).T - estimator = KMeans(n_clusters=clus)#构造聚类器 - estimator.fit(gf)#聚类 - label_pred = estimator.labels_ #获取聚类标签 - - aa=collections.Counter(label_pred) - - #print('aa=',aa) - v=pandas.Series(aa) - gg=list(v) - index_max=gg.index(max(gg)) - - centroids = estimator.cluster_centers_ #获取聚类中心 - #inertia = estimator.inertia_ # 获取聚类准则的总和 - center=centroids[index_max][0] - return round(center,2) + pv = list(pp) + if len(set(pv)) > 1: + gf = numpy.array([pv]).T + estimator = KMeans(n_clusters=clus) # 构造聚类器 + estimator.fit(gf) # 聚类 + label_pred = estimator.labels_ # 获取聚类标签 + + aa = collections.Counter(label_pred) + + # print('aa=',aa) + v = pandas.Series(aa) + gg = list(v) + index_max = gg.index(max(gg)) + + centroids = estimator.cluster_centers_ # 获取聚类中心 + # inertia = estimator.inertia_ # 获取聚类准则的总和 + center = centroids[index_max][0] + return round(center, 2) else: return pp.mean() - - + def out_put_price(opt): - + re = pandas.concat(opt) - - #print(re) - - oo=re[re['credit']==100] - oo1=re.dropna() - - #print(oo) - if oo.shape[0]>1: - return round(k_means22( oo['avg_price'] ,2 ),2) - elif oo1.shape[0]>0: - - re1=re.sort_values(by='credit',ascending=False) - re2=re1.sort_values(by='choose_index',ascending=False) - #print(re2) - #print('#'*20) + + # print(re) + + oo = re[re['credit'] == 100] + oo1 = re.dropna() + + # print(oo) + if oo.shape[0] > 1: + return round(k_means22(oo['avg_price'], 2), 2) + elif oo1.shape[0] > 0: + + re1 = re.sort_values(by='credit', ascending=False) + re2 = re1.sort_values(by='choose_index', ascending=False) + # print(re2) + # print('#'*20) return (re2['avg_price'].iloc[0]) else: - #print(ddd['avg'].dropna()) - kkm=k_means(ddd['avg'].dropna(),2) + # print(ddd['avg'].dropna()) + kkm = k_means(ddd['avg'].dropna(), 2) return kkm - + return int(out_put_price(opt)) @@ -435,25 +387,3 @@ ddd=some_district_information(ct,rl,lk,timd,timm) df=zhuge_out(ddd) """ - - - - - - - - - - - - - - - - - - - - - -