This commit is contained in:
2018-03-08 11:46:52 +08:00
parent 00b9a2ef1f
commit cd712cc6f8
2 changed files with 276 additions and 346 deletions
+2 -2
View File
@@ -6,8 +6,8 @@ import sell_price.zhuge0118A as sellPrice
s = time.time() s = time.time()
ct = '上海' ct = '上海'
rl = '闵行' rl = '嘉定'
lk = '上海康城' lk = '莱茵北郡'
weekend = datetime.datetime.strptime("2018-03-11", "%Y-%m-%d") weekend = datetime.datetime.strptime("2018-03-11", "%Y-%m-%d")
sellPrice.out_put_price(ct, rl, lk, weekend) sellPrice.out_put_price(ct, rl, lk, weekend)
+274 -344
View File
@@ -11,23 +11,11 @@ import numpy
import collections import collections
import re import re
from fangjia import timer from fangjia import timer
@timer
def url_lable(dmi1):
company_list=['lianjia','zhuge','5i5j','iwjw','fangdd','centanet', 'cityhouse','baixing','ganji','anjuke','58']
##有Bug
for p in company_list:
for k in dmi1.index:
ff=dmi1['url'][k]
if type(ff)==str:
if p in dmi1['url'][k]:
dmi1.loc[k,'url']=p
return dmi1
def url_to_root(url): def url_to_root(url):
company_list = ['lianjia', 'zhuge', '5i5j', 'iwjw', 'fangdd', 'centanet', 'cityhouse', 'baixing', 'ganji', company_list = ['lianjia', 'zhuge', '5i5j', 'iwjw', 'fangdd', 'centanet', 'cityhouse', 'baixing', 'ganji',
'anjuke', '58'] 'anjuke', '58.com']
for c in company_list: for c in company_list:
if c in url: if c in url:
return c return c
@@ -35,370 +23,334 @@ def url_to_root(url):
return url return url
# @timer @timer
# def url_lable(dmi1): def url_lable(dmi1):
# house_list = dmi1.to_dict('records')
# house_list = dmi1.to_dict('records') for h in house_list:
# for h in house_list: h["url"] = url_to_root(h["url"])
# h["url"] = url_to_root(h["url"]) dmi1 = pandas.DataFrame(house_list)
# dmi1 = pandas.DataFrame(house_list) return dmi1
# return dmi1
def drop_rubbish(dd1): def drop_rubbish(dd1):
for i in dd1.index:
for i in dd1.index: pp = dd1['url'][i]
pp=dd1['url'][i]
# print(pp)
if type(pp)==str:
if len(pp)>12 :
dd1=dd1.drop(i)
return dd1
#dd=drop_rubbish(dd1) # print(pp)
if type(pp) == str:
if len(pp) > 12:
dd1 = dd1.drop(i)
return dd1
# dd=drop_rubbish(dd1)
@timer @timer
def zhuge_out(ddd): def zhuge_out(ddd):
# print('我在诸葛文件内',ddd)
#print('我在诸葛文件内',ddd)
def k_means(pp, clus):
def k_means(pp,clus):
from sklearn.cluster import KMeans from sklearn.cluster import KMeans
pv=list(pp) pv = list(pp)
if len(set(pv))>clus: if len(set(pv)) > clus:
gf=numpy.array([pv]).T gf = numpy.array([pv]).T
estimator = KMeans(n_clusters=clus)#构造聚类器 estimator = KMeans(n_clusters=clus) # 构造聚类器
estimator.fit(gf)#聚类 estimator.fit(gf) # 聚类
label_pred = estimator.labels_ #获取聚类标签 label_pred = estimator.labels_ # 获取聚类标签
aa=collections.Counter(label_pred) aa = collections.Counter(label_pred)
#print('aa=',aa) # print('aa=',aa)
v=pandas.Series(aa) v = pandas.Series(aa)
gg=list(v) gg = list(v)
index_max=gg.index(max(gg)) index_max = gg.index(max(gg))
centroids = estimator.cluster_centers_ #获取聚类中心 centroids = estimator.cluster_centers_ # 获取聚类中心
#inertia = estimator.inertia_ # 获取聚类准则的总和 # inertia = estimator.inertia_ # 获取聚类准则的总和
center=centroids[index_max][0] center = centroids[index_max][0]
return int(center) return int(center)
else: else:
return int(pp.mean()) return int(pp.mean())
def spilit_number_from_string(dd): def spilit_number_from_string(dd):
for pl in dd.index: for pl in dd.index:
bte=dd['build_date'][pl] bte = dd['build_date'][pl]
if bte!=None: if bte != None:
dd['build_date'][pl]=re.sub("\D", "", str(bte)) dd['build_date'][pl] = re.sub("\D", "", str(bte))
most_build=Counter(dd['build_date']).most_common(1) most_build = Counter(dd['build_date']).most_common(1)
dd['build_date']=dd['build_date'].fillna(most_build[0][0]) dd['build_date'] = dd['build_date'].fillna(most_build[0][0])
return dd[dd['build_date']==most_build[0][0]] return dd[dd['build_date'] == most_build[0][0]]
def lower_dimention(dd): def lower_dimention(dd):
url_set=list( set(dd['url']) ) url_set = list(set(dd['url']))
url_price=pandas.Series() url_price = pandas.Series()
index_list=[] index_list = []
for i in url_set: for i in url_set:
dd1=dd[dd['url']==i] dd1 = dd[dd['url'] == i]
if dd1.shape[0]<=2: if dd1.shape[0] <= 2:
index_list.extend(list(dd1.index)) index_list.extend(list(dd1.index))
else: else:
most_price=k_means(dd1['total_price'],3)*1.0 most_price = k_means(dd1['total_price'], 3) * 1.0
url_price[i]=round(most_price,2) url_price[i] = round(most_price, 2)
for j in index_list: for j in index_list:
url_price[dd['url'][j]]=dd['total_price'][j] url_price[dd['url'][j]] = dd['total_price'][j]
return url_price return url_price
def k_means_ratio(pp, clus):
def k_means_ratio(pp,clus):
from sklearn.cluster import KMeans from sklearn.cluster import KMeans
pv=list(pp) pv = list(pp)
if len(set(pv))>3: if len(set(pv)) > 3:
gf=numpy.array([pv]).T gf = numpy.array([pv]).T
estimator = KMeans(n_clusters=clus)#构造聚类器 estimator = KMeans(n_clusters=clus) # 构造聚类器
estimator.fit(gf)#聚类 estimator.fit(gf) # 聚类
label_pred = estimator.labels_ #获取聚类标签 label_pred = estimator.labels_ # 获取聚类标签
aa=collections.Counter(label_pred) aa = collections.Counter(label_pred)
#print('aa=',aa) # print('aa=',aa)
v=pandas.Series(aa) v = pandas.Series(aa)
v=v.sort_values(ascending=False) v = v.sort_values(ascending=False)
ratio=v.iloc[0]/v.iloc[1] ratio = v.iloc[0] / v.iloc[1]
return round(ratio,2) return round(ratio, 2)
else: else:
return 1 return 1
def popo(dmi, columns):
def popo(dmi,columns):
most = Counter(dmi[columns]).most_common(1)
most=Counter(dmi[columns]).most_common(1) # print(dmi)
#print(dmi) dmi2 = dmi[dmi[columns] == most[0][0]]
dmi2=dmi[dmi[columns]==most[0][0]] # print('dmi2=',dmi2)
#print('dmi2=',dmi2)
dmiy = lower_dimention(dmi2)
dmiy=lower_dimention(dmi2) # print(dmiy)
#print(dmiy) ratio2 = (dmiy.max() - dmiy.min()) / dmiy.mean()
ratio2=(dmiy.max()-dmiy.min())/dmiy.mean() ww1 = round(k_means(dmiy, 3), 2)
ww1=round(k_means( dmiy ,3 ),2) ww2 = round(k_means(dmi2['area'].dropna(), 3), 2)
ww2=round(k_means(dmi2['area'].dropna(),3),2) ww3 = round(k_means(dmi2['room'].dropna(), 3), 2)
ww3=round(k_means(dmi2['room'].dropna(),3),2) ww4 = list(dmiy.index)
ww4=list(dmiy.index)
# ww5=round(k_means_ratio(dmi['total_price'].dropna(),3),2)
#ww5=round(k_means_ratio(dmi['total_price'].dropna(),3),2) ww5 = round(k_means_ratio(dmiy, 3), 2)
ww5=round(k_means_ratio(dmiy,3),2) # print('dmi2=',dmi2)
#print('dmi2=',dmi2)
if (ratio2 < 0.15 and len(dmiy) > 1) or (ratio2 > 1.2 and len(dmiy) > 1):
if (ratio2<0.15 and len(dmiy)>1) or (ratio2>1.2 and len(dmiy)>1) : return [round(ww1, 2), round(ww2, 2), round(ww3, 2), round(100, 2), ww4, ww5]
return [round(ww1,2),round(ww2,2),round(ww3,2),round(100,2),ww4,ww5] # print('area_filter',dd)
#print('area_filter',dd)
""" """
else: else:
return [round(ww1,2),round(ww2,2),round(ww3,2),round(len(dmiy),2),ww4,ww5] return [round(ww1,2),round(ww2,2),round(ww3,2),round(len(dmiy),2),ww4,ww5]
""" """
def room_define(dd, j):
pdd = numpy.array([[None, None, None, None, None, None],
def room_define(dd,j): [None, None, None, None, None, None],
[None, None, None, None, None, None]])
pdd=numpy.array([[None,None,None,None,None,None],
[None,None,None,None,None,None], d4 = dd[dd['room'] == j]
[None,None,None,None,None,None]])
# print('room=',j,'的样本个数为',d4.shape[0])
d4=dd[dd['room']==j] pdd[0] = popo(d4, 'room')
#print('room=',j,'的样本个数为',d4.shape[0]) most_area = Counter(d4['area']).most_common(1)
pdd[0]=popo(d4,'room') if most_area[0][1] >= 2:
most_area=Counter(d4['area']).most_common(1) d4area = d4[d4['area'] == most_area[0][0]]
pdd[1] = popo(d4area, 'area')
if most_area[0][1]>=2: most_floor = Counter(d4area['floor']).most_common(1)
d4area=d4[d4['area']==most_area[0][0]] if most_floor[0][1] >= 2:
d4floor = d4area[d4area['floor'] == most_floor[0][0]]
pdd[1]=popo(d4area,'area') # print('d4floor=',d4floor)
pdd[2] = popo(d4floor, 'floor')
most_floor=Counter(d4area['floor']).most_common(1)
return (pdd)
if most_floor[0][1]>=2:
d4floor=d4area[d4area['floor']==most_floor[0][0]] def dout(room_pair, dd):
#print('d4floor=',d4floor)
pdd[2]=popo(d4floor,'floor') bbb = []
# print('room_pair=',room_pair)
return ( pdd ) # print('dd=',dd)
def dout(room_pair,dd):
bbb=[]
#print('room_pair=',room_pair)
#print('dd=',dd)
for j in room_pair.index: for j in room_pair.index:
kgg = pandas.DataFrame(room_define(dd, j),
kgg=pandas.DataFrame(room_define(dd,j), columns=['total_price', 'area', 'room', 'credit', 'url_list', 'choose_index'])
columns=['total_price','area','room','credit','url_list','choose_index']) kgg['avg_price'] = kgg['total_price'] / kgg['area']
kgg['avg_price']=kgg['total_price']/kgg['area']
# print('kgg=',kgg)
#print('kgg=',kgg)
bbb.append(kgg)
bbb.append(kgg)
return bbb return bbb
def dsf_average_price(ct, rl, lk, timd, timm):
client = pymongo.MongoClient('192.168.0.136', 27017) # '192.168.0.65',27777
def dsf_average_price(ct,rl,lk,timd,timm):
client = pymongo.MongoClient('192.168.0.136',27017) #'192.168.0.65',27777
db = client.fangjia db = client.fangjia
seawater = db.average_statistics seawater = db.average_statistics
timd1=timd//100 timd1 = timd // 100
timm1=timm//100 timm1 = timm // 100
query = {"city":ct,"region":rl, query = {"city": ct, "region": rl,
"name":{"$in":lk}, "name": {"$in": lk},
"s_date":{"$gt":timd1,"$lt":timm1}} "s_date": {"$gt": timd1, "$lt": timm1}}
#lt= seawater.count(query) # lt= seawater.count(query)
#print(lt) # print(lt)
pos = list(seawater.find(query).limit(3000)) pos = list(seawater.find(query).limit(3000))
data=pandas.DataFrame(pos) data = pandas.DataFrame(pos)
del pos del pos
return data return data
dsf_name_list = {'soufang_new_price', 'soufang_esf_price',
dsf_name_list={'soufang_new_price','soufang_esf_price', 'anjuke_new_price', 'anjuke_esf_price',
'anjuke_new_price','anjuke_esf_price', 'sina_new_price', 'sina_esf_price',
'sina_new_price','sina_esf_price', 'ganji_esf_price', 'cric_new_price',
'ganji_esf_price','cric_new_price', 'cric_esf_price',
'cric_esf_price', 'lianjia_esf_price', 'zhugezhaofang_esf_price'
'lianjia_esf_price','zhugezhaofang_esf_price'
'fanggugu_esf_price', 'yungujia_esf_price'
'fanggugu_esf_price','yungujia_esf_price' }
}
company_list = ['lianjia', 'zhuge', '5i5j', 'iwjw', 'fangdd', 'centanet',
'cityhouse', 'baixing', 'ganji', 'anjuke', '58']
company_name = ['链家', '诸葛找房', '我爱我家', '爱屋吉屋', '房多多', '中原',
'城市房产', '百姓', '赶集', '安居客', '58同城']
company_values = [10, 10, 5, 5, 3, 2,
company_list=['lianjia','zhuge','5i5j','iwjw','fangdd','centanet', 1, 1, 1, 1, 1]
'cityhouse','baixing','ganji','anjuke','58']
# companypp=[]
company_name=['链家','诸葛找房','我爱我家','爱屋吉屋','房多多','中原',
'城市房产','百姓','赶集','安居客','58同城']
company_values=[10, 10, 5, 5, 3, 2,
1 , 1, 1, 1, 1]
#companypp=[]
import numpy as np import numpy as np
a=np.array([company_name,company_values]) a = np.array([company_name, company_values])
company_pandas=pandas.DataFrame(a.T,columns=['name','values'],index=company_list) company_pandas = pandas.DataFrame(a.T, columns=['name', 'values'], index=company_list)
name_pair=pandas.Series(company_list,index=company_name) name_pair = pandas.Series(company_list, index=company_name)
def add_floor(ppd): def add_floor(ppd):
if 'floor' not in ppd.columns: if 'floor' not in ppd.columns:
ppd['floor']= 6 ppd['floor'] = 6
return ppd return ppd
#ddd=url_lable(company_list,ddd)
@timer
def opfm(ddo,fm):
floor_column=ddo[fm].dropna() # ddd=url_lable(company_list,ddd)
if len(floor_column)>0:
@timer
#print('floor_column=',floor_column) def opfm(ddo, fm):
most_floor=Counter(floor_column).most_common(1)
#print('floor=',ddo['floor']) floor_column = ddo[fm].dropna()
ddo[fm]=ddo[fm].fillna(most_floor[0][0]) if len(floor_column) > 0:
# print('floor_column=',floor_column)
most_floor = Counter(floor_column).most_common(1)
# print('floor=',ddo['floor'])
ddo[fm] = ddo[fm].fillna(most_floor[0][0])
return ddo return ddo
else: else:
ddo[fm]=ddo[fm].fillna(2) ddo[fm] = ddo[fm].fillna(2)
return ddo return ddo
@timer @timer
def export_pp(ddd): def export_pp(ddd):
ddo=ddd
#ddo=ddo.drop(0)
#print('ddo=',ddo)
ddo=add_floor(ddo)
#print('ddo=',ddo)
if ddo.shape[0]>1:
ddo=opfm(ddo,'floor')
ddo=opfm(ddo,'room')
ddo = ddd
ddo['area']=(ddo['area']//2)*2+1 # ddo=ddo.drop(0)
# print('ddo=',ddo)
#dd1=url_lable(company_list,ddo) ddo = add_floor(ddo)
# print('ddo=',ddo)
#dd=drop_rubbish(dd1)
if ddo.shape[0] > 1:
#print(Counter(dd['url']))
dd=ddo ddo = opfm(ddo, 'floor')
mr=Counter(dd['room']) ddo = opfm(ddo, 'room')
room_pair=pandas.Series(mr)
room_pair1=room_pair.sort_values(ascending=False) ddo['area'] = (ddo['area'] // 2) * 2 + 1
room_pair=room_pair1.iloc[:3]
opt=dout(room_pair,dd) # dd1=url_lable(company_list,ddo)
# dd=drop_rubbish(dd1)
# print(Counter(dd['url']))
dd = ddo
mr = Counter(dd['room'])
room_pair = pandas.Series(mr)
room_pair1 = room_pair.sort_values(ascending=False)
room_pair = room_pair1.iloc[:3]
opt = dout(room_pair, dd)
return opt return opt
else: else:
return pandas.DataFrame() return pandas.DataFrame()
opt=export_pp(ddd) opt = export_pp(ddd)
#print('opt=',opt) # print('opt=',opt)
def k_means22(pp,clus): def k_means22(pp, clus):
from sklearn.cluster import KMeans from sklearn.cluster import KMeans
pv=list(pp) pv = list(pp)
if len(set(pv))>1: if len(set(pv)) > 1:
gf=numpy.array([pv]).T gf = numpy.array([pv]).T
estimator = KMeans(n_clusters=clus)#构造聚类器 estimator = KMeans(n_clusters=clus) # 构造聚类器
estimator.fit(gf)#聚类 estimator.fit(gf) # 聚类
label_pred = estimator.labels_ #获取聚类标签 label_pred = estimator.labels_ # 获取聚类标签
aa=collections.Counter(label_pred) aa = collections.Counter(label_pred)
#print('aa=',aa) # print('aa=',aa)
v=pandas.Series(aa) v = pandas.Series(aa)
gg=list(v) gg = list(v)
index_max=gg.index(max(gg)) index_max = gg.index(max(gg))
centroids = estimator.cluster_centers_ #获取聚类中心 centroids = estimator.cluster_centers_ # 获取聚类中心
#inertia = estimator.inertia_ # 获取聚类准则的总和 # inertia = estimator.inertia_ # 获取聚类准则的总和
center=centroids[index_max][0] center = centroids[index_max][0]
return round(center,2) return round(center, 2)
else: else:
return pp.mean() return pp.mean()
def out_put_price(opt): def out_put_price(opt):
re = pandas.concat(opt) re = pandas.concat(opt)
#print(re) # print(re)
oo=re[re['credit']==100] oo = re[re['credit'] == 100]
oo1=re.dropna() oo1 = re.dropna()
#print(oo) # print(oo)
if oo.shape[0]>1: if oo.shape[0] > 1:
return round(k_means22( oo['avg_price'] ,2 ),2) return round(k_means22(oo['avg_price'], 2), 2)
elif oo1.shape[0]>0: elif oo1.shape[0] > 0:
re1=re.sort_values(by='credit',ascending=False) re1 = re.sort_values(by='credit', ascending=False)
re2=re1.sort_values(by='choose_index',ascending=False) re2 = re1.sort_values(by='choose_index', ascending=False)
#print(re2) # print(re2)
#print('#'*20) # print('#'*20)
return (re2['avg_price'].iloc[0]) return (re2['avg_price'].iloc[0])
else: else:
#print(ddd['avg'].dropna()) # print(ddd['avg'].dropna())
kkm=k_means(ddd['avg'].dropna(),2) kkm = k_means(ddd['avg'].dropna(), 2)
return kkm return kkm
return int(out_put_price(opt)) return int(out_put_price(opt))
@@ -435,25 +387,3 @@ ddd=some_district_information(ct,rl,lk,timd,timm)
df=zhuge_out(ddd) df=zhuge_out(ddd)
""" """