This commit is contained in:
2018-03-08 11:46:52 +08:00
parent 00b9a2ef1f
commit cd712cc6f8
2 changed files with 276 additions and 346 deletions
+2 -2
View File
@@ -6,8 +6,8 @@ import sell_price.zhuge0118A as sellPrice
s = time.time()
ct = '上海'
rl = '闵行'
lk = '上海康城'
rl = '嘉定'
lk = '莱茵北郡'
weekend = datetime.datetime.strptime("2018-03-11", "%Y-%m-%d")
sellPrice.out_put_price(ct, rl, lk, weekend)
+274 -344
View File
@@ -11,23 +11,11 @@ import numpy
import collections
import re
from fangjia import timer
@timer
def url_lable(dmi1):
company_list=['lianjia','zhuge','5i5j','iwjw','fangdd','centanet', 'cityhouse','baixing','ganji','anjuke','58']
##有Bug
for p in company_list:
for k in dmi1.index:
ff=dmi1['url'][k]
if type(ff)==str:
if p in dmi1['url'][k]:
dmi1.loc[k,'url']=p
return dmi1
def url_to_root(url):
company_list = ['lianjia', 'zhuge', '5i5j', 'iwjw', 'fangdd', 'centanet', 'cityhouse', 'baixing', 'ganji',
'anjuke', '58']
company_list = ['lianjia', 'zhuge', '5i5j', 'iwjw', 'fangdd', 'centanet', 'cityhouse', 'baixing', 'ganji',
'anjuke', '58.com']
for c in company_list:
if c in url:
return c
@@ -35,370 +23,334 @@ def url_to_root(url):
return url
# @timer
# def url_lable(dmi1):
#
# house_list = dmi1.to_dict('records')
# for h in house_list:
# h["url"] = url_to_root(h["url"])
# dmi1 = pandas.DataFrame(house_list)
# return dmi1
@timer
def url_lable(dmi1):
house_list = dmi1.to_dict('records')
for h in house_list:
h["url"] = url_to_root(h["url"])
dmi1 = pandas.DataFrame(house_list)
return dmi1
def drop_rubbish(dd1):
for i in dd1.index:
pp=dd1['url'][i]
# print(pp)
if type(pp)==str:
if len(pp)>12 :
dd1=dd1.drop(i)
return dd1
for i in dd1.index:
pp = dd1['url'][i]
#dd=drop_rubbish(dd1)
# print(pp)
if type(pp) == str:
if len(pp) > 12:
dd1 = dd1.drop(i)
return dd1
# dd=drop_rubbish(dd1)
@timer
def zhuge_out(ddd):
#print('我在诸葛文件内',ddd)
def k_means(pp,clus):
# print('我在诸葛文件内',ddd)
def k_means(pp, clus):
from sklearn.cluster import KMeans
pv=list(pp)
if len(set(pv))>clus:
gf=numpy.array([pv]).T
estimator = KMeans(n_clusters=clus)#构造聚类器
estimator.fit(gf)#聚类
label_pred = estimator.labels_ #获取聚类标签
aa=collections.Counter(label_pred)
#print('aa=',aa)
v=pandas.Series(aa)
gg=list(v)
index_max=gg.index(max(gg))
centroids = estimator.cluster_centers_ #获取聚类中心
#inertia = estimator.inertia_ # 获取聚类准则的总和
center=centroids[index_max][0]
pv = list(pp)
if len(set(pv)) > clus:
gf = numpy.array([pv]).T
estimator = KMeans(n_clusters=clus) # 构造聚类器
estimator.fit(gf) # 聚类
label_pred = estimator.labels_ # 获取聚类标签
aa = collections.Counter(label_pred)
# print('aa=',aa)
v = pandas.Series(aa)
gg = list(v)
index_max = gg.index(max(gg))
centroids = estimator.cluster_centers_ # 获取聚类中心
# inertia = estimator.inertia_ # 获取聚类准则的总和
center = centroids[index_max][0]
return int(center)
else:
return int(pp.mean())
def spilit_number_from_string(dd):
for pl in dd.index:
bte=dd['build_date'][pl]
if bte!=None:
dd['build_date'][pl]=re.sub("\D", "", str(bte))
most_build=Counter(dd['build_date']).most_common(1)
dd['build_date']=dd['build_date'].fillna(most_build[0][0])
return dd[dd['build_date']==most_build[0][0]]
bte = dd['build_date'][pl]
if bte != None:
dd['build_date'][pl] = re.sub("\D", "", str(bte))
most_build = Counter(dd['build_date']).most_common(1)
dd['build_date'] = dd['build_date'].fillna(most_build[0][0])
return dd[dd['build_date'] == most_build[0][0]]
def lower_dimention(dd):
url_set=list( set(dd['url']) )
url_price=pandas.Series()
index_list=[]
url_set = list(set(dd['url']))
url_price = pandas.Series()
index_list = []
for i in url_set:
dd1=dd[dd['url']==i]
if dd1.shape[0]<=2:
dd1 = dd[dd['url'] == i]
if dd1.shape[0] <= 2:
index_list.extend(list(dd1.index))
else:
most_price=k_means(dd1['total_price'],3)*1.0
url_price[i]=round(most_price,2)
most_price = k_means(dd1['total_price'], 3) * 1.0
url_price[i] = round(most_price, 2)
for j in index_list:
url_price[dd['url'][j]]=dd['total_price'][j]
url_price[dd['url'][j]] = dd['total_price'][j]
return url_price
def k_means_ratio(pp,clus):
def k_means_ratio(pp, clus):
from sklearn.cluster import KMeans
pv=list(pp)
if len(set(pv))>3:
gf=numpy.array([pv]).T
estimator = KMeans(n_clusters=clus)#构造聚类器
estimator.fit(gf)#聚类
label_pred = estimator.labels_ #获取聚类标签
aa=collections.Counter(label_pred)
#print('aa=',aa)
v=pandas.Series(aa)
v=v.sort_values(ascending=False)
ratio=v.iloc[0]/v.iloc[1]
return round(ratio,2)
pv = list(pp)
if len(set(pv)) > 3:
gf = numpy.array([pv]).T
estimator = KMeans(n_clusters=clus) # 构造聚类器
estimator.fit(gf) # 聚类
label_pred = estimator.labels_ # 获取聚类标签
aa = collections.Counter(label_pred)
# print('aa=',aa)
v = pandas.Series(aa)
v = v.sort_values(ascending=False)
ratio = v.iloc[0] / v.iloc[1]
return round(ratio, 2)
else:
return 1
def popo(dmi,columns):
most=Counter(dmi[columns]).most_common(1)
#print(dmi)
dmi2=dmi[dmi[columns]==most[0][0]]
#print('dmi2=',dmi2)
dmiy=lower_dimention(dmi2)
#print(dmiy)
ratio2=(dmiy.max()-dmiy.min())/dmiy.mean()
ww1=round(k_means( dmiy ,3 ),2)
ww2=round(k_means(dmi2['area'].dropna(),3),2)
ww3=round(k_means(dmi2['room'].dropna(),3),2)
ww4=list(dmiy.index)
#ww5=round(k_means_ratio(dmi['total_price'].dropna(),3),2)
ww5=round(k_means_ratio(dmiy,3),2)
#print('dmi2=',dmi2)
if (ratio2<0.15 and len(dmiy)>1) or (ratio2>1.2 and len(dmiy)>1) :
return [round(ww1,2),round(ww2,2),round(ww3,2),round(100,2),ww4,ww5]
#print('area_filter',dd)
def popo(dmi, columns):
most = Counter(dmi[columns]).most_common(1)
# print(dmi)
dmi2 = dmi[dmi[columns] == most[0][0]]
# print('dmi2=',dmi2)
dmiy = lower_dimention(dmi2)
# print(dmiy)
ratio2 = (dmiy.max() - dmiy.min()) / dmiy.mean()
ww1 = round(k_means(dmiy, 3), 2)
ww2 = round(k_means(dmi2['area'].dropna(), 3), 2)
ww3 = round(k_means(dmi2['room'].dropna(), 3), 2)
ww4 = list(dmiy.index)
# ww5=round(k_means_ratio(dmi['total_price'].dropna(),3),2)
ww5 = round(k_means_ratio(dmiy, 3), 2)
# print('dmi2=',dmi2)
if (ratio2 < 0.15 and len(dmiy) > 1) or (ratio2 > 1.2 and len(dmiy) > 1):
return [round(ww1, 2), round(ww2, 2), round(ww3, 2), round(100, 2), ww4, ww5]
# print('area_filter',dd)
"""
else:
return [round(ww1,2),round(ww2,2),round(ww3,2),round(len(dmiy),2),ww4,ww5]
"""
def room_define(dd,j):
pdd=numpy.array([[None,None,None,None,None,None],
[None,None,None,None,None,None],
[None,None,None,None,None,None]])
d4=dd[dd['room']==j]
#print('room=',j,'的样本个数为',d4.shape[0])
pdd[0]=popo(d4,'room')
most_area=Counter(d4['area']).most_common(1)
if most_area[0][1]>=2:
d4area=d4[d4['area']==most_area[0][0]]
pdd[1]=popo(d4area,'area')
most_floor=Counter(d4area['floor']).most_common(1)
if most_floor[0][1]>=2:
d4floor=d4area[d4area['floor']==most_floor[0][0]]
#print('d4floor=',d4floor)
pdd[2]=popo(d4floor,'floor')
return ( pdd )
def dout(room_pair,dd):
bbb=[]
#print('room_pair=',room_pair)
#print('dd=',dd)
def room_define(dd, j):
pdd = numpy.array([[None, None, None, None, None, None],
[None, None, None, None, None, None],
[None, None, None, None, None, None]])
d4 = dd[dd['room'] == j]
# print('room=',j,'的样本个数为',d4.shape[0])
pdd[0] = popo(d4, 'room')
most_area = Counter(d4['area']).most_common(1)
if most_area[0][1] >= 2:
d4area = d4[d4['area'] == most_area[0][0]]
pdd[1] = popo(d4area, 'area')
most_floor = Counter(d4area['floor']).most_common(1)
if most_floor[0][1] >= 2:
d4floor = d4area[d4area['floor'] == most_floor[0][0]]
# print('d4floor=',d4floor)
pdd[2] = popo(d4floor, 'floor')
return (pdd)
def dout(room_pair, dd):
bbb = []
# print('room_pair=',room_pair)
# print('dd=',dd)
for j in room_pair.index:
kgg=pandas.DataFrame(room_define(dd,j),
columns=['total_price','area','room','credit','url_list','choose_index'])
kgg['avg_price']=kgg['total_price']/kgg['area']
#print('kgg=',kgg)
bbb.append(kgg)
kgg = pandas.DataFrame(room_define(dd, j),
columns=['total_price', 'area', 'room', 'credit', 'url_list', 'choose_index'])
kgg['avg_price'] = kgg['total_price'] / kgg['area']
# print('kgg=',kgg)
bbb.append(kgg)
return bbb
def dsf_average_price(ct,rl,lk,timd,timm):
client = pymongo.MongoClient('192.168.0.136',27017) #'192.168.0.65',27777
def dsf_average_price(ct, rl, lk, timd, timm):
client = pymongo.MongoClient('192.168.0.136', 27017) # '192.168.0.65',27777
db = client.fangjia
seawater = db.average_statistics
timd1=timd//100
timm1=timm//100
query = {"city":ct,"region":rl,
"name":{"$in":lk},
"s_date":{"$gt":timd1,"$lt":timm1}}
#lt= seawater.count(query)
#print(lt)
timd1 = timd // 100
timm1 = timm // 100
query = {"city": ct, "region": rl,
"name": {"$in": lk},
"s_date": {"$gt": timd1, "$lt": timm1}}
# lt= seawater.count(query)
# print(lt)
pos = list(seawater.find(query).limit(3000))
data=pandas.DataFrame(pos)
data = pandas.DataFrame(pos)
del pos
return data
dsf_name_list={'soufang_new_price','soufang_esf_price',
'anjuke_new_price','anjuke_esf_price',
'sina_new_price','sina_esf_price',
'ganji_esf_price','cric_new_price',
'cric_esf_price',
'lianjia_esf_price','zhugezhaofang_esf_price'
'fanggugu_esf_price','yungujia_esf_price'
}
company_list=['lianjia','zhuge','5i5j','iwjw','fangdd','centanet',
'cityhouse','baixing','ganji','anjuke','58']
company_name=['链家','诸葛找房','我爱我家','爱屋吉屋','房多多','中原',
'城市房产','百姓','赶集','安居客','58同城']
company_values=[10, 10, 5, 5, 3, 2,
1 , 1, 1, 1, 1]
#companypp=[]
dsf_name_list = {'soufang_new_price', 'soufang_esf_price',
'anjuke_new_price', 'anjuke_esf_price',
'sina_new_price', 'sina_esf_price',
'ganji_esf_price', 'cric_new_price',
'cric_esf_price',
'lianjia_esf_price', 'zhugezhaofang_esf_price'
'fanggugu_esf_price', 'yungujia_esf_price'
}
company_list = ['lianjia', 'zhuge', '5i5j', 'iwjw', 'fangdd', 'centanet',
'cityhouse', 'baixing', 'ganji', 'anjuke', '58']
company_name = ['链家', '诸葛找房', '我爱我家', '爱屋吉屋', '房多多', '中原',
'城市房产', '百姓', '赶集', '安居客', '58同城']
company_values = [10, 10, 5, 5, 3, 2,
1, 1, 1, 1, 1]
# companypp=[]
import numpy as np
a=np.array([company_name,company_values])
company_pandas=pandas.DataFrame(a.T,columns=['name','values'],index=company_list)
name_pair=pandas.Series(company_list,index=company_name)
a = np.array([company_name, company_values])
company_pandas = pandas.DataFrame(a.T, columns=['name', 'values'], index=company_list)
name_pair = pandas.Series(company_list, index=company_name)
def add_floor(ppd):
if 'floor' not in ppd.columns:
ppd['floor']= 6
ppd['floor'] = 6
return ppd
#ddd=url_lable(company_list,ddd)
@timer
def opfm(ddo,fm):
floor_column=ddo[fm].dropna()
if len(floor_column)>0:
#print('floor_column=',floor_column)
most_floor=Counter(floor_column).most_common(1)
#print('floor=',ddo['floor'])
ddo[fm]=ddo[fm].fillna(most_floor[0][0])
# ddd=url_lable(company_list,ddd)
@timer
def opfm(ddo, fm):
floor_column = ddo[fm].dropna()
if len(floor_column) > 0:
# print('floor_column=',floor_column)
most_floor = Counter(floor_column).most_common(1)
# print('floor=',ddo['floor'])
ddo[fm] = ddo[fm].fillna(most_floor[0][0])
return ddo
else:
ddo[fm]=ddo[fm].fillna(2)
ddo[fm] = ddo[fm].fillna(2)
return ddo
@timer
def export_pp(ddd):
ddo=ddd
#ddo=ddo.drop(0)
#print('ddo=',ddo)
ddo=add_floor(ddo)
#print('ddo=',ddo)
if ddo.shape[0]>1:
ddo=opfm(ddo,'floor')
ddo=opfm(ddo,'room')
ddo['area']=(ddo['area']//2)*2+1
#dd1=url_lable(company_list,ddo)
#dd=drop_rubbish(dd1)
#print(Counter(dd['url']))
dd=ddo
mr=Counter(dd['room'])
room_pair=pandas.Series(mr)
room_pair1=room_pair.sort_values(ascending=False)
room_pair=room_pair1.iloc[:3]
opt=dout(room_pair,dd)
ddo = ddd
# ddo=ddo.drop(0)
# print('ddo=',ddo)
ddo = add_floor(ddo)
# print('ddo=',ddo)
if ddo.shape[0] > 1:
ddo = opfm(ddo, 'floor')
ddo = opfm(ddo, 'room')
ddo['area'] = (ddo['area'] // 2) * 2 + 1
# dd1=url_lable(company_list,ddo)
# dd=drop_rubbish(dd1)
# print(Counter(dd['url']))
dd = ddo
mr = Counter(dd['room'])
room_pair = pandas.Series(mr)
room_pair1 = room_pair.sort_values(ascending=False)
room_pair = room_pair1.iloc[:3]
opt = dout(room_pair, dd)
return opt
else:
return pandas.DataFrame()
opt=export_pp(ddd)
#print('opt=',opt)
def k_means22(pp,clus):
opt = export_pp(ddd)
# print('opt=',opt)
def k_means22(pp, clus):
from sklearn.cluster import KMeans
pv=list(pp)
if len(set(pv))>1:
gf=numpy.array([pv]).T
estimator = KMeans(n_clusters=clus)#构造聚类器
estimator.fit(gf)#聚类
label_pred = estimator.labels_ #获取聚类标签
aa=collections.Counter(label_pred)
#print('aa=',aa)
v=pandas.Series(aa)
gg=list(v)
index_max=gg.index(max(gg))
centroids = estimator.cluster_centers_ #获取聚类中心
#inertia = estimator.inertia_ # 获取聚类准则的总和
center=centroids[index_max][0]
return round(center,2)
pv = list(pp)
if len(set(pv)) > 1:
gf = numpy.array([pv]).T
estimator = KMeans(n_clusters=clus) # 构造聚类器
estimator.fit(gf) # 聚类
label_pred = estimator.labels_ # 获取聚类标签
aa = collections.Counter(label_pred)
# print('aa=',aa)
v = pandas.Series(aa)
gg = list(v)
index_max = gg.index(max(gg))
centroids = estimator.cluster_centers_ # 获取聚类中心
# inertia = estimator.inertia_ # 获取聚类准则的总和
center = centroids[index_max][0]
return round(center, 2)
else:
return pp.mean()
def out_put_price(opt):
re = pandas.concat(opt)
#print(re)
oo=re[re['credit']==100]
oo1=re.dropna()
#print(oo)
if oo.shape[0]>1:
return round(k_means22( oo['avg_price'] ,2 ),2)
elif oo1.shape[0]>0:
re1=re.sort_values(by='credit',ascending=False)
re2=re1.sort_values(by='choose_index',ascending=False)
#print(re2)
#print('#'*20)
# print(re)
oo = re[re['credit'] == 100]
oo1 = re.dropna()
# print(oo)
if oo.shape[0] > 1:
return round(k_means22(oo['avg_price'], 2), 2)
elif oo1.shape[0] > 0:
re1 = re.sort_values(by='credit', ascending=False)
re2 = re1.sort_values(by='choose_index', ascending=False)
# print(re2)
# print('#'*20)
return (re2['avg_price'].iloc[0])
else:
#print(ddd['avg'].dropna())
kkm=k_means(ddd['avg'].dropna(),2)
# print(ddd['avg'].dropna())
kkm = k_means(ddd['avg'].dropna(), 2)
return kkm
return int(out_put_price(opt))
@@ -435,25 +387,3 @@ ddd=some_district_information(ct,rl,lk,timd,timm)
df=zhuge_out(ddd)
"""