新文件: PriceSystem/portrit/district_portrit.py

新文件:   PriceSystem/portrit/potential/__init__.py
	新文件:   PriceSystem/portrit/potential/mongoClients.py
This commit is contained in:
罗干
2018-05-16 19:20:55 +08:00
parent 30971b657c
commit f9a36721e9
5 changed files with 806 additions and 2 deletions
+663
View File
@@ -0,0 +1,663 @@
# -*- coding: utf-8 -*-
"""
Created on Wed Dec 27 15:48:51 2017
@author: Administrator
"""
import pymongo
import pandas
import re
from collections import Counter
import numpy
import collections
import potential
from sklearn.cluster import KMeans
import pandas as pd
import numpy as np
client105= pymongo.MongoClient('192.168.0.105',27777)
db105 = client105.fangjia
seawater105 = db105.seawater
client136= pymongo.MongoClient('192.168.0.136',27017)
db136 = client136.fangjia
seawater136 = db136.seawater
luogan=client136.luogan
import time
def some_district_information(ct,rl,lk):
#db = client.fangjia
#seawater = db.seawater
#seawater.find_one()
query = {"city":ct,"region":rl,
"district_name":lk}
fields={"city":1,"region":1,"district_name":1,
"height":1,"room":1,"hall":1,"area":1}
#lt= seawater105.count(query)
#print(lt)
pos = list( seawater105.find(query,fields).limit(100))
data=pandas.DataFrame(pos)
return data
def take_seaweed(ct,rl,lk):
#db = client.fangjia
#seawater = db.seawater
#seawater.find_one()
query = {"city":ct,"region":rl,
"name":lk}
fields={"city":1,"region":1,"name":1,"build_type":1,
"finish_building_date":1,"open_date":1,
"build_date":1,"creat_year":1}
lt= db136.seaweed.count(query)
#print(lt)
pos = list( db136.seaweed.find(query,fields).limit(lt))
data=pandas.DataFrame(pos)
return data
def k_means(pp,kk):
pv=list(pp)
if len(set(pv))>2:
gf=numpy.array([pv]).T
estimator = KMeans(n_clusters=kk)#构造聚类器
estimator.fit(gf)#聚类
label_pred = estimator.labels_ #获取聚类标签
aa=collections.Counter(label_pred)
v=pandas.Series(aa)
gg=list(v)
index_max=gg.index(max(gg))
centroids = estimator.cluster_centers_ #获取聚类中心
#inertia = estimator.inertia_ # 获取聚类准则的总和
center=centroids[index_max][0]
return (int(center))
else:
return pp.mean()
def are(dd):
ff='area'
if dd.shape[0]>0:
if ff in dd.columns:
dd1=dd[ff].dropna()
if len(dd1)>2:
area=k_means(dd1,3)
return area
else:
return None
else:
return None
else:
return None
def kmcenter(pp,m):
pv=list(pp)
if len(set(pv))>=2:
gf=numpy.array([pv]).T
estimator = KMeans(n_clusters=m)#构造聚类器
estimator.fit(gf)#聚类
label_pred = estimator.labels_ #获取聚类标签
aa=collections.Counter(label_pred)
v=pandas.Series(aa)
gg=list(v)
#print(gg)
#index_max=gg.index(max(gg))
centroids = estimator.cluster_centers_ #获取聚类中心
#inertia = estimator.inertia_ # 获取聚类准则的总和
value=list(centroids[:,0])
#center=centroids[index_max][0]
out=pandas.Series(gg,index=value)
return out
def chose_room_area(dd):
#print('dd=',dd)
tyty=['hall','room','area']
a1=np.array([None]*12)
a2=a1.reshape(3,4)
a3=pandas.DataFrame(a2,columns=['room','area','credit','df'])
wq=set(dd.columns)
if set(tyty).issubset(wq):
p=dd[tyty]
rm1=set(p['room'].dropna())
room_set=list(rm1)
qq=pandas.Series()
#qm=pandas.DataFrame()
#构造一个只含有None 的dataframe
for i in room_set:
pm=p[p['room']==i]
#print('p=',p)
if pm.shape[0]>6:
parea=pm['area'].dropna()
parea1=(parea//2)*2
if len(parea1)>3:
area_most= Counter(parea1).most_common(3)
nn=room_set.index(i)
if len(area_most)==1:
#qq[i]=area_most[0][0]
a3['room'][nn]=i
a3['area'][nn]=area_most[0][0]
a3['credit'][nn]=1
a3['df'][nn]=pandas.DataFrame()
elif len(area_most)==2 and (area_most[0][1]/area_most[1][1])>1.5:
qq[i]=area_most[0][0]
a3['room'][nn]=i
a3['area'][nn]=area_most[0][0]
a3['credit'][nn]=0.9
a3['df'][nn]=pandas.DataFrame()
elif (parea1.max()- parea1.min())/ parea1.min() <0.1:
a3['room'][nn]=i
a3['area'][nn]=k_means(parea1,2)
a3['credit'][nn]=0.9
a3['df'][nn]=pandas.DataFrame()
elif len(area_most)==2:
amt1=pandas.Series(dict(area_most))
amt11=amt1.sort_values(ascending=False)
#通过计数得出的每种面积的数量
cter=list(amt11)
#通过计数得出的每种面积
cindex=list(amt11.index)
amt2=kmcenter(parea1,2)
#print('parea1=',parea1)
#print('amt2=',amt2)
amt22=amt2.sort_values(ascending=False)
#通过聚类得出每种面积的数量
kmter=list(amt22)
#通过聚类得出每种面积
kmindex=list(amt22.index)
out=pandas.DataFrame()
out['count_number']=cter
out['count_area']=cindex
out['km_number']=kmter
out['km_area']=kmindex
a3['room'][nn]=i
a3['area'][nn]=area_most[0][0]
a3['credit'][nn]=0.8
a3['df'][nn]=out
else:
amt1=pandas.Series(dict(area_most))
amt11=amt1.sort_values(ascending=False)
#通过计数得出的每种面积的数量
cter=list(amt11)
#通过计数得出的每种面积
cindex=list(amt11.index)
amt2=kmcenter(parea1,3)
#print('parea1=',parea1)
#print('amt2=',amt2)
amt22=amt2.sort_values(ascending=False)
#通过聚类得出每种面积的数量
kmter=list(amt22)
#通过聚类得出每种面积
kmindex=list(amt22.index)
out=pandas.DataFrame()
out['count_number']=cter
out['count_area']=cindex
out['km_number']=kmter
out['km_area']=kmindex
a3['room'][nn]=i
a3['area'][nn]=area_most[0][0]
a3['credit'][nn]=0.8
a3['df'][nn]=out
#qq.append(out)
return a3
return a3
def room_area_select(room_area):
a1=room_area.dropna()
kk1=np.array([None]*6)
kk2=kk1.reshape(3,2)
kk3=pandas.DataFrame(kk2,columns=['room','area'])
a2=a1[a1['credit']>0.8]
for kk in list(a2.index):
kk3['area'][kk]=a1['area'][kk]
kk3['room'][kk]=a1['room'][kk]
#已经选取的room 对应的index
kk4=kk3.dropna()
ready_list=list(kk4.index)
a3=a1.drop(ready_list)
kk4_room=list(kk4['room'])
#a3_room=list(a3['room'])
print('kk4_room=',kk4_room)
for i in list(a3.index):
a3_area=a3['area'][i]
a3_room=a3['room'][i]
def yy(kk4_room,a3_room):
if len(kk4_room)>0:
for n in kk4_room:
if abs(n-a3_room)==1:
return n
else:
return -1
else:
return -1
vv_room=yy(kk4_room,a3_room)
def oppd(aditon,kk3,kk4,a3_room,i):
print('aditon=',aditon)
cb_max=aditon['count_number'].max()
print('cb_max=',cb_max)
count_number_index=pd.Series( list(aditon.index) ,
index=list(aditon['count_number'])
)
print('count_number_index=',count_number_index)
adition_index=count_number_index[cb_max]
print('adition_index=',adition_index)
def mix(adition_area):
if str(type(adition_area))!="<class 'pandas.core.series.Series'>":
return adition_area
else:
return adition_area.iloc[0]
adition_area=aditon['count_area'][adition_index]
adition_area1=mix(adition_area)
kk3['area'][i]=adition_area1
kk3['room'][i]=a3_room
kk4_room.append(i)
return kk3,kk4_room
print('kk4=',kk4)
kk5=kk4[kk4['room']==vv_room]
#if vv_room>0:
if kk5.shape[0]>0:
#kk5=kk4[kk4['room']==vv_room]
print('kk5=',kk5)
kk5_area=kk5['area'].iloc[0]
#print('kk5_area=',kk5_area)
if vv_room>a3_room:
if kk5_area-a3_area>10:
kk3['area'][i]=a3_area
kk3['room'][i]=a3_room
kk4_room.append(i)
else:
aditon=a3['df'][i]
aditon1=aditon[aditon['count_area']<kk5_area-10]
if aditon1.shape[0]>0:
addg=oppd(aditon1,kk3,kk4,a3_room,i)
kk3=addg[0]
kk4_room=addg[1]
else:
if kk5_area-a3_area<-10:
kk3['area'][i]=a3_area
kk3['room'][i]=a3_room
kk4_room.append(i)
else:
aditon=a3['df'][i]
aditon1=aditon[aditon['count_area']>kk5_area+10]
if aditon1.shape[0]>0:
###########################################
addg=oppd(aditon1,kk3,kk4,a3_room,i)
kk3=addg[0]
kk4_room=addg[1]
############################################
else:
aditon=a3['df'][i]
print('yyy=',aditon)
#####################
print(aditon,kk3,kk4,a3_room,i)
addg=oppd(aditon,kk3,kk4,a3_room,i)
kk3=addg[0]
print('lll=',kk3)
kk4_room=addg[1]
#####################
return kk3
def feature(ct,rl,lk):
aa=time.time()
dd= some_district_information(ct,rl,lk)
print('上=',time.time()-aa)
room_area=chose_room_area(dd)
print('room_area=',room_area)
vvb=room_area_select(room_area)
print('vvb=',vvb)
def hrh(ff):
if dd.shape[0]>3:
#room_area=chose_room_area(dd)
#print('room_area=',room_area)
#vvb=room_area_select(room_area)
#print('vvb=',vvb)
if ff in dd.columns:
dd1=dd[ff].dropna()
if len(dd1)>0:
height=Counter(dd1).most_common(1)[0][0]
print('*'*5)
return height*1.0
else:
return None
else:
return None
else:
return None
bb=time.time()
aab=['height','room','hall']
"""
height=hrh(dd,'height')
room=hrh(dd,'room')
hall=hrh(dd,'hall')
"""
bbb=list(map(hrh,aab))
area=are(dd)
print('下=',time.time()-bb)
return bbb[0] ,bbb[1], bbb[2],area,vvb
def open_d(dd,ff):
if dd.shape[0]>0:
if ff in dd.columns:
dd1=dd[ff].dropna()
#print('dd1=',dd1)
if len(dd1)>0:
height=list(dd1)[0].strftime("%Y")
print('!'*5)
return height
else:
return None
else:
return None
else:
return None
def fin(dd,ff):
if dd.shape[0]>0:
if ff in dd.columns:
dd1=dd[ff].dropna()
if len(dd1)>0:
height=re.sub("\D","",list(dd1)[0])[:4]
print('!'*5)
return height
else:
return None
else:
return None
else:
return None
def buid(dd,ff):
if dd.shape[0]>0:
if ff in dd.columns:
dd1=dd[ff].dropna()
if len(dd1)>0:
height=re.sub("\D", "", list(dd1)[0])
print('!'*5)
return height
else:
return None
else:
return None
else:
return None
def add_ff(ppd,hr):
if hr not in ppd.columns:
ppd[hr]= None
print('¥'*5)
return ppd
def ttt(ct,rl,lk):
df=take_seaweed(ct,rl,lk)
df=add_ff(df,'open_date')
df=add_ff(df,'finish_building_date')
df=add_ff(df,'build_date')
open_date=open_d(df,'open_date')
finish=fin(df,'finish_building_date')
build= buid(df,'build_date')
tt=[(open_date),(finish),(build)]
tt1=pandas.Series(tt)
tt11=tt1.dropna()
tt2=list(tt11)
def lgtc(tt2):
b=[]
for i in tt2:
if len(i)>2:
b.append(i)
return b
tt2=lgtc(tt2)
#print(lk,'tt1=',tt2)
if len(tt2)>0:
tt3=[]
for i in tt2:
if i!=None:
#print('#'*5,'i=',i)
tt3.append(int(i))
#print('tt3=',tt3)
if len(tt3)>0:
return tt3[0]
else:
return None
else:
return None
def city_list():
city = db136.server_area
#seawater.find_one()
pos= city.find({"display":{"$ne":"全国"}},
{"display":1,"city_grade":1,"_id":1}).sort([("city_grade",1),("_id",1)])
data=pandas.DataFrame(list(pos))
#print(data)
return data['display']
def region_list(city):
#client = pymongo.MongoClient('192.168.0.136',27017)
#db = client.fangjia
region = db136.region_block
#seawater.find_one()
query = {"city":city,"category":"region"}
fields1 = {"name":1}
pos = list(region.find(query,fields1))
data=pandas.DataFrame(pos)
return data['name']
def region_district_list(ct,rl):
#import pandas
#client1 = pymongo.MongoClient('192.168.0.136',27017)
#db1 = client1.fangjia
#seaweed1 = db1.seaweed
query1 = {"status":0,"cat":"district","city":ct,"region":rl}
fields1 = {"lat2":1,"lng2":1, "city":1,"region":1,
"cat":1,"name":1,'estate_type2':1,'alias':1}
lf=potential.find_seaweed(query1,fields1)
return lf
cit=city_list()
client182= pymongo.MongoClient('192.168.10.182',27017)
db=client182.fangjia.portrait_0516
for ct in cit[2:]:
#ppr=15
region=region_list(ct)#[ppr:ppr+1]
for rl in region:
lf=region_district_list(ct,rl)
if lf.shape[0]>1:
for lk in lf['name']:
vv=feature(ct,rl,lk)
print('#'*30)
print(ct,rl,lk,vv)
def operate_out(vv):
ww= {"city":ct,"region":rl,"district_name":lk,
"height":vv[0], "room":vv[1],
"hall":vv[2],"area":vv[3],'birth':ttt(ct,rl,lk)}
print('vv[4]==',vv[4])
#vv[4].to_excel('vv4.xls')
ww1=vv[4].dropna()
print('ww1',ww1.values)
for h in ww1.index:
print('ww=',ww1['area'][h])
ww['room'+str(h)]=ww1['area'][h]
#print('ww=',ww1['area'][h])
return ww
db.replace_one(
{"city":ct,"region":rl,"district_name":lk,},
operate_out(vv)
,True
)
#time.sleep(3)
+100
View File
@@ -0,0 +1,100 @@
from potential import mongoClients as mongoClients
import time
import pandas
def get_collection(db, name, host="192.168.0.136", port=27017):
return mongoClients.MongoClients(db, name, host, port).collection
db136="192.168.0.136"
db222='222.73.196.152'
def findHouse(query, fields=None):
seawater105 = get_collection("fangjia", "district_stat", db136, 27017)
for err_count in range(0, 10):
try:
lt= seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data=pandas.DataFrame(pos)
del pos
return data
except :
#err_count = err_count + 1
lt= seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data=pandas.DataFrame(pos)
del pos
return data
def find_seaweed(query, fields=None):
#dirlt=p[0]
#client1 = pymongo.MongoClient('192.168.0.136',27017)
#db1 = client1.fangjia
#seaweed1 = db1.seaweed
seawater105 = get_collection("fangjia", "seaweed", "192.168.0.136", 27017)
for err_count in range(0, 10):
try:
lt= seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data=pandas.DataFrame(pos)
del pos
return data
except :
#err_count = err_count + 1
lt= seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data=pandas.DataFrame(pos)
del pos
return data
raise Exception("yang ben cha xun shi bai1")
@@ -0,0 +1,33 @@
# -*- coding: utf-8 -*-
"""
Created on Wed Nov 15 22:45:06 2017
@author: wenyan
"""
from pymongo import MongoClient
class MongoClients:
cache = {}
def __init__(self, db, name, host="127.0.0.0", port=27017):
key = db + "_" + name + "_" + host + "_" + str(port)
if key in MongoClients.cache.keys():
self.collection = MongoClients.cache[key]
else:
print("init " + key)
collection = MongoClient(host, port).get_database(db).get_collection(name)
MongoClients.cache[key] = collection
self.collection = MongoClients.cache[key]
"""
def find(self, *args, **kwargs):
return self.collection.find(*args, **kwargs)
def find2(self, query, fields, call):
for item in self.collection.find(query):
call(item)
"""
+10 -2
View File
@@ -21,8 +21,16 @@ s = time.time()
ct='太原'
rl='小店'
lk='兴武小区'#杨浦凤城四村闵行上海康城厦门同安大唐世家二期青岛市北东方小区合肥庐阳海亮红玺台青岛市北东方小区
rl='万柏林'
lk='大唐世家玉河苑'
'''
ct='太原'
rl='万柏林'
lk='下元重机宿舍'
'''
#杨浦凤城四村闵行上海康城厦门同安大唐世家二期青岛市北东方小区合肥庐阳海亮红玺台青岛市北东方小区
#深圳龙岗浪背村宝安吉祥公寓门头沟 泷悦长安北京上海闵行金盛小区江苏昆山象屿珑庭###浦东交通新村
View File