diff --git a/PriceSystem/portrit/district_portrit.py b/PriceSystem/portrit/district_portrit.py new file mode 100644 index 0000000..615e821 --- /dev/null +++ b/PriceSystem/portrit/district_portrit.py @@ -0,0 +1,663 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Dec 27 15:48:51 2017 + +@author: Administrator +""" +import pymongo +import pandas +import re +from collections import Counter +import numpy +import collections +import potential +from sklearn.cluster import KMeans +import pandas as pd +import numpy as np + + + +client105= pymongo.MongoClient('192.168.0.105',27777) +db105 = client105.fangjia +seawater105 = db105.seawater + + +client136= pymongo.MongoClient('192.168.0.136',27017) +db136 = client136.fangjia +seawater136 = db136.seawater +luogan=client136.luogan +import time + + + + +def some_district_information(ct,rl,lk): + #db = client.fangjia + #seawater = db.seawater + #seawater.find_one() + query = {"city":ct,"region":rl, + "district_name":lk} + + fields={"city":1,"region":1,"district_name":1, + + "height":1,"room":1,"hall":1,"area":1} + + #lt= seawater105.count(query) + #print(lt) + pos = list( seawater105.find(query,fields).limit(100)) + data=pandas.DataFrame(pos) + return data + + +def take_seaweed(ct,rl,lk): + #db = client.fangjia + #seawater = db.seawater + #seawater.find_one() + query = {"city":ct,"region":rl, + "name":lk} + + fields={"city":1,"region":1,"name":1,"build_type":1, + "finish_building_date":1,"open_date":1, + "build_date":1,"creat_year":1} + + lt= db136.seaweed.count(query) + #print(lt) + pos = list( db136.seaweed.find(query,fields).limit(lt)) + data=pandas.DataFrame(pos) + return data + +def k_means(pp,kk): + + pv=list(pp) + if len(set(pv))>2: + gf=numpy.array([pv]).T + estimator = KMeans(n_clusters=kk)#构造聚类器 + estimator.fit(gf)#聚类 + label_pred = estimator.labels_ #获取聚类标签 + aa=collections.Counter(label_pred) + v=pandas.Series(aa) + gg=list(v) + index_max=gg.index(max(gg)) + + centroids = estimator.cluster_centers_ #获取聚类中心 + #inertia = estimator.inertia_ # 获取聚类准则的总和 + center=centroids[index_max][0] + return (int(center)) + else: + return pp.mean() + + +def are(dd): + ff='area' + + if dd.shape[0]>0: + + if ff in dd.columns: + + dd1=dd[ff].dropna() + if len(dd1)>2: + area=k_means(dd1,3) + + return area + else: + return None + else: + return None + else: + return None + + + + +def kmcenter(pp,m): + + pv=list(pp) + if len(set(pv))>=2: + gf=numpy.array([pv]).T + estimator = KMeans(n_clusters=m)#构造聚类器 + estimator.fit(gf)#聚类 + label_pred = estimator.labels_ #获取聚类标签 + aa=collections.Counter(label_pred) + v=pandas.Series(aa) + gg=list(v) + #print(gg) + #index_max=gg.index(max(gg)) + + centroids = estimator.cluster_centers_ #获取聚类中心 + #inertia = estimator.inertia_ # 获取聚类准则的总和 + value=list(centroids[:,0]) + #center=centroids[index_max][0] + out=pandas.Series(gg,index=value) + return out + + + + +def chose_room_area(dd): + + #print('dd=',dd) + + tyty=['hall','room','area'] + + + a1=np.array([None]*12) + a2=a1.reshape(3,4) + a3=pandas.DataFrame(a2,columns=['room','area','credit','df']) + + wq=set(dd.columns) + + if set(tyty).issubset(wq): + p=dd[tyty] + rm1=set(p['room'].dropna()) + room_set=list(rm1) + + qq=pandas.Series() + + #qm=pandas.DataFrame() + #构造一个只含有None 的dataframe + + for i in room_set: + pm=p[p['room']==i] + + #print('p=',p) + + if pm.shape[0]>6: + + parea=pm['area'].dropna() + parea1=(parea//2)*2 + + if len(parea1)>3: + + + area_most= Counter(parea1).most_common(3) + nn=room_set.index(i) + if len(area_most)==1: + + + #qq[i]=area_most[0][0] + a3['room'][nn]=i + a3['area'][nn]=area_most[0][0] + a3['credit'][nn]=1 + a3['df'][nn]=pandas.DataFrame() + + elif len(area_most)==2 and (area_most[0][1]/area_most[1][1])>1.5: + + qq[i]=area_most[0][0] + + a3['room'][nn]=i + a3['area'][nn]=area_most[0][0] + a3['credit'][nn]=0.9 + a3['df'][nn]=pandas.DataFrame() + elif (parea1.max()- parea1.min())/ parea1.min() <0.1: + a3['room'][nn]=i + a3['area'][nn]=k_means(parea1,2) + a3['credit'][nn]=0.9 + a3['df'][nn]=pandas.DataFrame() + + elif len(area_most)==2: + + amt1=pandas.Series(dict(area_most)) + amt11=amt1.sort_values(ascending=False) + + #通过计数得出的每种面积的数量 + cter=list(amt11) + #通过计数得出的每种面积 + cindex=list(amt11.index) + amt2=kmcenter(parea1,2) + #print('parea1=',parea1) + + #print('amt2=',amt2) + amt22=amt2.sort_values(ascending=False) + #通过聚类得出每种面积的数量 + kmter=list(amt22) + #通过聚类得出每种面积 + kmindex=list(amt22.index) + + out=pandas.DataFrame() + out['count_number']=cter + out['count_area']=cindex + out['km_number']=kmter + out['km_area']=kmindex + + + a3['room'][nn]=i + a3['area'][nn]=area_most[0][0] + a3['credit'][nn]=0.8 + a3['df'][nn]=out + + + + + else: + + amt1=pandas.Series(dict(area_most)) + amt11=amt1.sort_values(ascending=False) + + #通过计数得出的每种面积的数量 + cter=list(amt11) + #通过计数得出的每种面积 + cindex=list(amt11.index) + amt2=kmcenter(parea1,3) + #print('parea1=',parea1) + + #print('amt2=',amt2) + amt22=amt2.sort_values(ascending=False) + #通过聚类得出每种面积的数量 + kmter=list(amt22) + #通过聚类得出每种面积 + kmindex=list(amt22.index) + + out=pandas.DataFrame() + out['count_number']=cter + out['count_area']=cindex + out['km_number']=kmter + out['km_area']=kmindex + + + a3['room'][nn]=i + a3['area'][nn]=area_most[0][0] + a3['credit'][nn]=0.8 + a3['df'][nn]=out + + + #qq.append(out) + return a3 + return a3 + + + + +def room_area_select(room_area): + a1=room_area.dropna() + + kk1=np.array([None]*6) + kk2=kk1.reshape(3,2) + kk3=pandas.DataFrame(kk2,columns=['room','area']) + + a2=a1[a1['credit']>0.8] + for kk in list(a2.index): + + kk3['area'][kk]=a1['area'][kk] + kk3['room'][kk]=a1['room'][kk] + #已经选取的room 对应的index + kk4=kk3.dropna() + ready_list=list(kk4.index) + + a3=a1.drop(ready_list) + + kk4_room=list(kk4['room']) + #a3_room=list(a3['room']) + print('kk4_room=',kk4_room) + for i in list(a3.index): + + a3_area=a3['area'][i] + a3_room=a3['room'][i] + + def yy(kk4_room,a3_room): + if len(kk4_room)>0: + for n in kk4_room: + if abs(n-a3_room)==1: + return n + else: + return -1 + else: + return -1 + vv_room=yy(kk4_room,a3_room) + + + def oppd(aditon,kk3,kk4,a3_room,i): + + print('aditon=',aditon) + + cb_max=aditon['count_number'].max() + + print('cb_max=',cb_max) + + count_number_index=pd.Series( list(aditon.index) , + index=list(aditon['count_number']) + + ) + + print('count_number_index=',count_number_index) + + adition_index=count_number_index[cb_max] + + print('adition_index=',adition_index) + + def mix(adition_area): + if str(type(adition_area))!="": + return adition_area + else: + return adition_area.iloc[0] + + + + adition_area=aditon['count_area'][adition_index] + + adition_area1=mix(adition_area) + kk3['area'][i]=adition_area1 + kk3['room'][i]=a3_room + kk4_room.append(i) + + return kk3,kk4_room + print('kk4=',kk4) + kk5=kk4[kk4['room']==vv_room] + #if vv_room>0: + if kk5.shape[0]>0: + + #kk5=kk4[kk4['room']==vv_room] + print('kk5=',kk5) + kk5_area=kk5['area'].iloc[0] + #print('kk5_area=',kk5_area) + + if vv_room>a3_room: + if kk5_area-a3_area>10: + kk3['area'][i]=a3_area + kk3['room'][i]=a3_room + kk4_room.append(i) + else: + aditon=a3['df'][i] + aditon1=aditon[aditon['count_area']0: + addg=oppd(aditon1,kk3,kk4,a3_room,i) + kk3=addg[0] + kk4_room=addg[1] + else: + if kk5_area-a3_area<-10: + kk3['area'][i]=a3_area + kk3['room'][i]=a3_room + kk4_room.append(i) + else: + aditon=a3['df'][i] + aditon1=aditon[aditon['count_area']>kk5_area+10] + if aditon1.shape[0]>0: + ########################################### + addg=oppd(aditon1,kk3,kk4,a3_room,i) + kk3=addg[0] + kk4_room=addg[1] + ############################################ + + else: + aditon=a3['df'][i] + print('yyy=',aditon) + ##################### + print(aditon,kk3,kk4,a3_room,i) + addg=oppd(aditon,kk3,kk4,a3_room,i) + kk3=addg[0] + print('lll=',kk3) + kk4_room=addg[1] + ##################### + return kk3 + + + + +def feature(ct,rl,lk): + aa=time.time() + dd= some_district_information(ct,rl,lk) + + print('上=',time.time()-aa) + + room_area=chose_room_area(dd) + print('room_area=',room_area) + vvb=room_area_select(room_area) + print('vvb=',vvb) + + def hrh(ff): + + if dd.shape[0]>3: + + #room_area=chose_room_area(dd) + #print('room_area=',room_area) + #vvb=room_area_select(room_area) + #print('vvb=',vvb) + + + + if ff in dd.columns: + + dd1=dd[ff].dropna() + if len(dd1)>0: + height=Counter(dd1).most_common(1)[0][0] + print('*'*5) + return height*1.0 + else: + return None + else: + return None + else: + return None + bb=time.time() + aab=['height','room','hall'] + """ + height=hrh(dd,'height') + room=hrh(dd,'room') + hall=hrh(dd,'hall') + """ + bbb=list(map(hrh,aab)) + area=are(dd) + print('下=',time.time()-bb) + return bbb[0] ,bbb[1], bbb[2],area,vvb + +def open_d(dd,ff): + if dd.shape[0]>0: + + if ff in dd.columns: + + dd1=dd[ff].dropna() + #print('dd1=',dd1) + if len(dd1)>0: + height=list(dd1)[0].strftime("%Y") + print('!'*5) + return height + else: + return None + else: + return None + else: + return None + +def fin(dd,ff): + if dd.shape[0]>0: + + if ff in dd.columns: + + dd1=dd[ff].dropna() + if len(dd1)>0: + + height=re.sub("\D","",list(dd1)[0])[:4] + print('!'*5) + return height + else: + return None + else: + return None + else: + return None + +def buid(dd,ff): + if dd.shape[0]>0: + + if ff in dd.columns: + + dd1=dd[ff].dropna() + if len(dd1)>0: + height=re.sub("\D", "", list(dd1)[0]) + print('!'*5) + return height + else: + return None + else: + return None + else: + return None + +def add_ff(ppd,hr): + if hr not in ppd.columns: + ppd[hr]= None + print('¥'*5) + return ppd + +def ttt(ct,rl,lk): + df=take_seaweed(ct,rl,lk) + df=add_ff(df,'open_date') + df=add_ff(df,'finish_building_date') + df=add_ff(df,'build_date') + + + open_date=open_d(df,'open_date') + finish=fin(df,'finish_building_date') + + build= buid(df,'build_date') + + tt=[(open_date),(finish),(build)] + + tt1=pandas.Series(tt) + tt11=tt1.dropna() + tt2=list(tt11) + def lgtc(tt2): + b=[] + for i in tt2: + if len(i)>2: + b.append(i) + return b + tt2=lgtc(tt2) + + #print(lk,'tt1=',tt2) + + if len(tt2)>0: + + tt3=[] + for i in tt2: + if i!=None: + #print('#'*5,'i=',i) + + + tt3.append(int(i)) + #print('tt3=',tt3) + if len(tt3)>0: + + return tt3[0] + else: + return None + else: + return None + + + + + +def city_list(): + + city = db136.server_area + #seawater.find_one() + + pos= city.find({"display":{"$ne":"全国"}}, + {"display":1,"city_grade":1,"_id":1}).sort([("city_grade",1),("_id",1)]) + data=pandas.DataFrame(list(pos)) + #print(data) + return data['display'] + + + + + +def region_list(city): + #client = pymongo.MongoClient('192.168.0.136',27017) + #db = client.fangjia + region = db136.region_block + #seawater.find_one() + query = {"city":city,"category":"region"} + fields1 = {"name":1} + pos = list(region.find(query,fields1)) + + + data=pandas.DataFrame(pos) + return data['name'] + +def region_district_list(ct,rl): + #import pandas + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + + query1 = {"status":0,"cat":"district","city":ct,"region":rl} + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, + "cat":1,"name":1,'estate_type2':1,'alias':1} + + lf=potential.find_seaweed(query1,fields1) + return lf + +cit=city_list() + +client182= pymongo.MongoClient('192.168.10.182',27017) + +db=client182.fangjia.portrait_0516 +for ct in cit[2:]: + + #ppr=15 + + region=region_list(ct)#[ppr:ppr+1] + + for rl in region: + + lf=region_district_list(ct,rl) + + + + if lf.shape[0]>1: + + + for lk in lf['name']: + vv=feature(ct,rl,lk) + print('#'*30) + print(ct,rl,lk,vv) + + def operate_out(vv): + + ww= {"city":ct,"region":rl,"district_name":lk, + "height":vv[0], "room":vv[1], + "hall":vv[2],"area":vv[3],'birth':ttt(ct,rl,lk)} + print('vv[4]==',vv[4]) + #vv[4].to_excel('vv4.xls') + ww1=vv[4].dropna() + + print('ww1',ww1.values) + + + for h in ww1.index: + print('ww=',ww1['area'][h]) + ww['room'+str(h)]=ww1['area'][h] + #print('ww=',ww1['area'][h]) + return ww + + + + + + db.replace_one( + + {"city":ct,"region":rl,"district_name":lk,}, + + operate_out(vv) + + ,True + ) + #time.sleep(3) + + + + + + + + + + + + + + + + + diff --git a/PriceSystem/portrit/potential/__init__.py b/PriceSystem/portrit/potential/__init__.py new file mode 100644 index 0000000..634f471 --- /dev/null +++ b/PriceSystem/portrit/potential/__init__.py @@ -0,0 +1,100 @@ +from potential import mongoClients as mongoClients +import time +import pandas + +def get_collection(db, name, host="192.168.0.136", port=27017): + return mongoClients.MongoClients(db, name, host, port).collection + +db136="192.168.0.136" + +db222='222.73.196.152' + + +def findHouse(query, fields=None): + + seawater105 = get_collection("fangjia", "district_stat", db136, 27017) + for err_count in range(0, 10): + + try: + + lt= seawater105.count(query) + pos = list(seawater105.find(query).limit(lt)) + data=pandas.DataFrame(pos) + del pos + return data + + except : + + #err_count = err_count + 1 + lt= seawater105.count(query) + pos = list(seawater105.find(query).limit(lt)) + data=pandas.DataFrame(pos) + del pos + return data + + + + + + + +def find_seaweed(query, fields=None): + + #dirlt=p[0] + + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + + seawater105 = get_collection("fangjia", "seaweed", "192.168.0.136", 27017) + for err_count in range(0, 10): + + try: + + lt= seawater105.count(query) + + pos = list(seawater105.find(query).limit(lt)) + data=pandas.DataFrame(pos) + del pos + return data + + except : + + #err_count = err_count + 1 + lt= seawater105.count(query) + pos = list(seawater105.find(query).limit(lt)) + data=pandas.DataFrame(pos) + del pos + return data + + raise Exception("yang ben cha xun shi bai1") + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/PriceSystem/portrit/potential/mongoClients.py b/PriceSystem/portrit/potential/mongoClients.py new file mode 100644 index 0000000..9c1f3c2 --- /dev/null +++ b/PriceSystem/portrit/potential/mongoClients.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Nov 15 22:45:06 2017 + +@author: wenyan +""" + +from pymongo import MongoClient + + +class MongoClients: + + cache = {} + + def __init__(self, db, name, host="127.0.0.0", port=27017): + + key = db + "_" + name + "_" + host + "_" + str(port) + if key in MongoClients.cache.keys(): + self.collection = MongoClients.cache[key] + else: + print("init " + key) + collection = MongoClient(host, port).get_database(db).get_collection(name) + MongoClients.cache[key] = collection + self.collection = MongoClients.cache[key] + """ + def find(self, *args, **kwargs): + return self.collection.find(*args, **kwargs) + + def find2(self, query, fields, call): + + for item in self.collection.find(query): + call(item) + """ diff --git a/PriceSystem/test.py b/PriceSystem/test.py index 66a0c54..dbfc837 100644 --- a/PriceSystem/test.py +++ b/PriceSystem/test.py @@ -21,8 +21,16 @@ s = time.time() ct='太原' -rl='小店' -lk='兴武小区'#杨浦凤城四村闵行上海康城厦门同安大唐世家二期青岛市北东方小区合肥庐阳海亮红玺台青岛市北东方小区 +rl='万柏林' +lk='大唐世家玉河苑' +''' + +ct='太原' +rl='万柏林' +lk='下元重机宿舍' +''' + +#杨浦凤城四村闵行上海康城厦门同安大唐世家二期青岛市北东方小区合肥庐阳海亮红玺台青岛市北东方小区 #深圳龙岗浪背村宝安吉祥公寓门头沟 泷悦长安北京上海闵行金盛小区江苏昆山象屿珑庭###浦东交通新村 diff --git a/logs/myeasylog.log b/logs/myeasylog.log new file mode 100644 index 0000000..e69de29