diff --git a/PriceSystem/Rent_Receiver.py b/PriceSystem/Rent_Receiver.py new file mode 100644 index 0000000..fca0f15 --- /dev/null +++ b/PriceSystem/Rent_Receiver.py @@ -0,0 +1,66 @@ +import json +import multiprocessing +import time +import redis +import config +import rent_price.district_map0413 as rentPrice +import fangjia +import datetime +import pytz +import os + +QUEUE = "rent_cmd" +#redis_pool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_db()) +redis_pool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_rent_redis_db()) +logger = fangjia.get_logger("Rent_Receiver.py") + + +def worker(): + client = redis.Redis(connection_pool=redis_pool) + while True: + try: + cmd = client.lpop(QUEUE) + except Exception as ex: + cmd = None + if cmd is None: + time.sleep(60) + logger.info("等待接受计算命令") + else: + cmd = format_cmd(cmd) + start_time = time.time() + try: + rentPrice.save_to_mongo(cmd["city"], cmd["region"], cmd["name"],cmd['timd'],cmd['timm']) + end_time = time.time() + logger.info("Calc SUC:" + str(cmd) + "\t计算成功,耗时:" + str(end_time - start_time)) + except BaseException as ex: + logger.exception(ex) + end_time = time.time() + logger.error("Calc ERROR :" + str(cmd) + "\t计算失败,耗时:" + str(end_time - start_time)) + + +def format_cmd(cmd): + cmd = json.loads(cmd) + + return cmd + + +if __name__ == "__main__": + user_home = os.path.expanduser('~') + lock_file = user_home + "/price.sys.stop" + if os.path.exists(lock_file): + os.remove(lock_file) + pro_num = config.get_worker_num() + pool = multiprocessing.Pool(processes=pro_num) + for pid in range(0, pro_num): + pool.apply_async(worker, ()) + pool.close() + logger.info("平米租金系统启动成功") + while True: + if os.path.exists(lock_file): + pool.terminate() + break + else: + time.sleep(3) + if os.path.exists(lock_file): + os.remove(lock_file) + pool.join() diff --git a/PriceSystem/Rent_Sender.py b/PriceSystem/Rent_Sender.py new file mode 100644 index 0000000..55d3676 --- /dev/null +++ b/PriceSystem/Rent_Sender.py @@ -0,0 +1,110 @@ +import json +import redis +import fangjia +import config +import datetime +import sys +import time + +from datetime import date + +import calendar +#import pytz + +def input_time(day): + #from dateutil.parser import parse + #today=parse(str(ti)) + + today=datetime.datetime.now() + + #first = datetime.date(day=1, month=today.month, year=today.year) + + lastMonth = today - datetime.timedelta(days=day) + + def plus(k): + if k<10: + return '0'+str(k) + else: + return str(k) + y=lastMonth.year + m=lastMonth.month + #d=lastMonth.day + day=calendar.monthrange(y,m)[1] + + cc=str(y)+plus(m)+plus(day) + #bb=parse(cc) + #pacific = pytz.timezone('Asia/Shanghai') + #return pacific.localize(bb) + return int(cc) + + +class CJsonEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, datetime.datetime): + return obj.strftime('%Y-%m-%d') + elif isinstance(obj, date): + return obj.strftime('%Y-%m-%d') + else: + return json.JSONEncoder.default(self, obj) + + +QUEUE = "rent_cmd" +#redisPool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_db()) + +redisPool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_rent_redis_db()) + +client = redis.Redis(connection_pool=redisPool) +logger = fangjia.get_logger("Rent_Sender.py") + + +def get_district(city): + timm=input_time(0) + timd=input_time(360) + seaweeds = [] + keys = {"city": 1, "region": 1, "name": 1, "_id": 0} + for dis in fangjia.get_collection("fangjia", "seaweed").find({"cat": "district", "status": 0, "city": city}, keys): + dis["timm"] = timm + dis['timd']=timd + + #dis["timm"] = 20170401 + #dis['timd']=20160401 + seaweeds.append(dis) + + return seaweeds + + +def send_cmd(seaweed): + json_cmd = json.dumps(seaweed, ensure_ascii=False, cls=CJsonEncoder) + client.rpush(QUEUE, json_cmd) + logger.info("命令发送成功:" + json_cmd) + + +def send_district_calc_cmd(): + for city in fangjia.get_all_city(): + for seaweed in get_district(city): + send_cmd(seaweed) + + + +def district_calc_cmd_finish(): + count = client.llen(QUEUE) + if count == 0: + return True + else: + return False + + + +if __name__ == "__main__": + start_time = datetime.datetime.utcnow() + send_district_calc_cmd() + while not district_calc_cmd_finish(): + time.sleep(5) + logger.info("等待小区均价计算完成") + #send_city_calc_cmd() + logger.info("城市均价计算完成") + #all_calc_cmd_finish(start_time) + sys.exit(0) + + + diff --git a/PriceSystem/Rent_SpeedMonitor.py b/PriceSystem/Rent_SpeedMonitor.py new file mode 100644 index 0000000..73be9a4 --- /dev/null +++ b/PriceSystem/Rent_SpeedMonitor.py @@ -0,0 +1,31 @@ +import json +import multiprocessing +import time +import redis +import config +import fangjia + +QUEUE = "rent_cmd" +redis_pool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_db()) + +logger = fangjia.get_logger("Rent_SpeedMonitor") + + +def monitor(pname): + client = redis.Redis(connection_pool=redis_pool) + while True: + pre_counter = client.llen(QUEUE) + time.sleep(60) + now_counter = client.llen(QUEUE) + logger.info(pname + "#####:".zfill(20) + str(pre_counter - now_counter) + "/min") + + +def format_cmd(cmd): + return json.loads(cmd) + + +if __name__ == "__main__": + pool = multiprocessing.Pool(processes=1) + pool.apply_async(monitor, ("Monitor-001",)) + pool.close() + pool.join() diff --git a/PriceSystem/fix/__pycache__/__init__.cpython-36.pyc b/PriceSystem/fix/__pycache__/__init__.cpython-36.pyc deleted file mode 100644 index c77574c..0000000 Binary files a/PriceSystem/fix/__pycache__/__init__.cpython-36.pyc and /dev/null differ diff --git a/PriceSystem/fix/__pycache__/price_fix.cpython-36.pyc b/PriceSystem/fix/__pycache__/price_fix.cpython-36.pyc deleted file mode 100644 index 6fde09d..0000000 Binary files a/PriceSystem/fix/__pycache__/price_fix.cpython-36.pyc and /dev/null differ diff --git a/PriceSystem/rent_price/__init__.py b/PriceSystem/rent_price/__init__.py new file mode 100644 index 0000000..8a42d88 --- /dev/null +++ b/PriceSystem/rent_price/__init__.py @@ -0,0 +1 @@ +from rent_price import district_map0413 diff --git a/PriceSystem/rent_price/district_map0413.py b/PriceSystem/rent_price/district_map0413.py new file mode 100644 index 0000000..99be67f --- /dev/null +++ b/PriceSystem/rent_price/district_map0413.py @@ -0,0 +1,1126 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Nov 1 18:50:26 2017 + +@author: Administrator +""" + +import os + +##print( os.getcwd()) +#from collections import Counter +import pymongo +#from pymongo import MongoClient +import numpy as np +import pandas as pd +import random +import collections +import pymongo +import sklearn.cluster +import pandas +import numpy +import sys, time + +import fangjia +import datetime +import re +import pandas +from fangjia import timer +import dateutil +import feature +import fangjia +import config +import calendar +import pytz + + + +def common_rent_price(ct,rl,p,timd,timm,region_feature,mongodb): + + + + def f_del(qq,g): + + if g!= None: + + return qq + else: + del qq['region'] + return qq + + def add_rent_type(ppd): + if 'rent_type' not in ppd.columns: + ppd['rent_type']='整租' + return ppd + + def sample_pool(ct,rl,timd,timm): + query = {"city":ct,"cat":"rent","region":rl, + "p_date":{"$gt":timd,"$lt":timm}} + #query=f_del(query,rl) + + fields1 = {"total_price":1,"area":1, "city":1,"region":1, + "district_name":1,"rent_type":1,"fitment":1,"block":1} + + + data=fangjia.find_sample_pool(query,fields1) + if data.shape[0]>10: + + ##print('数据条数=',data.shape[0]) + return data[['district_name','total_price','area']] + else: + return numpy.array([]) + + def disanganf_price(ct,rl,lk,timd,timm): + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #dsf = db1.average_statistics + #tt=timd//100 + #tt1=timm//100 + tt=pandas.Timestamp(str(timd)) + + tt1=pandas.Timestamp(str(timm)) + query = {"city":ct,"region":rl, + "district_name":lk,"weekend":{"$gt":tt,"$lt":tt1}} + fields={'city':1,'region':1,'district_name':1,'weekend':1,'calc_value':1} + #pos = list() + #数据转化为数组,数组的元素为字典 + #pos=list(dsf136.find(query)) + + #data1=pandas.DataFrame(pos) + data1=fangjia.find_disanganf_price(query,fields) + ###print('data1=',data1) + + if data1.shape[0]>1: + #data=data1[data1['credit']>0.4] + data=data1 + + if data.shape[0]>=1: + + a_set={'calc_value','weekend'} + b_set=set(data.columns) + + + if a_set & b_set==a_set: + + + #choose_class=['city','region','name','fangjia_price','credit', + # 's_date'] + ###print(data) + dc1=data.sort_values(by=['weekend'],ascending=False) + + ###print('dc1=',dc1) + dc=dc1['calc_value'].values[0] + + #dc=data['fangjia_price'].mean() + del tt,data1,data,dc1 + return dc + else: + del tt,data1,data + return 0 + else: + del tt,data1 + return 0 + + def add_block(ppd): + if 'block' not in ppd.columns: + ppd['block']='没有' + return ppd + + def add_room(ppd): + if 'room' not in ppd.columns: + ppd['room']=2 + return ppd + + + def take_location(ct,rl,p): + + dirlt=p[0] + + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + seaweed136=fangjia.get_collection("fangjia","seaweed", config.get_mongodb_host(),config.get_mongodb_port()) + query1 = {"status":0,"cat":"district","city":ct, + "region":rl, "name":dirlt} + + for err_count in range(0, 10): + + try: + ltt= seaweed136.count(query1) + if ltt>0: + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1,"cat":1,"name":1} + lct= seaweed136.find(query1, fields1) + lf=pandas.DataFrame(list(lct)) + + #lf=fangjia.find_seaweed(query1, fields1) + le=lf + le.index=le['name'] + cluns=lf.columns + if ('lng2' in cluns )and ('lat2' in cluns): + lr=le[['lng2','lat2']] + del lf,le,cluns + return lr + else: + del lf,le,cluns + return numpy.array([]) + + else: + return numpy.array([]) + except: + + ltt= seaweed136.count(query1) + if ltt>0: + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1,"cat":1,"name":1} + lct= seaweed136.find(query1, fields1) + lf=pandas.DataFrame(list(lct)) + + #lf=fangjia.find_seaweed(query1, fields1) + le=lf + le.index=le['name'] + cluns=lf.columns + if ('lng2' in cluns )and ('lat2' in cluns): + lr=le[['lng2','lat2']] + del lf,le,cluns + return lr + else: + del lf,le,cluns + return numpy.array([]) + + else: + return numpy.array([]) + + + + def some_district_information(ct,rl,lk,timd,timm): + + query = {"city":ct,"cat":"rent","region":rl,#'rent_type':'整租', + "district_name":{"$in":lk}, + "p_date":{"$gt":timd,"$lt":timm}} + + return fangjia.findHouse(query) + + + + def neighbor(ct,rl,lng,lat,scale,p): + estype=p[2] + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db136.seaweed + kkk=scale/1000 + longtitude=lng + latitude=lat + + query1 = {"status":0,"cat":"district", 'city':ct,'region':rl, + "lat2":{"$gt":int(latitude-kkk*10**10/0.86),"$lt":int(latitude+kkk*10**10/0.86)}, + "lng2":{"$gt":int(longtitude-kkk*10**10),"$lt":int(longtitude+kkk*10**10)}} + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1,"cat":1,"name":1,'estate_type2':1} + + #lct= list(seaweed136.find(query1, fields1)) + + + #lf=pandas.DataFrame(lct) + + lf=fangjia.find_seaweed(query1, fields1) + + le=lf[lf['estate_type2']==estype] + + le.index=le['name'] + + lr=le[['lng2','lat2','estate_type2']] + #lr['name1']=le['name'] + + dirst1=list(lr.index) + + dirst=dirst1[:50] + + ##print('小区个数= ',len(dirst)) + + #j=time.time() + pos=some_district_information(ct,rl,dirst,timd,timm) + + #print('j_time=',time.time()-j) + + data=pandas.DataFrame(pos) + + if data.shape[0]>1: + data=add_block(data) + data=add_room(data) + + #需要提取的特征 + choose_class=['total_price','area','room', + 'district_name','p_date','url','block'] + + dc=data[choose_class] + del pos,data,dirst,dirst1,lf,le,lr + return dc + else: + del pos,data,dirst,dirst1,lf,le,lr + return numpy.array([]) + + def most_frecuncy_rent(dd2): + + ppp=dd2['total_price'] + pvv=list(ppp) + gv=(numpy.array(pvv)//5)*5+2 + mostn=collections.Counter(gv).most_common(1) + if len(mostn)>0: + del ppp,pvv,gv + return mostn[0][0] + else: + del ppp,pvv,gv + return 0 + + def mean_rent_price(ct,rl,lk,timd,timm): + + dd= some_district_information(ct,rl,lk,timd,timm)#这行数字是日期 + if dd.shape[0]>1: + dd['total_price']=dd['total_price']/dd['area'] + cut_rule=dd['total_price'].mean()-1.5*dd['total_price'].std() + aw=dd[dd['total_price']>cut_rule] + a1=aw['total_price'].dropna() + mean=a1.mean() + return mean + + else: + return '没有小区租房数据' + + + def no_source_price(ct,rl,location,rf,p,timd,timm): + #rent_ratio=rf[0] + #rent_most=rf[1] + hg=location.shape[0] + + if hg==0: + return '没有经纬度数据' + else: + #h=time.time() + dd=neighbor(ct,rl,location['lng2'][0],location['lat2'][0],5,p) + #print('h_time=',time.time()-h) + if dd.shape[0]>=8: + + #dd1=neighbor(location['lng2'][0],location['lat2'][0],5) + other=RENT_no_data_mostblock_price(ct,rl,dd,rf,p,timd,timm) + ##print('other= ' ,other) + del dd + return other + else: + dd1=neighbor(ct,rl,location['lng2'][0],location['lat2'][0],15,p) + + other1=RENT_no_data_mostblock_price(ct,rl,dd1,rf,p,timd,timm) + ##print('other1=',other1) + del dd,dd1 + return other1 + + + def no_data_mostblock_price(dd): + + ds=dd[['total_price','area','district_name','url','block']] + ds['total_price']=ds['total_price']/ds['area'] + ds1=ds.fillna(method='ffill') + ds2=ds1.dropna() + + + ppp=ds2['total_price'] + pvv=list(ppp) + gv=(np.array(pvv)//5)*5+2 + mostn=Counter(gv).most_common(1) + ##print('mostn=',mostn) + + return mostn[0][0] + + + + + + + + def RENT_no_data_mostblock_price(ct,rl,dd,rf,p,timd,timm): + lk=p[0] + rent_ratio=rf[0] + most_rent=rf[1] + if dd.shape[0]>1: + + ds=dd[['total_price','area','district_name','url','block']] + ds['total_price']=ds['total_price']/ds['area'] + ds1=ds.fillna(method='ffill') + ds2=ds1.dropna() + + ###print('ds2==',ds2) + + + #ds.to_excel('dfff.xls') + + blrl=rf[1] + ##print('blrl',blrl) + + dd2=ds2[ds2['total_price']>0.5*blrl] + #dd3=dd2[dd2['total_price']<2*blrl/900] + if dd2.shape[0]>5: + + ppp=dd2['total_price'] + ###print('ppp=',ppp) + lgg=k_means(ppp) + ###print('lgg=',lgg) + del ds,ds1,ds2,dd2,ppp + return lgg + else: + del ds,ds1,ds2,dd2 + + if type(rent_ratio)!=str: + dff=disanganf_price(ct,rl,lk,timd,timm) + ##print('dff=',dff,'*'*20) + if dff>100: + return round(dff/rent_ratio,2) + elif type(most_rent)!=str: + return most_rent + else: + return '没有数据' + elif type(most_rent)!=str: + return most_rent + else: + return '没有数据' + + elif type(rent_ratio)!=str: + + dff=disanganf_price(ct,rl,lk,timd,timm) + ##print('dff=',dff,'*'*20) + if dff>100: + return round(dff/rent_ratio,2) + elif type(most_rent)!=str: + return most_rent + else: + return '没有数据' + elif type(most_rent)!=str: + return most_rent + else: + return '没有数据' + + + def k_means(pp): + from sklearn.cluster import KMeans + pv=list(pp.dropna()) + gf=numpy.array([pv]).T + estimator = KMeans(n_clusters=3)#构造聚类器 + estimator.fit(gf)#聚类 + label_pred = estimator.labels_ #获取聚类标签 + aa=collections.Counter(label_pred) + v=pandas.Series(aa) + gg=list(v) + index_max=gg.index(max(gg)) + #yy=np.array([label_pred]).T + #un=np.hstack((gf,yy)) + #uv=pd.DataFrame(un,columns=['price','label']) + centroids = estimator.cluster_centers_ #获取聚类中心 + #inertia = estimator.inertia_ # 获取聚类准则的总和 + center=centroids[index_max][0] + del pv,gf,aa,v,gg,centroids + return (round(center,2)) + + + ''' + def dff(i,j,timd,timm): + reef=rent_partial_sell(i,j,timd,timm) + ##print('$$$$$$$$$$') + if type(reef[0])!=str: + return reef + else: + ##print('&&&&&&&&&&&&&') + return rent_partial_sell(i,None,timd,timm) + ''' + + + + def region_district_list_name(ct,rl,lkk): + #import pandas + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + + query1 = {"status":0,"cat":"district","city":ct,"region":rl,"name":lkk} + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, + "cat":1,"name":1,'estate_type2':1,'alias':1} + + #lct= list( seaweed136.find(query1, fields1)) + + + #lf=pandas.DataFrame(lct) + + lf=fangjia.find_region_district_list_name(query1, fields1) + + #lf=fangjia.seaweed(query1, fields1) + return lf + + + def strip_villa(ct,rl,p,timd,timm): + lw=p[0] + #estype=p[2] + alias=p[1] + + ###print('alias=',alias) + + def filt(x): + + return type(x)==str and len(x)>1 + + bb=list(filter(filt,alias)) + ###print('bb=',bb) + bb.extend([lw]) + #lk1=list(set(lk)) + ###print('本小区名=',lw) + ###print('type(lw)=',type(bb)) + + ###print('小区名列表=',bb) + + cc1= some_district_information(ct,rl,bb,timd,timm) + + cc1= add_room(cc1) + + ##print('cc1本小区数据条数=',cc1.shape[0]) + + if cc1.shape[0]>2: + + del lw,alias,bb + return cc1 + else: + + if '别墅' in lw: + + lkk=lw.strip('别墅') + cc=region_district_list_name(ct,rl,lkk) + ##print('cc=',cc) + if cc.shape[0]>1: + + cc1=list(cc['alias'].dropna()) + cc2=list(filter(filt,alias)) + bb.extend([lkk]) + #clk1=list(set(clk)) + cc2=some_district_information(ct,rl,bb,timd,timm) + if cc2.shape[0]>2: + del lw,alias,bb,cc1,cc + return cc2 + else: + del lw,alias,bb,cc1,cc + return cc1 + else: + return cc1 + else: + return cc1 + + + + + + + def consider_villa(ct,rl,p,timd,timm): + + ###print('villa p=',p) + + estype=p[2] + df=strip_villa(ct,rl,p,timd,timm) + if df.shape[0]>1: + + if estype=='普通住宅': + all_index=set(df.index) + vv=fetch_villa_index(df) + ww=all_index-vv + + d1=df.loc[ww,:] + if d1.shape[0]>3: + return d1 + else: + return df + elif estype=='别墅': + vv=fetch_villa_index(df) + d1=df.loc[vv,:] + if d1.shape[0]>3: + return d1 + else: + return df + else: + return df + else: + return df + + + def fetch_villa_index(dd): + + #dd=dd.dropna() + d1=dd[dd['area']>150] + + + villa_label=["独栋", "联排", "叠加", "双拼", + "花园面积", "泳池", "别墅", "带花园"] + bb=list() + for i in d1.index: + we=d1['name'][i] + for j in villa_label: + if j in we: + bb.append(i) + d21=dd[dd['room']>=6] + d2=d21[d21['area']>=360] + + d3=dd[dd['area']>500] + + villa=set(bb)|set(d2.index)|set(d3.index) + + return villa + + + ###print('cc1p=',p) + + def front_step_time(day): + now = datetime.datetime.now() + front = now - datetime.timedelta(days=day) + d1 = front.strftime('%Y%m%d') + return int(d1) + + def k_means_clus(pp1, clus): + + from sklearn.cluster import KMeans + pv = list(pp1) + if len(set(pv)) > clus: + gf = numpy.array([pv]).T + estimator = KMeans(n_clusters=clus) # 构造聚类器 + estimator.fit(gf) # 聚类 + label_pred = estimator.labels_ # 获取聚类标签 + aa = collections.Counter(label_pred) + v = pandas.Series(aa) + gg = list(v) + index_max = gg.index(max(gg)) + + centroids = estimator.cluster_centers_ # 获取聚类中心 + + ###print(centroids,gg) + # inertia = estimator.inertia_ # 获取聚类准则的总和 + center = centroids[index_max][0] + return (int(center)) + else: + return int(pp1.mean()) + + #e=time.time() + cc1= consider_villa(ct,rl,p,timd,timm) + + cc=add_rent_type(cc1) + #print('e_time=',time.time()-e) + + ###print('cc=',cc['p_date']) + def take_some_district_featture(ct,rl,lk): + + query_feature = {"city": ct, "region": rl,'district_name':lk} + #"district_name": {"$in": dirst_plus}} + + ppp = fangjia.dis_feature(query_feature) + ###print('ox'*10,ppp) + + if 'area' in ppp.columns: + are=ppp['area'].iloc[0] + ##print('most frequent area =',are) + return are + else: + return -1 + + + def rent_filter(ap,ook): + + if ook>0: + rdd=ap[(ap['area']>ook*0.8)&(ap['area']=before] + + ##print('day=',day,tfp) + + + + if tfp.shape[0]>3: + + tfp1=k_means_clus(tfp["total_price"],3) + return tfp1 + elif tfp.shape[0]>0: + ak=list(tfp['total_price']) + + ak1=[] + for j in ak: + ak2=abs((j-nd)/nd) + ak1.append(ak2) + ak3=pd.DataFrame() + ak3['price']=ak + ak3['rat']=ak1 + ak4=ak3[ak3['rat']<0.15] + ak5=ak4.sort_values(by='rat') + ##print('ak5=',ak5) + if ak5.shape[0]>0: + return ak5['price'].iloc[0] + else: + return -1 + + else: + return -1 + + + def map_time_price(aw,nd): + + + price_15= time_filter_price(aw,30,nd) + price_30= time_filter_price(aw,60,nd) + price_60= time_filter_price(aw,120,nd) + + + + if price_15>0: + ##print('first_15',price_15) + return price_15 + elif price_30>0: + ##print('first_30',price_30) + return price_30 + elif price_60>0: + ##print('first_60=',price_60) + return price_60 + else: + return k_means_clus(aw["total_price"],3) + + def t_xigma(mp,nd): + + if mp<=3*nd: + return mp + else: + return nd + if cc.shape[0]>6: + + a_set={'room','total_price','area','rent_type','hall','fitment','p_date'} + b_set=set(cc.columns) + + if a_set & b_set==a_set: + ap=cc[['room','total_price','area','rent_type','hall','fitment','p_date']] + #f=time.time() + ook=take_some_district_featture(ct,rl,p[0]) + #print('f_time=',time.time()-f) + ##print('ook=',ook) + av=rent_filter(ap,ook) + av['total_price']=av['total_price']/av['area'] + location=take_location(ct,rl,p) + if location.shape[0]==1: + g=time.time() + nd=no_source_price(ct,rl,location,region_feature,p,timd,timm) + #print('g_time=',time.time()-g) + if type(nd)!=str: + ##print('nd=',nd) + aw=av[av['total_price']>0.6*nd] + ###print('aw=',aw[['p_date','total_price','area','rent_type']]) + if aw.shape[0]>5: + mp=map_time_price(aw,nd) + + md=t_xigma(mp,nd) + ##print('EEEEEEEEEEEE') + return round(md,2) + else: + return nd + else: + return nd + + else: + return '小区经纬度有问题' + else: + location=take_location(ct,rl,p) + + if location.shape[0]==1: + nd=no_source_price(ct,rl,location,region_feature,p,timd,timm) + return nd + else: + + location=take_location(ct,rl,p) + + if location.shape[0]==1: + nd=no_source_price(ct,rl,location,region_feature,p,timd,timm) + + return nd + + else: + return '小区经纬度有问题' +################################################################################################################ +################################################################################################################ +################################################################################################################ +################################################################################################################ + + + +def save_to_mongo(ct,rl,lk,timd,timm): + + + + lg_db=fangjia.get_collection("fangjia_stat",config.get_sqm_rentprice_db(), config.get_mongodb_host(),27017) + def region_district_list(ct,rl,lk): + #3333333333333333333333333333333333333333333333 + #import pandas + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + + query1 = {"status":0,"cat":"district","city":ct,"region":rl,'name':lk} + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, + "cat":1,"name":1,'estate_type2':1,'alias':1} + + return fangjia.find_seaweed(query1, fields1) + ''' + lct= list(seaweed136.find(query1, fields1)) + + + lf=pandas.DataFrame(lct) + return lf + ''' + + + def ghhh(kkg): #555555555555555555555555555 + kkg=kkg.fillna('无') + a=list() + for i in kkg.index: + b=list(kkg.loc[[i],:].values.flatten()) + a.append(b) + return a + def add_alias(ppd): #4444444444444444444444444444444444444444444444444444444 + if 'alias' not in ppd.columns: + ppd['alias']='无' + return ppd + + + + def f_del(qq,g): + + if g!= None: + + return qq + else: + del qq['region'] + return qq + + + def add_eatype(ppd): + if 'estate_type2' not in ppd.columns: + ppd['estate_type2']='普通住宅' + + return ppd + + def before_month_lastday(ti,num): + from dateutil.parser import parse + today=dateutil.parser.parse(str(ti)) + + #first = datetime.date(day=1, month=today.month, year=today.year) + + lastMonth = today - datetime.timedelta(days=num) + + def plus(k): + if k<10: + return '0'+str(k) + else: + return str(k) + y=lastMonth.year + m=lastMonth.month + #d=lastMonth.day + day=calendar.monthrange(y,m)[1] + + cc=str(y)+plus(m)+plus(day) + bb=parse(cc) + pacific = pytz.timezone('Asia/Shanghai') + return pacific.localize(bb) + + + + def timestamp_to_int(dd): + + dd['week1']=1 + + for i in dd.index: + + f1=dd['weekend'][i].strftime("%Y-%m-%d") + f2 = re.sub("\D", "", f1) + f3=int(f2) + dd['week1'][i]=f3 + ###print(dd) + return dd + + + + + + + def dis_make_up_price(ct,rl,lk,ti): + + + #lgg=lg_db + #save_lg=client136.fangjia_stat.test_lg5 + + #cc=before_month_lastday(ti,65) + #cc_1=before_month_lastday(ti,0) + cc2=before_month_lastday(ti,32) + ##print('cc=',cc) + ##print('cc_1=',cc_1) + ##print(cc) + + #print('cc2=',cc2) + + #stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) + stat=lg_db + query = {"city":ct,"region":rl,"district_name":lk, + "cat":"district"} + + pos1=list(stat.find(query).sort([("weekend",-1)]).limit(3)) + + pos=pandas.DataFrame(pos1) + + if pos.shape[0]>=1: + ##print('pos=',pos) + #pos=pos[(pos['weekend']>cc)&(pos['weekend']0: + #print('pos=',pos) + pos=pos.sort_values(by='weekend',ascending=False) + pp1=pos['avg_price'].iloc[0] + ##print('pp1=',pp1) + if type(pp1)!=None and type(pp1)!=str: + return pp1 + else: + return None + else: + return None + else: + return None + + + + def return_operate(ppu,mm): + + if ppu==None: + return 0 + else: + return round(10000*(mm-ppu)/ppu,2) + ''' + def month_lastday(ti): + from dateutil.parser import parse + today=dateutil.parser.parse(str(ti)) + + #first = datetime.date(day=1, month=today.month, year=today.year) + + lastMonth = today# - datetime.timedelta(days=35) + + def plus(k): + if k<10: + return '0'+str(k) + else: + return str(k) + y=lastMonth.year + m=lastMonth.month + #d=lastMonth.day + day=calendar.monthrange(y,m)[1] + + cc=str(y)+plus(m)+plus(day) + bb=parse(cc) + pacific = pytz.timezone('Asia/Shanghai') + return pacific.localize(bb) + ''' + def region_feature(ct,rl,weekend): + #client182 = pymongo.MongoClient('192.168.10.182',27017) + #db182 = client182.fangjia_stat.rent_1221 + query1 = {"city":ct,"region":rl,'weekend':before_month_lastday(timm,0)} + #a1=list(db182.find(query1)) + #a2=pd.DataFrame(a1) + a2=fangjia.rent_region_feature(query1, None) + if a2.shape[0]>0: + ratio=a2['ratio'].iloc[0] + price=a2['avg_price'].iloc[0] + return (ratio,price) + else: + return () + def add_region_fea(ct,rl,weekend): + #timm=20180131 + + #timd=20170303 + d1=region_feature(ct,rl,None) + #print('d1=',d1) + if len(d1)>0: + return d1 + else: + feature.region_feature.save_region_feature(ct,rl,timd,timm) + + return region_feature(ct,rl,None) + + + #a=time.time() + lf=region_district_list(ct,rl,lk) #3333333333333333333333333333333333333333333333 + #disname=list(lf['name']) + + #print('a_time=',time.time()-a) + + ###print('len(lf)',len(lf)) + #b=time.time() + lf=add_alias(lf) #4444444444444444444444444444444444444444444444444444444 + lf=add_eatype(lf) + lf['estate_type2']=lf['estate_type2'].fillna('普通住宅') + + p=ghhh(lf[['name','alias','estate_type2']])[0] #555555555555555555555555555 + + + + region_feature=add_region_fea(ct,rl,None) + + #print('region_feature=',region_feature) + #print('b_time=',time.time()-b) + + + + def get_weekend(tt): + + from dateutil.parser import parse + bb=parse(tt) + pacific = pytz.timezone('Asia/Shanghai') + return pacific.localize(bb) + #timme=dateutil.parser.parse(str(timm)) + timme=before_month_lastday(timm,0) + + ##print('timme=',timme) + ###print(len(disname),len(disname)) + ###print('region_feature',region_feature) + ##print('pos=',pos) + #c=time.time() + mm=common_rent_price(ct,rl,p,timd,timm,region_feature,None) + #print('c_time=',time.time()-c) + ##print(ct,rl,p[0],mm) + + #d=time.time() + mmup=dis_make_up_price(ct,rl,p[0],timm) + #print('mmup=',mmup) + #print('mm=',mm) + make_up=return_operate(mmup,mm) + #print('d_time=',time.time()-d) + ###print('p=',p) + date=datetime.datetime.now() + #weekend=20161231 + print("######################## ","ct=",ct,"rl=",rl,"lk=",p[0],"mm=",mm,make_up) + """ + client1 = pymongo.MongoClient('192.168.0.136',27017) + db1 = client1.fangjia_stat + stat = db1.test_lg55 + + + #stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) + save_lg.save({"city":ct,"region":rl,"district_name":p[0],"make_up":make_up, + "avg_price":mm, "c_date":date,"weekend":timme,"cat":'district'}) + save_lg=client136.fangjia_stat.rent_1221 + """ + if type(mm)!=str: + + mm=float(mm) + #get_sqm_rentprice_db() + save_lg=lg_db + save_lg.replace_one( + + {"city":ct,"region":rl,"district_name":p[0],"weekend":timme}, + + {"city":ct,"region":rl,"district_name":p[0],"make_up":make_up, + "avg_price":mm, "c_date":date,"weekend":timme,"cat":'district'},True + ) + + + + + +''' +ct='嘉兴' +rl='海盐' +lk='保利壹品' +timm=20180413 + +timd=20170503 +a=time.time() +save_to_mongo(ct,rl,lk,timd,timm) + +##print(time.time()-a) +''' + + + + + +''' +client105 = pymongo.MongoClient('192.168.0.105',27777) #'192.168.0.65',27777 +db105 = client105.fangjia +seawater105 = db105.seawater + +client136 = pymongo.MongoClient('192.168.0.136',27017) +db136 = client136.fangjia +dsf136 = db136.average_statistics +seaweed136 = db136.seaweed +some_district_information + + +def city_list(): #11111111111111111111111111111111111111111111111111111111111111111 + #client = pymongo.MongoClient('192.168.0.136',27017) #'192.168.0.65',27777 + #db = client.fangjia + city = db136.server_area + #city= fangjia.get_collection("fangjia","server_area", '192.168.0.136',27017) + #seawater.find_one() + + + pos=city.find({"display":{"$ne":"全国"}}, + {"display":1,"city_grade":1,"_id":1}).sort([("city_grade",1),("_id",1)]) + + + + data=pandas.DataFrame(list(pos)) + del pos + return data['display'] + + +def region_list(city): #22222222222222222222222222222222222222222222222 + + #client = pymongo.MongoClient('192.168.0.136',27017) + #db = client.fangjia + region = db136.region_block + #region = fangjia.get_collection("fangjia","region_block", '192.168.0.136',27017) + #seawater.find_one() + query = {"city":city,"category":"region"} + fields1 = {"name":1} + pos = list(region.find(query,fields1)) + + data=pandas.DataFrame(pos) + del pos + return data['name'] + + + +ctii=city_list()#1111111111111111111111111111111111111111111111111111111111 +cti1=list(ctii) + + + + +timm=20180131 + +timd=20170303 + +#cti1=a22 + + +kk=1 +cti=cti1[kk:kk+3] + + + +for ct in cti: + region=region_list(ct) + + + ##print('region=',region) + for rl in region: + save_to_mongo(ct,rl,timd,timm) + + +''' + + + + diff --git a/PriceSystem/rent_test.py b/PriceSystem/rent_test.py new file mode 100644 index 0000000..fe2a926 --- /dev/null +++ b/PriceSystem/rent_test.py @@ -0,0 +1,84 @@ +import datetime +import fangjia +import pytz +#fangjia.send_dingding_msg("测试") + +import time + +import rent_price.district_map0413 as rentPrice + +def get_weekend(tt): + + from dateutil.parser import parse + bb=parse(tt) + pacific = pytz.timezone('Asia/Shanghai') + return pacific.localize(bb) + +weekend = get_weekend('20180415') + +ct='上海' +rl='黄浦' +lk='伯爵居'#上海闵行闵浦新苑一村普陀曹杨一村上海 +#黄浦伯爵居静安泰安里长宁新泾北苑 +timm=20170430 + +timd=20160603 +a=time.time() +rentPrice.save_to_mongo(ct,rl,lk,timd,timm) + +print(time.time()-a) + + + + + + + + + + + + +''' +s = time.time() + +ct='上海' +rl='浦东' +lk='滨江一品苑' + + + +#for i in range(0, 100):深圳福田世纪豪庭 + +rentPrice.out_put_price(ct, rl, lk, weekend) +e = time.time() + +print(e - s) +''' + +''' + + +### +import pandas as pd + +a=pd.read_excel('小区调查.xlsx') + +for i in a.index: + ct=a['城市'][i] + rl=a['区域'][i] + lk=a['小区名'][i] + sellPrice.out_put_price(ct, rl, lk, weekend) + +''' + +''' + +@fangjia.cache(1000) +def sum1(a, b): + print(111) + return a + b + + +print(sum1(1,2)) +'''