新文件: feature/__init__.py

新文件:   feature/region_feature.py
This commit is contained in:
罗干
2018-04-24 17:23:22 +08:00
parent ad8a960049
commit 111da4aade
12 changed files with 865 additions and 189 deletions
+2 -2
View File
@@ -10,8 +10,8 @@ import pytz
import os
QUEUE = "calc_cmd"
redis_pool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_db())
#redis_pool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=3)
#redis_pool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_db())
redis_pool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=2)
logger = fangjia.get_logger("CalcCmdReceiver")
+33
View File
@@ -56,3 +56,36 @@ def get_seaweed_host():
def get_seaweed_port():
return 27017
##########################################3
#the code below is about square meter rent price
def get_65_host():
return 'seawater65.mongo.com'
def get_sqm_rentprice_db():
return "rent_region_feature1"
def get_lg_mongo():
return 'luogan.mongo.com'
def get_rent_redis_db():
return 5
+115 -21
View File
@@ -25,7 +25,7 @@ def get_logger(name):
file_handler = logging.FileHandler("/home/goojia/logs/" + name + ".log")
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel("ERROR")
logger.setLevel("INFO")
return logger
@@ -39,7 +39,7 @@ def get_performance_logger(name):
file_handler = logging.FileHandler("/home/goojia/logs/" + name + ".performance.log")
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel("ERROR")
logger.setLevel("INFO")
return logger
now = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
@@ -48,24 +48,20 @@ def get_performance_logger(name):
break
now = now + datetime.timedelta(days=1)
performance_logger = get_performance_logger("performance")
def timer(func):
def _timer(*args, **kw):
begin = time.time()
begin = time.clock()
rs = func(*args, **kw)
end = time.time()
end = time.clock()
elapsed = end - begin
if elapsed > config.get_timeout():
arg_list = list()
for arg in args:
arg_list.append(str(arg))
if elapsed > config.get_timeout() * 10:
performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list))
else:
performance_logger.error("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list))
performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list))
return rs
return _timer
@@ -125,10 +121,6 @@ def get_region(city):
def get_weekend():
"""
返回本周周末
:return:
"""
now = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
for day in range(1, 9):
if calendar.weekday(now.year, now.month, now.day) == 6:
@@ -155,6 +147,7 @@ def send_ding_msg(msg):
requests.request("POST", url, data=json.dumps(text_body, ensure_ascii=False).encode('utf8'), headers=headers)
@timer
def findHouse(query, fields=None):
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
@@ -256,7 +249,7 @@ def find_disanganf_price(query, fields=None):
# db1 = client1.fangjia
# dsf = db1.average_statistics
# s = time.time()
seawater105 = get_collection("fangjia", "average_statistics", config.get_mongodb_host(), 27017)
seawater105 = get_collection("fangjia", "average_statistics", "192.168.0.136", 27017)
for err_count in range(0, 10):
try:
@@ -280,15 +273,13 @@ def find_disanganf_price(query, fields=None):
raise Exception("yang ben cha xun shi bai1")
@timer
def dis_feature(query, fields=None):
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# dsf = db1.average_statistics
# s = time.time()
seawater105 = get_collection("fangjia_stat", "district_feature", config.get_mongodb_host(), config.get_mongodb_port())
#seawater105 = get_collection("luogan", "district_feature", '192.168.0.136',27017)
seawater105 = get_collection("fangjia_stat", "district_feature", "192.168.0.136", 27017)
for err_count in range(0, 10):
try:
@@ -312,7 +303,7 @@ def dis_feature(query, fields=None):
raise Exception("yang ben cha xun shi bai1")
@timer
def find_seaweed(query, fields=None):
# dirlt=p[0]
@@ -320,11 +311,12 @@ def find_seaweed(query, fields=None):
# db1 = client1.fangjia
# seaweed1 = db1.seaweed
# s = time.time()
seawater105 = get_collection("fangjia", "seaweed", config.get_seaweed_host(), config.get_seaweed_port())
seawater105 = get_collection("fangjia", "seaweed", "192.168.0.136", 27017)
for err_count in range(0, 10):
try:
# print('flag5')
#lt = seawater105.count(query)
pos = list(seawater105.find(query))
data = pandas.DataFrame(pos)
@@ -333,6 +325,9 @@ def find_seaweed(query, fields=None):
return data
except:
# err_count = err_count + 1
#lt = seawater105.count(query)
pos = list(seawater105.find(query))
data = pandas.DataFrame(pos)
del pos
@@ -346,7 +341,7 @@ def find_region_district_list_name(query, fields=None):
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# seaweed1 = db1.seaweed
seawater105 = get_collection("fangjia", "seaweed", config.get_seaweed_host(), config.get_seaweed_port())
seawater105 = get_collection("fangjia", "seaweed", "192.168.0.136", 27017)
for err_count in range(0, 10):
try:
@@ -366,3 +361,102 @@ def find_region_district_list_name(query, fields=None):
data = pandas.DataFrame(pos)
del pos
return data
##########################################################################################
# the code below is about computing rent price /m**2
def rent_sample(query, fields):
# s = time.time()
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
for err_count in range(0, 10):
try:
# lt = seawater105.count(query)
lt = 1000
# print('flag2')
pos=list(seawater105.find(query,fields).limit(lt))
data=pandas.DataFrame(pos)
if data.shape[0]>10:
#print('数据条数=',data.shape[0])
return data[['district_name','total_price','area','p_date']]
else:
return pandas.DataFrame()
except:
lt = 1000
# print('flag2')
pos=list(seawater105.find(query,fields).limit(lt))
data=pandas.DataFrame(pos)
if data.shape[0]>10:
#print('数据条数=',data.shape[0])
return data[['district_name','total_price','area','p_date']]
else:
return pandas.DataFrame()
raise Exception("样本查询失败!")
def rent_disanganf_series(query, fields):
# s = time.time()
seawater105 = get_collection("fangjia", "district_stat", '192.168.0.136', 27017)
for err_count in range(0, 10):
try:
pos=list(seawater105.find(query,fields))
#print(pos)
data1=pandas.DataFrame(pos)
one_date=list(set(data1['weekend']))[0]
data=data1[data1['weekend']==one_date]
d0=list(data['calc_value'])
d1=list(data['district_name'])
d2=pandas.Series(d0,index=d1)
return d2
except:
# print('flag2')
pos=list(seawater105.find(query,fields))
data1=pandas.DataFrame(pos)
one_date=list(set(data1['weekend']))[0]
data=data1[data1['weekend']==one_date]
d0=list(data['calc_value'])
d1=list(data['district_name'])
d2=pandas.Series(d0,index=d1)
return d2
raise Exception("样本查询失败!")
def rent_region_feature(query, fields):
rf = get_collection("fangjia_stat", config.get_sqm_rentprice_db(), '192.168.0.136', 27017)
#client182 = pymongo.MongoClient('192.168.10.182',27017)
#db182 = client182.fangjia_stat.rent_1221
for err_count in range(0, 10):
try:
pos=list(rf.find(query,fields))
pos1=pandas.DataFrame(pos)
return pos1
except:
pos=list(rf.find(query,fields))
pos1=pandas.DataFrame(pos)
return pos1
raise Exception("样本查询失败!")
"""
query1 = {"city":ct,"region":rl,'weekend':month_lastday(timm)}
a1=list(db182.find(query1))
a2=pd.DataFrame(a1)
if a2.shape[0]>0:
ratio=a2['ratio'].iloc[0]
price=a2['avg_price'].iloc[0]
return (ratio,price)
else:
return ()
"""
+1
View File
@@ -0,0 +1 @@
from feature import region_feature
+410
View File
@@ -0,0 +1,410 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Mar 27 13:23:56 2018
@author: luogan
"""
import os
##print( os.getcwd())
#from collections import Counter
import pymongo
#from pymongo import MongoClient
import numpy as np
import random
import collections
import pymongo
import sklearn.cluster
import pandas
import numpy
import sys, time
#import pp
import fangjia
import datetime
import re
import pandas
import dateutil
import time
import fangjia
import config
import datetime
import pytz
import pandas as pd
import calendar
def ghhh(kkg): #555555555555555555555555555
kkg=kkg.fillna('无')
a=list()
for i in kkg.index:
b=list(kkg.loc[[i],:].values.flatten())
a.append(b)
return a
def f_del(qq,g):
if g!= None:
return qq
else:
del qq['region']
return qq
def sample_pool(ct,rl,timd,timm):
query = {"city":ct,"cat":"rent","region":rl,
"p_date":{"$gt":timd,"$lt":timm}}
#query=f_del(query,rl)
fields = {"total_price":1,"area":1, "city":1,"region":1,'p_date':1,
"district_name":1,"rent_type":1,"fitment":1,"block":1}
#lt= seawater.count(query)
###print(lt)
#pos = list()
#for s in seawater.find(query,fields1).skip(10000).limit(lt):
return fangjia.rent_sample(query, fields)
def k_means(pp):
from sklearn.cluster import KMeans
pv=list(pp.dropna())
gf=numpy.array([pv]).T
estimator = KMeans(n_clusters=3)#构造聚类器
estimator.fit(gf)#聚类
label_pred = estimator.labels_ #获取聚类标签
aa=collections.Counter(label_pred)
v=pandas.Series(aa)
gg=list(v)
index_max=gg.index(max(gg))
#yy=np.array([label_pred]).T
#un=np.hstack((gf,yy))
#uv=pd.DataFrame(un,columns=['price','label'])
centroids = estimator.cluster_centers_ #获取聚类中心
#inertia = estimator.inertia_ # 获取聚类准则的总和
center=centroids[index_max][0]
del pv,gf,aa,v,gg,centroids
return (round(center,2))
def get_weekend():
#from dateutil.parser import parse
#bb=parse(tt)
now_time=datetime.datetime.now()
before_time = now_time-datetime.timedelta(days=14)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize( before_time )
def disanganf_price(ct,rl,lk,timd,timm):
fields={'calc_value':1,'weekend':1,'district_name':1}
query = {"city":ct,"region":rl,
"district_name":{'$in':lk},'weekend':{'$gt':get_weekend()}}
#pos = list()
#数据转化为数组,数组的元素为字典
return fangjia.rent_disanganf_series(query, fields)
def rent_partial_sell(ct,rl,timd,timm):
rele0=sample_pool(ct,rl,timd,timm)
##print(rele0)
rele1=rele0[['district_name','p_date','area','total_price']]
rele1['total_price']=rele1['total_price']/rele1['area']
#print('up_rele.shape[0]=',rele1.shape[0])
rele=rele0.drop_duplicates(['district_name'])
p1=rele['district_name'].dropna()
p2=list(set(p1))
###print('*****************************************************************')
dd=disanganf_price(ct,rl,p2,timd,timm)
###print('rele.shape[0]=',rele.shape[0])
if rele.shape[0]>10:
rele['total_price']=rele['total_price']/rele['area']
#rent_most=k_means(rele['total_price'])
rele['sell_rent_ratio']=1
rv=rele[['total_price','district_name','area','sell_rent_ratio']].dropna()
rv1=rv.values
ro=[]
sell_rent_ratio=[]
for k in range(len(rv1)):
namee=rv1[k][1]
pee1= rele1[rele1['district_name']==namee]
pee2=pee1.sort_values(by='p_date',ascending=False)
price=pee2['total_price'].iloc[0]
ro.append(price)
#price=rv1[k][0]
if namee in dd.index:
sell=dd[namee]
ppoo=sell/price
sell_rent_ratio.append(ppoo)
#rv1[k][3]=sell/price
###print('rele0=',rele)
rent_most=k_means(pandas.Series(ro))
#rele=pd.DataFrame(rv1,columns=rv.columns)
roseries=pandas.Series(sell_rent_ratio)
roseries=roseries[roseries>5]
#rele=rele[rele['sell_rent_ratio']>5]
##print('sell_rent_ratio=',list(rele['sell_rent_ratio'])[:5])
##print(rele)
##print('#######################')
##print(rele.shape[0])
if len(roseries)>10:
#rent_most=k_means(rele['total_price'])
##print('rele.shape[0]=',rele.shape[0])
#print('yyyyyyyyyyyyyyyyyyyyyy=',len(roseries))
rent_sell=k_means(roseries.dropna())
del rele
return rent_sell,rent_most
else:
del rele
return '没有区域租售比',rent_most
else:
del rele
return '没有区域租售比','没有区域租金数据'
def dff(i,j,timd,timm): #666666666666666666666666666666666666666666666666666666
reef=rent_partial_sell(i,j,timd,timm)
##print('$$$$$$$$$$')
if type(reef[0])!=str:
return reef
else:
##print('&&&&&&&&&&&&&')
return rent_partial_sell(i,None,timd,timm)
def add_eatype(ppd):
if 'estate_type2' not in ppd.columns:
ppd['estate_type2']='普通住宅'
return ppd
def timestamp_to_int(dd):
dd['week1']=1
for i in dd.index:
f1=dd['weekend'][i].strftime("%Y-%m-%d")
f2 = re.sub("\D", "", f1)
f3=int(f2)
dd['week1'][i]=f3
###print(dd)
return dd
'''
def before_month_lastday(ti):
today=dateutil.parser.parse(str(ti))
#first = datetime.date(day=1, month=today.month, year=today.year)
lastMonth = today - datetime.timedelta(days=35)
def plus(k):
if k<10:
return '0'+str(k)
else:
return str(k)
y=lastMonth.year
m=lastMonth.month
#d=lastMonth.day
day=calendar.monthrange(y,m)[1]
cc=str(y)+plus(m)+plus(day)
return int(cc)
'''
def before_month_lastday(ti,num):
from dateutil.parser import parse
today=dateutil.parser.parse(str(ti))
#first = datetime.date(day=1, month=today.month, year=today.year)
lastMonth = today - datetime.timedelta(days=num)
def plus(k):
if k<10:
return '0'+str(k)
else:
return str(k)
y=lastMonth.year
m=lastMonth.month
#d=lastMonth.day
day=calendar.monthrange(y,m)[1]
cc=str(y)+plus(m)+plus(day)
bb=parse(cc)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(bb)
def region_make_up_price(ct,rl,lk,ti):
lgg=fangjia.get_collection("fangjia_stat",config.get_sqm_rentprice_db(), config.get_mongodb_host(),27017)
#save_lg=client136.fangjia_stat.test_lg5
cc=before_month_lastday(ti,32)
#print('ti=',ti)
#print('cc=',cc)
#stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017)
stat=lgg
query = {"city":ct,"region":rl,"cat":"region",'weekend':cc,'district_name':None}
pos1=list(stat.find(query).sort([("weekend",-1)]).limit(3))
pos=pandas.DataFrame(pos1)
#pos=timestamp_to_int(pos)
#pos=pos[pos['week1']>cc]
##print('5'*10)
##print('uuuuuuuuuuuuuuuu=',pos)
if pos.shape[0]>=1:
pos=pos.sort_values(by='weekend',ascending=False)
pp1=pos['avg_price'].iloc[0]
#print('pp1=',pp1)
if type(pp1)!=None and type(pp1)!=str:
return pp1
else:
return None
else:
return None
def return_operate(ppu,mm):
if ppu==None:
return 0
else:
return round(10000*(mm-ppu)/ppu,2)
'''
def before_month_lastday(ti):
today=dateutil.parser.parse(str(ti))
#first = datetime.date(day=1, month=today.month, year=today.year)
lastMonth = today - datetime.timedelta(days=31)
cc=str(lastMonth.year)+str(lastMonth.month)+str(lastMonth.day)
return int(cc)
'''
#dd1=sample_pool(ct,rl,timd,timm)
def save_region_feature(ct,rl,timd,timm):
a=time.time()
region_feature=dff(ct,rl,timd,timm)
#print('region_feature=',region_feature)
mmup=region_make_up_price(ct,rl,None,timm)
#print('mmup_region=',mmup)
#print('this month region price',region_feature[1])
make_up=return_operate(mmup,region_feature[1])
date=datetime.datetime.now()
def get_weekend(tt):
from dateutil.parser import parse
bb=parse(tt)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(bb)
def month_lastday(ti):
from dateutil.parser import parse
today=dateutil.parser.parse(str(ti))
#first = datetime.date(day=1, month=today.month, year=today.year)
lastMonth = today# - datetime.timedelta(days=35)
def plus(k):
if k<10:
return '0'+str(k)
else:
return str(k)
y=lastMonth.year
m=lastMonth.month
#d=lastMonth.day
day=calendar.monthrange(y,m)[1]
cc=str(y)+plus(m)+plus(day)
bb=parse(cc)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(bb)
timme=month_lastday(timm)
##print('region_timme=',timme)
#stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017)
##print("######################## ","ct=",ct,"rl=",rl,"makeup=",make_up)
#save_lg = fangjia.get_collection("fangjia_stat", "rent_1221", config.get_district_stat_host(), config.get_district_stat_port())
avg=region_feature[1]
if type(avg)!=str:
avg=float(avg)
save_lg = fangjia.get_collection("fangjia_stat",config.get_sqm_rentprice_db(), config.get_mongodb_host(),27017)
save_lg.replace_one(
{"city":ct,"region":rl,"weekend":timme},
{"city":ct,"region":rl,"district_name":None,"make_up":make_up,'ratio':region_feature[0],
"avg_price":region_feature[1], "c_date":date,"weekend":timme,"cat":'region'},True
)
##print(time.time()-a)
###print(len(disname),len(disname))
Binary file not shown.
+98 -40
View File
@@ -32,11 +32,11 @@ estype='普通住宅'
#上海闵行上海康城上海青浦南山雨果
'''
client = MongoClient('luogan.mongo.com',27017)
client = MongoClient('192.168.10.182',27017)
#dt=client.fangjia.district_stat_test
dt=client.fangjia.district_stat_shanghai_d
#dt=client.fangjia.district_stat
dt=client.fangjia.district_stat_all_0416
def price_list(ct,rl,lk):
@@ -52,7 +52,7 @@ def price_list(ct,rl,lk):
dl1=list(dl)
##print(len(dl1))
df=pandas.DataFrame(dl1)
#print('df=',df[['weekend','avg_price','calc_value']])
if df.shape[0]>0:
def add_block(df):
@@ -118,37 +118,47 @@ lk='上海经济建设小区'
estype='普通住宅'
'''
def compare(dd1,name,credit,calc_value):
y=list(dd1[name].dropna())
##print('y=',y)
kk=len(y)
kk1=kk
y1=y[:kk1]
x1=list(range(kk1))
##print('y1=',y1)
fo=line_regre(x1,y1,1)
##print('fo=',fo)
##print('y[-1]',y[-1])
#plot1(list(range(kk)) ,y)
ratio=calc_value/fo
if credit>=0.8 and(ratio>0.8 and ratio<1.4) :
##print('calc_value1',calc_value)
return calc_value
elif ratio>0.9 and ratio<1.1:
##print('ratio',ratio)
##print('calc_value2',calc_value)
return calc_value
#print('y=',y)
kk=len(y)
if kk>2:
#kk1=kk-1
kk1=kk
y1=y[:kk1]
x1=list(range(kk1))
#print('y1=',y1)
#plot1(list(range(kk)) ,y)
fo=line_regre(x1,y1,1)
ratio=calc_value/fo
#print('fo=',fo,name)
if credit>=0.75 and(ratio>0.8 and ratio<1.4) :
#print('cccccc')
return calc_value,1
elif ratio>0.85 and ratio<1.2:
##print('ratio',ratio)
#print('ddddddddddddd')
return calc_value,1
else:
#print('eeeeeeeeeeeeee')
return fo,0
else:
##print('fo',y[-1])
return fo
return calc_value,0
def avg_calc(dd1,credit,calc_value):
m=compare(dd1,'calc_value',credit,calc_value)
n=compare(dd1,'avg_price',credit,calc_value)
#print('m=',m)
#print('n=',n)
if m[1]==1 and n[1]==1:
return m[0]
else :
return n[0]
def out_price(ct,rl,lk,calc):
dd=price_list(ct,rl,lk)
@@ -176,12 +186,12 @@ def out_price(ct,rl,lk,calc):
##print('####################')
return compare(dd1,'calc_value',credit,calc_value),near_id
return compare(dd1,'calc_value',credit,calc_value)[0],near_id
else:
return compare(dd,'avg_price',credit,calc_value),near_id
return avg_calc(dd1,credit,calc_value),near_id
else:
return compare(dd,'avg_price',credit,calc_value),near_id
return avg_calc(dd,credit,calc_value),near_id
else:
return []
@@ -189,12 +199,17 @@ def out_price(ct,rl,lk,calc):
#pp=out_price(ct,rl,lk)
def update(ct,rl,lk,calc):
dd= out_price(ct,rl,lk,calc)
return dd[0]
'''
if len(dd)>0:
return dd[0]
else:
return -1
'''
def update(ct,rl,lk,calc):
dd= out_price(ct,rl,lk,calc)
#return dd[0]
print(dd)
if len(dd)>0:
data={}
@@ -204,7 +219,8 @@ def update(ct,rl,lk,calc):
if len(dd)>0:
dt.update_one({"_id": dd[1]},
{"$set": data})
'''
'''
#洛阳西工人名西路8号院
@@ -224,3 +240,45 @@ estype='普通住宅'
calc=38000
update(ct,rl,lk,calc)
'''
'''
calc=38000
def get_district():
seaweeds = []
keys = {"city": 1, "region": 1, "name": 1, "_id": 0}
for dis in fangjia.get_collection("fangjia", "seaweed").find({ "cat": "district", "status": 0,'city':'上海'}, keys):
#for dis in fangjia.get_collection("fangjia", "seaweed").find({ "city": "上海","cat": "district", "status": 0}, keys):
seaweeds.append(dis)
return seaweeds
dd=get_district()
for k in dd:
ct=k['city']
rl=k['region']
lk=k['name']
update(ct,rl,lk,calc)
#update(ct,rl,lk)
'''
+4 -4
View File
@@ -21,7 +21,7 @@ class CJsonEncoder(json.JSONEncoder):
QUEUE = "calc_cmd"
#redisPool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_db())
redisPool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=3)
redisPool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=2)
client = redis.Redis(connection_pool=redisPool)
logger = fangjia.get_logger("CalcCmdSender")
@@ -42,13 +42,13 @@ def get_weekend(tt):
def get_district():
seaweeds = []
keys = {"city": 1, "region": 1, "name": 1, "_id": 0}
for dis in fangjia.get_collection("fangjia", "seaweed").find({ "cat": "district", "status": 0,'city':'常州'}, keys):
#for dis in fangjia.get_collection("fangjia", "seaweed").find({ "city": "上海","cat": "district", "status": 0}, keys):
for dis in fangjia.get_collection("fangjia", "seaweed").find({ "cat": "district", "status": 0,'city':'上海'}, keys):
#for dis in fangjia.get_collection("fangjia", "seaweed").find({ "cat": "district", "status": 0}, keys):
seaweeds.append(dis)
return seaweeds
weekend = get_weekend('20180408')
weekend = get_weekend('20180422')
#send_cmd({"city": "西安", "region": "雁塔", "name": "鸿景花园", "weekend": get_weekend('20180325')})
if __name__ == "__main__":
for seaweed in get_district():
+177 -113
View File
@@ -29,7 +29,7 @@ import datetime
import config
from fangjia import timer
import fix
import time
# import yesterday
@@ -549,12 +549,12 @@ def commonprice(ct, rl, p):
#return '纬度数据有误,有多个经纬度', 0
else:
# t1=time.time()
t1=time.time()
#####print("location['lng2'][0]",location['lng2'].iloc[0])
dd = neighbor(location['lng2'].iloc[0], location['lat2'].iloc[0], 5, ct, rl, p)
# ####print(time.time()-t1,'t1'*10)
print(time.time()-t1,'t1'*10)
# print('dd=',dd,'dd'*20)
if dd.shape[0] >= factor:
# ####print('$'*50)
@@ -593,9 +593,9 @@ def commonprice(ct, rl, p):
else:
dd6 = neighbor_km(location['lng2'].iloc[0], location['lat2'].iloc[0], 10, ct, rl, p)
# ####print('dd2=',dd2,'@'*60)
if dd6.shape[0] >= 20:
dd6 = neighbor_km(location['lng2'].iloc[0], location['lat2'].iloc[0], 20, ct, rl, p)
#print('dd2=',dd2,'@'*60)
if dd6.shape[0] >= 3:
# svr_time=time.time()
other1 = svr(dd6, lk)
@@ -1006,58 +1006,8 @@ def commonprice(ct, rl, p):
dg22 = middle_operate(junjia_lzf, junjia_anst)
return junjia_max(mmax,dg22)
lk = p[0]
location = take_location(ct, rl, p)
#####print('location',location)
# ttp=time.time()
neighb = no_source_price(ct, rl, location, p)
# ####print(time.time()-ttp,'m'*20)
junjia = dsfjujia(ct, rl, lk)
print('junjia=',junjia)
print('neighb=',neighb)
'''
dglg=dsplg(junjia,neighb,2)
dggg=dsplg(junjia,neighb,1)
'''
def drop_ganji(junjia):
a = {'ganji_esf_price', 'anjuke_esf_price'}
# ####print('a=',a)
b = set(list(junjia.index))
# ####print('b=',b)
c = a & b
u=list(b-a)
if len(c) == 2 and len(junjia)>2:
gj=junjia['ganji_esf_price']
ajk=junjia['anjuke_esf_price']
ratio=abs((gj-ajk)/min(gj,ajk))
other=junjia[u[0]]
ott=(other-max(gj,ajk))/min(gj,ajk)
if ratio<0.1 and ott>0.3:
return junjia.drop('ganji_esf_price')
else:
return junjia
else:
return junjia
judp=drop_ganji(junjia)
dglg = dglg_out1(judp, neighb[0])
#print('dglg=',dglg)
# dggg=dglg_out1(junjia,neighb[0])
# ###print('dglg=',dglg)
# ####print('dggg=',dggg)
#tt=time.time()
data = consider_villa(ct, rl, p)
#data.to_excel('data.xls')
def add_weight(s1):
choose_columns=['url','area', 'district_name', 'name', 'p_date', 'region',
@@ -1092,6 +1042,60 @@ def commonprice(ct, rl, p):
else:
return s1
def drop_ganji(junjia):
a = {'ganji_esf_price', 'anjuke_esf_price'}
# ####print('a=',a)
b = set(list(junjia.index))
# ####print('b=',b)
c = a & b
u=list(b-a)
if len(c) == 2 and len(junjia)>2:
gj=junjia['ganji_esf_price']
ajk=junjia['anjuke_esf_price']
ratio=abs((gj-ajk)/min(gj,ajk))
other=junjia[u[0]]
ott=(other-max(gj,ajk))/min(gj,ajk)
if ratio<0.1 and ott>0.3:
return junjia.drop('ganji_esf_price')
else:
return junjia
else:
return junjia
lk = p[0]
location = take_location(ct, rl, p)
#####print('location',location)
# ttp=time.time()
neighb = no_source_price(ct, rl, location, p)
# ####print(time.time()-ttp,'m'*20)
junjia = dsfjujia(ct, rl, lk)
#print('junjia=',junjia)
print('neighb=',neighb)
'''
dglg=dsplg(junjia,neighb,2)
dggg=dsplg(junjia,neighb,1)
'''
judp=drop_ganji(junjia)
dglg = dglg_out1(judp, neighb[0])
print('dglg=',dglg)
# dggg=dglg_out1(junjia,neighb[0])
# ###print('dglg=',dglg)
# ####print('dggg=',dggg)
#tt=time.time()
data = consider_villa(ct, rl, p)
#data.to_excel('data.xls')
#data.to_excel('ddw.xls')
##print(time.time()-tt,'<>'*26)
#######################################################
@@ -1114,13 +1118,24 @@ def commonprice(ct, rl, p):
#print(type(ppd1['area'][1]))
#ppd1[['total_price','area']].to_excel('ppd1.xls')
ppd1['total_price'] = ppd1['total_price'] / ppd1['area']
# ####print('ppd1time',time.time()-t3,'o'*36)
# ppd1.loc['total_price']=ppd1['total_price']/ppd1['area']
# ppd2=ppd1
# if type(neighb) is not str:
def take_region_price(ct,rl):
def before_step_time(day):
now = datetime.datetime.now()
front = now - datetime.timedelta(days=day)
return front
aaw=before_step_time(14)
query1 = {"city": ct, "region": rl,'weekend':{'$gt':aaw}}
lf = fangjia.find_region_price(query1)
#print('lf================',lf)
if lf.shape[0]>0:
return lf['calc_value'].iloc[-1]
else:
return -1
if ppd1.shape[0] > 0:
@@ -1129,7 +1144,12 @@ def commonprice(ct, rl, p):
ph = ppd2[ppd2['total_price'] > 0.7 * neighb]
return ph
else:
return ppd2
sss=take_region_price(ct,rl)
if sss>0:
ph1 = ppd2[ppd2['total_price'] > 0.5 *sss]
return ph1
else:
return ppd2
# ppd3=ppd2[ppd2['total_price']>0.6*neighb]
@@ -1159,17 +1179,19 @@ def commonprice(ct, rl, p):
length = len(l_and_z)
length1 = len(allf)
#print('#########')
#a=datetime.datetime.now()
#timeArray = a.strftime("%Y%m%d")
#print('timeArray=',timeArray)
#front_step_time(360)
def ss1(junjia,dglg,ppd3):
sample_size = len(junjia)
if dglg != None:
# ####print('junjia.shape',len(junjia))
dglg_dd=ppd3[(ppd3['total_price']>0.85*dglg) & (ppd3['total_price']<1.2*dglg)]
#dglg_dd.to_excel('dglg_dd.xls')
##print(dglg_dd)
#print(dglg_dd['p_date'])
if dglg_dd.shape[0]>2:
dglg1= k_means_clus(dglg_dd['total_price'].dropna(),3)
@@ -1178,8 +1200,44 @@ def commonprice(ct, rl, p):
return dglg,float('%.2f' % credit['0.85']), value_type['第三方'], sample_size
else:
return 0,float('%.2f' % credit['0.85']), value_type['第三方'], sample_size
dggg=ss1(junjia,dglg,ppd3)
def consider_time_ss1(junjia,dglg,ppd3,day):
sample_size = len(junjia)
if dglg != None:
# ####print('junjia.shape',len(junjia))
dglg_dd=ppd3[(ppd3['total_price']>0.85*dglg) & (ppd3['total_price']<1.2*dglg)]
#dglg_dd.to_excel('dglg_dd.xls')
#print(dglg_dd[['p_date','total_price','url']])
dglg_dd=dglg_dd[dglg_dd['p_date']>=day]
if dglg_dd.shape[0]>2:
dglg1= k_means_clus(dglg_dd['total_price'].dropna(),3)
return dglg1, float('%.2f' % credit['0.85']), value_type['第三方'], sample_size
else:
return -2, float('%.2f' % credit['0.85']), value_type['第三方'], sample_size
else:
return -2,float('%.2f' % credit['0.85']), value_type['第三方'], sample_size
def map_time_price(junjia,dglg,ppd3):
day_15=front_step_time(15)
day_30=front_step_time(30)
day_60=front_step_time(60)
price_15= consider_time_ss1(junjia,dglg,ppd3,day_15)
price_30= consider_time_ss1(junjia,dglg,ppd3,day_30)
price_60= consider_time_ss1(junjia,dglg,ppd3,day_60)
if price_15[0]>0:
print('first_15')
return price_15
elif price_30[0]>0:
print('first_30')
return price_30
elif price_60[0]>0:
print('first_60')
return price_60
else:
return ss1(junjia,dglg,ppd3)
dggg=map_time_price(junjia,dglg,ppd3)
if dggg[0]>0 :
return dggg
@@ -1201,14 +1259,7 @@ def commonprice(ct, rl, p):
return int(gm), float('%.2f' % credit['0.8']), value_type['自身样本'], sample_size # 优质样本
elif data.shape[0] > 0:
#####print('诸葛找房算法')
#####print('neighb',neighb)
# from zhuge import zhuge_consist
data['avg'] = data['total_price'] / data['area']
def guolv_data11(data, neighb):
@@ -1218,19 +1269,19 @@ def commonprice(ct, rl, p):
ddd11 = data[data['avg'] > 0.7 * neighb]
return ddd11
else:
return data
sss=take_region_price(ct,rl)
if sss>0:
ddd22 = data[data['avg'] > 0.5 * sss]
return ddd22
else:
return data
ddd11 = guolv_data11(data, neighb[0])
# ####print('ddd11=',ddd11)
#print('ddd11=',ddd11)
if ddd11.shape[0] > 2:
# s = zhuge_consist.url_lable(ddd11)
# s1 = zhuge_consist.drop_rubbish(s)
# ####print('s1=',s1)
# s2 = set(s1)
# ####print('s2=',s2)
if ddd11.shape[0] > 3:
from zhuge import zhuge_consist
@@ -1243,10 +1294,10 @@ def commonprice(ct, rl, p):
#cc1 = ddd11['avg'].dropna()
# ####print('cc1=',cc1)
#print(ppd3['total_price'],'ppppppppppppppp')
kmzhuge = k_means_clus(ppd3['total_price'], 3)
#print(ppd3['total_price'])
junjia['zhuge_logic'] = s
junjia['zhuge_logic'] = kmzhuge
junjia['kmzg'] = kmzhuge
@@ -1277,15 +1328,7 @@ def commonprice(ct, rl, p):
return junjia
junjia1 = drop_ganji(junjia)
#####print('ddd11',ddd11)
#print(junjia1)
#print(junjia_front)
#####print('junjia_ganji=',junjia1,'$'*100)
cc = group(junjia_front, junjia1, 1,10)
# junjia_km=k_means(junjia)
#print('cc=',cc)
############
if len(cc) > 0:
cc1 = pandas.DataFrame(cc, columns=['price', 'count'])
cc2 = cc1.sort_values(by='count', ascending=False)
@@ -1293,18 +1336,22 @@ def commonprice(ct, rl, p):
def two_one_choose(cc2):
qwrt=set(cc2['count'])
if len(qwrt)==1:
if len(qwrt)>1:
return cc2['price'].iloc[0]
else :
return junjia['kmzg']
else:
return junjia['zhuge_logic']
sample_size = ddd11.shape[0]
return two_one_choose(cc2) * 1.0, float('%.2f' % credit['0.7']), value_type[
'第三方'], sample_size
else:
#print('777777')
sample_size = ddd11.shape[0]
#print('junjia=',junjia)
return junjia['kmzg'] * 1.0, float('%.2f' % credit['0.6']), value_type[
'自身样本'], sample_size
#print('ddd11=',ddd11)
junjia_km = compelex(ddd11, s, neighb[0], junjia)
return junjia_km
@@ -1317,6 +1364,8 @@ def commonprice(ct, rl, p):
cc1 = ddd11['avg'].dropna()
kmzhuge = k_means_clus(cc1, 2)
sample_size = ddd11.shape[0]
#print('#########')
return kmzhuge, float('%.2f' % credit['0.6']), value_type['自身样本'], sample_size
else:
return neighb[0], float('%.2f' % credit['0.5']), value_type['周边楼盘'], neighb[1]
@@ -1327,6 +1376,10 @@ def commonprice(ct, rl, p):
return dglg, float('%.2f' % credit['0.8']), value_type['第三方'], sample_size # '优质样本'
else:
#print('lgggggg=')
#print('neighb[0]=',neighb[0])
return neighb[0], float('%.2f' % credit['0.5']), value_type['周边楼盘'], neighb[1]
elif dglg != None:
@@ -1357,9 +1410,6 @@ def commonprice(ct, rl, p):
else:
# ####print('*'*100)
# ####print(neighb)
return neighb[0], float('%.2f' % credit['0.5']), value_type['周边楼盘'], neighb[1]
@@ -1379,10 +1429,12 @@ def ghhh(kkg):
return a
stat = fangjia.get_collection("fangjia", "district_stat", config.get_district_stat_host(), config.get_district_stat_port())
pt = fangjia.get_collection("fangjia_log", "problem_command", config.get_log_mongodb_host(), config.get_log_mongodb_port())
#stat = fangjia.get_collection("fangjia", "changzhou_6", config.get_district_stat_host(), config.get_district_stat_port())
#stat = fangjia.get_collection("fangjia", "district_stat", config.get_district_stat_host(), config.get_district_stat_port())
#pt = fangjia.get_collection("fangjia_log", "problem_command", config.get_log_mongodb_host(), config.get_log_mongodb_port())
#stat = fangjia.get_collection("fangjia", "Copy_of_district_stat_shanghai_april_ten", '192.168.10.182', 27017)
stat = fangjia.get_collection("fangjia_stat", "shanghai0419", '192.168.10.182', 27017)
pt = fangjia.get_collection("fangjia_log", "problem_command", '192.168.10.182', 27017)
#################################################################
def region_district_list_lk(ct, rl, lk):
@@ -1420,7 +1472,7 @@ def last_sunday(weekend):
def make_up_price(ct, rl, lk, weekend):
####print('lg=',ct,rl,lk)
# last_sunday=last_sunday(weekend)
# last_sunday=last_sunday(weekend)南湖国际中心二期
# save_lg=client136.fangjia.district_stat
if lk != None:
@@ -1505,13 +1557,13 @@ def pt_down(ct,rl,p,weekend,c_date,cmd_estate_type,mm):
)
@timer
def stat_save(ct, rl, p, weekend, mm, c_date, m_date, make_up, estate_type):
def stat_save(ct, rl, p, weekend, mm, c_date, m_date, make_up, estate_type,avg):
stat.replace_one(
{"city": ct, "region": rl, "district_name": p[0], "weekend": weekend},
{"city": ct, "region": rl, "district_name": p[0], 'credit': mm[1],
"calc_value": int(mm[0]), "weekend": weekend, "c_date": c_date, 'avg_price':int(mm[0]),
"calc_value": int(mm[0]), "weekend": weekend, "c_date": c_date, 'avg_price':int(avg),
"m_date": m_date, 'value_type': int(mm[2]), 'weekend_markup': make_up,
'changed_markup': make_up, 'markup': make_up, 'sample_size': mm[3],
'estate_type': estate_type}, True
@@ -1533,6 +1585,7 @@ def pt_up(ct, rl, p, weekend, c_date, cmd_estate_type):
@timer
def out_put_price(ct, rl, lk, weekend):
lf = region_district_list_lk(ct, rl, lk)
if lf.shape[0]>0:
lf = add_alias(lf)
p = ghhh(lf[['name', 'alias', 'estate_type2']])[0]
@@ -1555,7 +1608,7 @@ def out_put_price(ct, rl, lk, weekend):
##print(type(mm[0]))
if type(mm[0]) != str:
print(ct, rl, lk, mm, '$' * 20)
# ####print(lf['estate_type2'].iloc[0],'*'*10)
@@ -1580,13 +1633,24 @@ def out_put_price(ct, rl, lk, weekend):
#####print('make_up',make_up,'*'*20)
#####print('credic',mm[1])
#avg=fix.price_fix.update(ct,rl,lk,int(mm[0]))
###print('avg_price=',avg)
def avg_compare(ct,rl,lk,mm):
avg=fix.price_fix.update(ct,rl,lk,int(mm[0]))
#print('nihe=',avg)
#print('mm[0]',mm[0])
if avg>0:
return avg
else:
return int(mm[0])
#s1=time.time()
avg=avg_compare(ct,rl,lk,mm)
#print('s1_time=',time.time()-s1)
#avg=mm[0]
print('mm=',mm)
print(ct, rl, lk, avg, 'avg=',avg,'$' * 20)
try:
stat_save(ct,rl,p,weekend,mm,c_date,m_date,make_up,estate_type)
stat_save(ct,rl,p,weekend,mm,c_date,m_date,make_up,estate_type,avg)
pt_up(ct,rl,p,weekend,c_date,cmd_estate_type)
'''
@@ -1616,7 +1680,7 @@ def out_put_price(ct, rl, lk, weekend):
except:
stat_save(ct,rl,p,weekend,mm,c_date,m_date,make_up,estate_type)
stat_save(ct,rl,p,weekend,mm,c_date,m_date,make_up,estate_type,avg)
+25 -9
View File
@@ -14,30 +14,35 @@ def get_weekend(tt):
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(bb)
weekend = get_weekend('20180415')
weekend = get_weekend('20180422')
s = time.time()
ct='嘉兴'
rl='海盐'
lk='保利壹品'
ct='上海'
rl='黄浦'
lk='伯爵居'#杨浦凤城四村闵行上海康城厦门同安大唐世家二期青岛市北东方小区合肥庐阳海亮红玺台青岛市北东方小区
#深圳龙岗浪背村宝安吉祥公寓门头沟 泷悦长安北京上海闵行金盛小区江苏昆山象屿珑庭
#for i in range(0, 100):深圳福田世纪豪庭
#for i in range(0, 100):深圳福田世纪豪庭北京海淀前沙涧襄阳樊城国色天襄
#北区济南章丘朝阳南区上海奉贤嘉安公寓
sellPrice.out_put_price(ct, rl, lk, weekend)
e = time.time()
print(e - s)
'''
###
import pandas as pd
a=pd.read_excel('小区调查.xlsx')
a=pd.read_excel('跑价格(1).xlsx')
for i in a.index:
ct=a['城市'][i]
@@ -45,7 +50,18 @@ for i in a.index:
lk=a['小区名'][i]
sellPrice.out_put_price(ct, rl, lk, weekend)
'''
'''
'''