1777 lines
58 KiB
Python
1777 lines
58 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""
|
|
Created on Fri Sep 22 18:18:15 2017
|
|
|
|
@author: 罗干
|
|
|
|
1;本算法在构建的过程中得到王文艳的很多帮助
|
|
特此鸣谢!
|
|
|
|
"""
|
|
#导入相关的库
|
|
import os
|
|
|
|
#####print(os.getcwd())
|
|
import random
|
|
import collections
|
|
import pymongo
|
|
import sklearn.cluster
|
|
import pandas
|
|
import numpy
|
|
import numpy as np
|
|
import time
|
|
import fangjia
|
|
#svm向量回归用于计算无房源小区的房价
|
|
import sklearn.svm
|
|
import datetime
|
|
import config
|
|
from fangjia import timer
|
|
import fix
|
|
import time
|
|
|
|
# import yesterday
|
|
|
|
#本函数用于时间处理,即时间前推或后推处理,并且按照int类型输出
|
|
|
|
def front_step_time(weekend,day):
|
|
#now = datetime.datetime.now()
|
|
now=weekend
|
|
front = now - datetime.timedelta(days=day)
|
|
d1 = front.strftime('%Y%m%d')
|
|
return int(d1)
|
|
|
|
|
|
|
|
|
|
############################################################
|
|
############################################################
|
|
# 存到数据库的日期
|
|
|
|
|
|
# date="2018-02-04"
|
|
############################################################
|
|
############################################################
|
|
|
|
#均价的可信度定义
|
|
# 0.9表示人工指导价,本算法还未加入
|
|
#0.88 当第三方价格样本量在5个及以上时,通过函数置换出的价格,由于由第三方均价所出的价格已经非常准确,避免不良房源的干扰,房源不参与计算
|
|
#0.85 第三方均价可以置换出某个价格,并且参考房源所出的价格
|
|
#0.80 采用链家和诸葛找房合并后的样本
|
|
#0.75 未采用
|
|
#0.7 同时参考样本和未能置换出价格的第三方均价
|
|
#0.6 仅仅采用房源样本聚类
|
|
#0.5 无房源也无第三方均价,采用周边小区计算
|
|
credit = pandas.Series([0.9,0.88,0.85, 0.8,0.75, 0.7, 0.6, 0.5], index=['0.9', '0.88','0.85','0.8','0.75', '0.7', '0.6', '0.5'])
|
|
|
|
#计算来源
|
|
value_type = pandas.Series([1, 2, 3, 4, 5, 6, 7],
|
|
index=['指导价', '自身样本', '第三方', '交易数据', '周边楼盘', '推导', '优质样本'])
|
|
|
|
|
|
# 1:指导价 2:自身样本 3:第三方 4:交易数据 5:周边楼盘, 6:推导
|
|
|
|
|
|
#计算均价函数入口
|
|
def commonprice(ct, rl, p,weekend):
|
|
|
|
|
|
|
|
# 提取房源的数据时间上限,如从20170301之后的数据
|
|
some_district_information_time = front_step_time(weekend,360)
|
|
|
|
# 提周边小区均价的数据时间上限,如从20170801之后的数据
|
|
zhoubian_xiaoqu_time = front_step_time(weekend,200)
|
|
|
|
# 第三方均价的数据时间上限,如从201703之后的数据
|
|
dsfjujia_time = front_step_time(weekend,200) // 100
|
|
|
|
#增加DataFrame的板块 block
|
|
def add_block(ppd):
|
|
if 'block' not in ppd.columns:
|
|
ppd['block'] = '没有'
|
|
return ppd
|
|
#增加DataFrame的 room
|
|
def add_room(ppd):
|
|
if 'room' not in ppd.columns:
|
|
ppd['room'] = 2
|
|
return ppd
|
|
|
|
@timer
|
|
|
|
#从数据库中读取房源数据,seawater
|
|
def some_district_information(ct, rl, lk):
|
|
|
|
query = {"city": ct, "cat": "sell", "region": rl,
|
|
"district_name": {"$in": lk},
|
|
"p_date": {"$gt": some_district_information_time}}
|
|
|
|
# ['total_price', 'area', 'region', 'district_name', 'url', 'block', 'p_date']
|
|
|
|
fields1 = {'total_price': 1, 'area': 1, 'region': 1, 'district_name': 1, 'url': 1, 'p_date': 1, "name": 1}
|
|
|
|
return fangjia.findHouse(query, fields1)
|
|
"""
|
|
return fangjia.find_house_from_elastic_search(ct, rl, lk,
|
|
"lianjia.com", some_district_information_time)
|
|
"""
|
|
#从数据库中读取第三方均价,"fangjia", "average_statistics"
|
|
def disanganf_price(ct, rl, lk, tt):
|
|
# db1 = client1.fangjia
|
|
# dsf = db1.average_statistics
|
|
query = {"city": ct, "region": rl,
|
|
"name": lk, "s_date": {"$gt": tt}}
|
|
# pos = list(dsf136.find(query))
|
|
|
|
# data1=pandas.DataFrame(pos)
|
|
data1 = fangjia.find_disanganf_price(query)
|
|
|
|
if data1.shape[0] > 1:
|
|
a_set = {'fangjia_price'}
|
|
b_set = set(data1.columns)
|
|
if a_set & b_set == a_set:
|
|
|
|
data = data1[data1['credit'] > 0.7]
|
|
if data.shape[0] >= 1:
|
|
choose_class = ['city', 'region', 'name', 'fangjia_price', 'credit',
|
|
's_date']
|
|
dc = data[choose_class]
|
|
return dc
|
|
else:
|
|
return numpy.array([])
|
|
else:
|
|
|
|
return numpy.array([])
|
|
else:
|
|
return numpy.array([])
|
|
#提取某个小区的经纬度
|
|
def take_location(ct, rl, p):
|
|
|
|
dirlt = p[0]
|
|
|
|
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
|
# db1 = client1.fangjia
|
|
# seaweed1 = db1.seaweed
|
|
seaweed136 = fangjia.get_collection("fangjia", "seaweed", config.get_seaweed_host(), 27017)
|
|
query1 = {"status": 0, "cat": "district", "city": ct,
|
|
"region": rl, "name": dirlt}
|
|
|
|
for err_count in range(0, 10):
|
|
|
|
try:
|
|
ltt = seaweed136.count(query1)
|
|
if ltt > 0:
|
|
fields1 = {"lat2": 1, "lng2": 1, "city": 1, "region": 1, "cat": 1, "name": 1}
|
|
lct = seaweed136.find(query1, fields1)
|
|
######print('lct=',lct)
|
|
lf = pandas.DataFrame(list(lct))
|
|
|
|
######print('lf=',lf)
|
|
|
|
# lf=fangjia.find_seaweed(query1, fields1)
|
|
le = lf
|
|
le.index = le['name']
|
|
cluns = lf.columns
|
|
if ('lng2' in cluns) and ('lat2' in cluns):
|
|
lr = le[['lng2', 'lat2']]
|
|
del lf, le, cluns
|
|
return lr
|
|
else:
|
|
del lf, le, cluns
|
|
return numpy.array([])
|
|
|
|
else:
|
|
return numpy.array([])
|
|
except:
|
|
|
|
ltt = seaweed136.count(query1)
|
|
if ltt > 0:
|
|
fields1 = {"lat2": 1, "lng2": 1, "city": 1, "region": 1, "cat": 1, "name": 1}
|
|
lct = seaweed136.find(query1, fields1)
|
|
lf = pandas.DataFrame(list(lct))
|
|
|
|
# lf=fangjia.find_seaweed(query1, fields1)
|
|
le = lf
|
|
le.index = le['name']
|
|
cluns = lf.columns
|
|
if ('lng2' in cluns) and ('lat2' in cluns):
|
|
lr = le[['lng2', 'lat2']]
|
|
del lf, le, cluns
|
|
return lr
|
|
else:
|
|
del lf, le, cluns
|
|
return numpy.array([])
|
|
|
|
else:
|
|
return numpy.array([])
|
|
#以下几个函数用来判断数据来源
|
|
#判断某条房源是否来自链家
|
|
def lianjia(ur):
|
|
# #####print(ur)
|
|
if 'lianjia' in ur:
|
|
return 0
|
|
else:
|
|
return 1
|
|
#判断某条房源是否来自诸葛找房
|
|
def zhuge(ur):
|
|
if 'zhuge' in ur:
|
|
return 0
|
|
else:
|
|
return 1
|
|
#判断某条房源是否来自我爱我家
|
|
def wawj(ur):
|
|
if '5i5j' in ur:
|
|
return 0
|
|
else:
|
|
return 1
|
|
#爱屋及屋
|
|
def iwjw(ur):
|
|
if 'iwjw' in ur:
|
|
return 0
|
|
else:
|
|
return 1
|
|
#房估估
|
|
def fangdd(ur):
|
|
if 'fangdd' in ur:
|
|
return 0
|
|
else:
|
|
return 1
|
|
#悟空找房
|
|
def wkzf(ur):
|
|
if 'wkzf' in ur:
|
|
return 0
|
|
else:
|
|
return 1
|
|
|
|
def label(dd, f):
|
|
aa = list()
|
|
|
|
for i in dd.index:
|
|
d2f = dd['url'][i]
|
|
if type(d2f) == str:
|
|
if f(dd['url'][i]) == 0:
|
|
aa.append(i)
|
|
return set(aa)
|
|
|
|
@timer
|
|
#最新的一条数据
|
|
def new_data(ds):
|
|
bb = set(ds['name'])
|
|
bb1 = list(bb)
|
|
x1 = pandas.Series()
|
|
for i in bb1:
|
|
a1 = ds[ds['name'] == i]
|
|
######print('^^^^^^^^')
|
|
a2 = a1.sort_values(by='s_date', ascending=False)
|
|
######print('a2=',a2)
|
|
x1[i] = a2.iloc[0]['fangjia_price']
|
|
return x1
|
|
|
|
def zhoubian_xiaoqu(ct, rl, dirst):
|
|
|
|
# 提取周边小聚的均价,本方法不用房源,直接用上一批数据的价格来计算无房源小区
|
|
###################
|
|
###################
|
|
tii = zhoubian_xiaoqu_time
|
|
###################
|
|
###################
|
|
|
|
tt = tii // 100 - 1
|
|
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
|
# db1 = client1.fangjia
|
|
# dsf = db1.average_statistics
|
|
query = {"city": ct, "region": rl,
|
|
"name": {"$in": dirst}, "s_date": {"$gt": tt}}
|
|
# pos = list( dsf136.find(query))
|
|
|
|
# data=pandas.DataFrame(pos)
|
|
data = fangjia.find_disanganf_price(query)
|
|
# #####print(data)
|
|
if data.shape[0] >= 1:
|
|
# data=add_block(data)
|
|
data = add_room(data)
|
|
|
|
a_set = {'city', 'region', 'name', 'fangjia_price', 'credit',
|
|
's_date'}
|
|
b_set = set(data.columns)
|
|
|
|
if a_set & b_set == a_set:
|
|
|
|
choose_class = ['city', 'region', 'name', 'fangjia_price', 'credit',
|
|
's_date']
|
|
dc = data[choose_class]
|
|
return dc
|
|
else:
|
|
return numpy.array([])
|
|
return numpy.array([])
|
|
else:
|
|
|
|
return numpy.array([])
|
|
|
|
def hh(ppp, df):
|
|
ppp['price'] = None
|
|
aa = list(df.index)
|
|
# #####print('aa=',aa)
|
|
for i in ppp.index:
|
|
kk = ppp['district_name'][i]
|
|
# #####print('kk=',kk)
|
|
if kk in aa:
|
|
# ppp['price'][i]=df[kk]
|
|
ppp.loc[i, 'price'] = df[kk]
|
|
# #####print('df[kk]',df[kk])
|
|
return ppp
|
|
|
|
#搜索周边小区,并且输出周边小区的房价
|
|
#在提取周边小区时,按照距离远近进行排序,距离的定义是曼哈顿距离
|
|
@timer
|
|
def neighbor(lng, lat, scale, ct, rl, p):
|
|
|
|
#搜索周边小区
|
|
###################
|
|
###################
|
|
# timm=20171122
|
|
###################
|
|
###################
|
|
#小区的物业类型,有普通住宅和别墅
|
|
estype = p[2]
|
|
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
|
# db1 = client1.fangjia
|
|
# seaweed1 = db1.seaweed
|
|
kkk = scale / 1000
|
|
longtitude = lng
|
|
latitude = lat
|
|
#搜索周边的一定经纬度范围内的小区
|
|
query1 = {"status": 0, "cat": "district",
|
|
"lat2": {"$gt": int(latitude - kkk * 10 ** 10 / 0.86), "$lt": int(latitude + kkk * 10 ** 10 / 0.86)},
|
|
"lng2": {"$gt": int(longtitude - kkk * 10 ** 10), "$lt": int(longtitude + kkk * 10 ** 10)}}
|
|
# fields1 = {"lat2":1,"lng2":1, "city":1,"region":1,"cat":1,"name":1}
|
|
|
|
fields1 = {"lat2": 1, "lng2": 1, "city": 1, "region": 1,
|
|
"cat": 1, "name": 1, 'estate_type2': 1}
|
|
|
|
# lct= list( seaweed136.find(query1, fields1))
|
|
|
|
# lf=pandas.DataFrame(lct)
|
|
lf = fangjia.find_seaweed(query1, fields1)
|
|
|
|
######print('lf',lf[['estate_type2','lng2', 'lat2']])
|
|
|
|
#在提取周边小区时需要考虑小区的物业类型
|
|
le = lf[lf['estate_type2'] == estype]
|
|
######print(le['name'])
|
|
|
|
le.index = le['name']
|
|
|
|
lr = le[['lng2', 'lat2']]
|
|
|
|
lr['name1']=le['name']
|
|
|
|
#计算周边小区距离中心小区的距离,本距离为曼哈顿距离
|
|
def sort_manhaten(lr):
|
|
lr1=lr.values
|
|
####print(lr1)
|
|
lr2=[]
|
|
|
|
for m in range(lr1.shape[0]):
|
|
######print(m)
|
|
distance=abs(lng-lr1[m][0])+abs(lat-lr1[m][1])
|
|
lr2.append(distance/100000)
|
|
|
|
lr3=np.c_[lr1,lr2]
|
|
lrr=lr3[lr3[:,3].argsort()]
|
|
|
|
|
|
return lrr
|
|
|
|
|
|
lr1=sort_manhaten(lr)
|
|
|
|
dirst1 = list(lr1[:,2])
|
|
|
|
#random.shuffle(dirst1)
|
|
|
|
#提取周边的80个小区
|
|
dirst = dirst1[:80]
|
|
######print('dirst=',dirst)
|
|
|
|
dirst_plus = dirst + [p[0]]
|
|
|
|
query_feature = {"city": ct, "region": rl,
|
|
"district_name": {"$in": dirst_plus}}
|
|
|
|
ppp = fangjia.dis_feature(query_feature)
|
|
######print('ox'*10,ppp)
|
|
|
|
# #####print('小区个数= ',len(dirst))
|
|
df1 = zhoubian_xiaoqu(ct, rl, dirst)
|
|
######print('df1=',df1,'*'*20)
|
|
if df1.shape[0] > 1:
|
|
df = new_data(df1)
|
|
#####print('df=',df)
|
|
vv = hh(ppp, df)
|
|
#####print('#'*36)
|
|
######print('vv=',vv)
|
|
return vv
|
|
else:
|
|
return numpy.array([])
|
|
|
|
#本函数与上一个函数基本相同,仅仅缺少一行代码,le = lf[lf['estate_type2'] == estype]
|
|
#读取周边小区价格,不考虑其物业类型,原因是别墅或商住楼周边的同物业类型的小区非常少
|
|
#故而提取周边小区时不考虑其物业类型
|
|
def neighbor_km(lng, lat, scale, ct, rl, p):
|
|
|
|
###################
|
|
###################
|
|
# timm=20171122
|
|
###################
|
|
###################
|
|
|
|
#estype = p[2]
|
|
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
|
# db1 = client1.fangjia
|
|
# seaweed1 = db1.seaweed
|
|
kkk = scale / 1000
|
|
longtitude = lng
|
|
latitude = lat
|
|
|
|
query1 = {"status": 0, "cat": "district",
|
|
"lat2": {"$gt": int(latitude - kkk * 10 ** 10 / 0.86), "$lt": int(latitude + kkk * 10 ** 10 / 0.86)},
|
|
"lng2": {"$gt": int(longtitude - kkk * 10 ** 10), "$lt": int(longtitude + kkk * 10 ** 10)}}
|
|
# fields1 = {"lat2":1,"lng2":1, "city":1,"region":1,"cat":1,"name":1}
|
|
|
|
fields1 = {"lat2": 1, "lng2": 1, "city": 1, "region": 1,#
|
|
"cat": 1, "name": 1, 'estate_type2': 1}
|
|
|
|
# lct= list( seaweed136.find(query1, fields1))
|
|
|
|
# lf=pandas.DataFrame(lct)
|
|
lf = fangjia.find_seaweed(query1, fields1)
|
|
|
|
######print('lf',lf[['estate_type2','lng2', 'lat2']])
|
|
|
|
#le = lf[lf['estate_type2'] == estype]
|
|
le = lf
|
|
######print('le',le[['estate_type2','lng2', 'lat2']])
|
|
|
|
le.index = le['name']
|
|
|
|
lr = le[['lng2', 'lat2']]
|
|
lr['name1']=le['name']
|
|
|
|
|
|
def sort_manhaten(lr):
|
|
lr1=lr.values
|
|
####print(lr1)
|
|
lr2=[]
|
|
|
|
for m in range(lr1.shape[0]):
|
|
######print(m)
|
|
distance=abs(lng-lr1[m][0])+abs(lat-lr1[m][1])
|
|
lr2.append(distance/100000)
|
|
|
|
lr3=np.c_[lr1,lr2]
|
|
lrr=lr3[lr3[:,3].argsort()]
|
|
|
|
|
|
return lrr
|
|
|
|
|
|
lr1=sort_manhaten(lr)
|
|
|
|
dirst1 = list(lr1[:,2])
|
|
|
|
#random.shuffle(dirst1)
|
|
|
|
dirst = dirst1[:80]
|
|
|
|
# #####print('dirst=',dirst)
|
|
|
|
dirst_plus = dirst + [p[0]]
|
|
|
|
query_feature = {"city": ct, "region": rl,
|
|
"district_name": {"$in": dirst_plus}}
|
|
|
|
ppp = fangjia.dis_feature(query_feature)
|
|
# #####print('ox'*10,ppp)
|
|
|
|
# #####print('小区个数= ',len(dirst))
|
|
df1 = zhoubian_xiaoqu(ct, rl, dirst)
|
|
######print('df1=',df1,'*'*20)
|
|
if df1.shape[0] > 1:
|
|
df = new_data(df1)
|
|
######print('df=',df)
|
|
vv = hh(ppp, df)
|
|
# #####print('#'*36)
|
|
# #####print('vv=',vv)
|
|
return vv
|
|
else:
|
|
return numpy.array([])
|
|
#向量回归计算小区均价
|
|
def svr(vv, lk):
|
|
# 提取要计算小区的特征
|
|
vk = vv[vv['district_name'] == lk]
|
|
# 删除要计算小区的空特征
|
|
# #####print('vk=',vk)
|
|
if vk.shape[0] > 0:
|
|
|
|
vk1 = vk.iloc[0].dropna()
|
|
|
|
# 将要计算小区的特征提取出来
|
|
|
|
vk2 = list(set(vk1.index) - {'_id', 'district_name', 'region', 'city'})
|
|
|
|
# #####print('vk2=',vk2)
|
|
if len(vk2) > 1:
|
|
vvo = vv[vv['district_name'] != lk]
|
|
vk3 = vk2 + ['price']
|
|
vk31 = list(set(vk3))
|
|
vvo1 = vvo[vk31].dropna()
|
|
|
|
x1 = vvo1.drop('price', axis=1)
|
|
# #####print('x1=',x1)
|
|
y = vvo1['price']
|
|
# #####print('y',y)
|
|
#向量回归
|
|
if len(x1) > 2:
|
|
|
|
clf = sklearn.svm.SVR()
|
|
clf.fit(x1, y)
|
|
pp1 = vk[vk31].drop('price', axis=1)
|
|
result = clf.predict(pp1)
|
|
# #####print (result)
|
|
return int(result[0])
|
|
else:
|
|
#聚类
|
|
km = vv['price'].dropna()
|
|
return int(k_means(km))
|
|
|
|
|
|
else:
|
|
km = vv['price'].dropna()
|
|
return int(k_means(km))
|
|
|
|
else:
|
|
km = vv['price'].dropna()
|
|
return int(k_means(km))
|
|
#计算无房源小区的均价
|
|
def no_source_price(ct, rl, location, p):
|
|
|
|
|
|
lk = p[0]
|
|
hg = location.shape[0]
|
|
|
|
'''
|
|
|
|
def choose_factor(p):
|
|
typ=p[2]
|
|
if typ=='普通住宅':
|
|
return 20
|
|
else:
|
|
return 5
|
|
'''
|
|
|
|
#factor=choose_factor(p)
|
|
|
|
factor=20
|
|
|
|
if hg == 0:
|
|
return '没有经纬度数据', 0
|
|
|
|
#elif hg > 1:
|
|
#return '纬度数据有误,有多个经纬度', 0
|
|
|
|
else:
|
|
#t1=time.time()
|
|
|
|
######print("location['lng2'][0]",location['lng2'].iloc[0])
|
|
|
|
dd = neighbor(location['lng2'].iloc[0], location['lat2'].iloc[0], 5, ct, rl, p)
|
|
##print(time.time()-t1,'t1'*10)
|
|
##print('dd=',dd,'dd'*20)
|
|
if dd.shape[0] >= factor:
|
|
# #####print('$'*50)
|
|
# dd1=neighbor(location['lng2'][0],location['lat2'][0],5)
|
|
# #####print('dd',dd)
|
|
# svr_time=time.time()
|
|
other = svr(dd, lk)
|
|
# #####print(time.time()-svr_time,'svr'*10)
|
|
#####print('other= ', other, '9' * 20)
|
|
return other, dd.shape[0]
|
|
else:
|
|
# t2=time.time()
|
|
dd1 = neighbor(location['lng2'].iloc[0], location['lat2'].iloc[0], 10, ct, rl, p)
|
|
# #####print(time.time()-t2,'t2'*10)
|
|
# #####print('dd1=',dd1)
|
|
if dd1.shape[0] >= factor:
|
|
# #####print('$'*50)
|
|
|
|
# svr_time=time.time()
|
|
other1 = svr(dd1, lk)
|
|
# #####print(time.time()-svr_time,'svr'*10)
|
|
##print('other1=',other1)
|
|
return other1, dd1.shape[0]
|
|
else:
|
|
# t3=time.time()
|
|
dd2 = neighbor(location['lng2'].iloc[0], location['lat2'].iloc[0], 20, ct, rl, p)
|
|
# #####print(time.time()-t3,'t3'*10)
|
|
#print('dd2.shape', dd2 ,'@' * 60)
|
|
if dd2.shape[0] >= factor:
|
|
######print('$' * 50)
|
|
#svr_time = time.time()
|
|
other1 = svr(dd2, lk)
|
|
######print(time.time() - svr_time, 'e' * 10)
|
|
##print('other1=',other1)
|
|
return other1, dd2.shape[0]
|
|
|
|
else:
|
|
#不考虑物业类型
|
|
dd6 = neighbor_km(location['lng2'].iloc[0], location['lat2'].iloc[0], 20, ct, rl, p)
|
|
##print('dd2=',dd2,'@'*60)
|
|
if dd6.shape[0] >= 3:
|
|
# svr_time=time.time()
|
|
other1 = svr(dd6, lk)
|
|
|
|
######print('#'*20)
|
|
# #####print(time.time()-svr_time,'svr'*10)
|
|
# #####print('other1=',other1)
|
|
return other1, dd6.shape[0]
|
|
else:
|
|
return '没有周围小区数据', 0
|
|
|
|
def region_district_list(ct, rl):
|
|
# import pandas
|
|
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
|
# db1 = client1.fangjia
|
|
# seaweed1 = db1.seaweed
|
|
|
|
query1 = {"status": 0, "cat": "district", "city": ct, "region": rl}
|
|
fields1 = {"lat2": 1, "lng2": 1, "city": 1, "region": 1,
|
|
"cat": 1, "name": 1, 'estate_type2': 1, 'alias': 1}
|
|
|
|
# lct= list(seaweed136.find(query1, fields1))
|
|
# lf=pandas.DataFrame(lct)
|
|
lf = fangjia.find_seaweed(query1, fields1)
|
|
return lf
|
|
|
|
#聚类算法
|
|
def k_means(pp1):
|
|
from sklearn.cluster import KMeans
|
|
pv = list(pp1)
|
|
if len(set(pv)) > 3:
|
|
gf = numpy.array([pv]).T
|
|
estimator = KMeans(n_clusters=3) # 构造聚类器
|
|
estimator.fit(gf) # 聚类
|
|
label_pred = estimator.labels_ # 获取聚类标签
|
|
aa = collections.Counter(label_pred)
|
|
v = pandas.Series(aa)
|
|
gg = list(v)
|
|
index_max = gg.index(max(gg))
|
|
|
|
centroids = estimator.cluster_centers_ # 获取聚类中心
|
|
# inertia = estimator.inertia_ # 获取聚类准则的总和
|
|
center = centroids[index_max][0]
|
|
return (int(center))
|
|
else:
|
|
return int(pp1.mean())
|
|
|
|
def region_district_list_name(ct, rl, lkk):
|
|
# import pandas
|
|
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
|
# db1 = client1.fangjia
|
|
# seaweed1 = db1.seaweed
|
|
|
|
query1 = {"status": 0, "cat": "district", "city": ct, "region": rl, "name": lkk}
|
|
fields1 = {"lat2": 1, "lng2": 1, "city": 1, "region": 1,
|
|
"cat": 1, "name": 1, 'estate_type2': 1, 'alias': 1}
|
|
|
|
# lct= list( seaweed136.find(query1, fields1))
|
|
|
|
# lf=pandas.DataFrame(lct)
|
|
lf = fangjia.find_seaweed(query1, fields1)
|
|
return lf
|
|
|
|
def strip_villa(ct, rl, p):
|
|
lw = p[0]
|
|
# estype=p[2]
|
|
alias = p[1]
|
|
|
|
# #####print('alias=',alias)
|
|
|
|
def filt(x):
|
|
|
|
return type(x) == str and len(x) > 1
|
|
|
|
bb = list(filter(filt, alias))
|
|
# #####print('bb=',bb)
|
|
bb.extend([lw])
|
|
# lk1=list(set(lk))
|
|
|
|
# #####print('本小区名=',lw)
|
|
|
|
# #####print('type(lw)=',type(bb))
|
|
|
|
# #####print('小区名列表=',bb)
|
|
|
|
cc1 = some_district_information(ct, rl, bb)
|
|
|
|
cc1 = add_room(cc1)
|
|
|
|
#####print('cc1本小区数据条数=', cc1.shape[0])
|
|
|
|
if cc1.shape[0] > 2:
|
|
|
|
del lw, alias, bb
|
|
return cc1
|
|
else:
|
|
|
|
if '别墅' in lw:
|
|
|
|
lkk = lw.strip('别墅')
|
|
cc = region_district_list_name(ct, rl, lkk)
|
|
# #####print('cc=',cc)
|
|
if cc.shape[0] > 1:
|
|
|
|
cc1 = list(cc['alias'].dropna())
|
|
cc2 = list(filter(filt, alias))
|
|
bb.extend([lkk])
|
|
# clk1=list(set(clk))
|
|
cc2 = some_district_information(ct, rl, bb)
|
|
if cc2.shape[0] > 2:
|
|
del lw, alias, bb, cc1, cc
|
|
return cc2
|
|
else:
|
|
del lw, alias, bb, cc1, cc
|
|
return cc1
|
|
else:
|
|
return cc1
|
|
else:
|
|
return cc1
|
|
|
|
@timer
|
|
def consider_villa(ct, rl, p):
|
|
|
|
# #####print('villa p=',p)
|
|
|
|
estype = p[2]
|
|
df = strip_villa(ct, rl, p)
|
|
if df.shape[0] > 1:
|
|
|
|
if estype == '普通住宅':
|
|
all_index = set(df.index)
|
|
vv = fetch_villa_index(df)
|
|
ww = all_index - vv
|
|
|
|
d1 = df.loc[ww, :]
|
|
if d1.shape[0] > 3:
|
|
return d1
|
|
else:
|
|
return df
|
|
elif estype == '别墅':
|
|
vv = fetch_villa_index(df)
|
|
d1 = df.loc[vv, :]
|
|
if d1.shape[0] > 3:
|
|
return d1
|
|
else:
|
|
return df
|
|
else:
|
|
return df
|
|
else:
|
|
return df
|
|
|
|
def fetch_villa_index(dd):
|
|
|
|
# dd=dd.dropna()
|
|
d1 = dd[dd['area'] > 150]
|
|
|
|
# #####print(dd)
|
|
|
|
villa_label = ["独栋", "联排", "叠加", "双拼",
|
|
"花园面积", "泳池", "别墅", "带花园"]
|
|
bb = list()
|
|
for i in d1.index:
|
|
we = d1['name'][i]
|
|
for j in villa_label:
|
|
if j in we:
|
|
bb.append(i)
|
|
d21 = dd[dd['room'] >= 6]
|
|
d2 = d21[d21['area'] >= 360]
|
|
|
|
d3 = dd[dd['area'] > 500]
|
|
|
|
villa = set(bb) | set(d2.index) | set(d3.index)
|
|
|
|
return villa
|
|
|
|
def dsfjujia(ct, rl, lk):
|
|
|
|
query = {"city": ct, "region": rl,
|
|
"name": lk, "s_date": {"$gt": dsfjujia_time}}
|
|
|
|
data = fangjia.find_disanganf_price(query)
|
|
|
|
alist = ['anjuke_esf_price', 'ganji_esf_price',
|
|
'lianjia_esf_price', 'soufang_esf_price',
|
|
'tuboshi_esf_price', 'zhugezhaofang_esf_price',
|
|
'yungujia_esf_price', 'fanggugu_esf_price'
|
|
]
|
|
a1list = set(alist)
|
|
blist = set(data.columns)
|
|
|
|
clist = a1list & blist
|
|
|
|
ddq = list(clist)
|
|
|
|
data1 = data[ddq]
|
|
|
|
ser = pandas.Series()
|
|
|
|
for i in ddq:
|
|
pdd = data1[i].dropna()
|
|
# ser[i]=k_means(pdd)
|
|
ser[i] = list(pdd)[-1]
|
|
# #####print('采用抓取的第三方均价')
|
|
return ser
|
|
|
|
def sfjj(ll_zz, aa_zz, dddf):
|
|
|
|
dxx = pandas.Series()
|
|
for i in ll_zz:
|
|
if i in aa_zz:
|
|
dxx[i] = dddf[i]
|
|
return int(dxx.mean())
|
|
|
|
"""
|
|
tt=timm//100
|
|
|
|
lk=p[0]
|
|
dsfp1=disanganf_price(ct,rl,lk,tt)
|
|
|
|
data= consider_villa(ct,rl,p,timm)
|
|
"""
|
|
|
|
#######################################################
|
|
"""
|
|
junjia=dsfjujia(ct,rl,lk)
|
|
ll_zz={'zhugezhaofang_esf_price','lianjia_esf_price'}
|
|
aa_zz=set(junjia.index)
|
|
jiaji_lz= list(ll_zz & aa_zz)
|
|
"""
|
|
|
|
def k_means_clus(pp1, clus):
|
|
|
|
from sklearn.cluster import KMeans
|
|
pv = list(pp1)
|
|
if len(set(pv)) > clus:
|
|
gf = numpy.array([pv]).T
|
|
estimator = KMeans(n_clusters=clus) # 构造聚类器
|
|
estimator.fit(gf) # 聚类
|
|
label_pred = estimator.labels_ # 获取聚类标签
|
|
aa = collections.Counter(label_pred)
|
|
v = pandas.Series(aa)
|
|
gg = list(v)
|
|
index_max = gg.index(max(gg))
|
|
|
|
centroids = estimator.cluster_centers_ # 获取聚类中心
|
|
|
|
##print(centroids,gg)
|
|
# inertia = estimator.inertia_ # 获取聚类准则的总和
|
|
center = centroids[index_max][0]
|
|
return (int(center))
|
|
else:
|
|
return int(pp1.mean())
|
|
|
|
def group(junjia_lzf, junjia_anst, ctr,per):
|
|
|
|
bb = []
|
|
for i in junjia_lzf.index:
|
|
# #####print(junjia_lzf)
|
|
someprice = junjia_lzf[i]
|
|
|
|
def compare15(b):
|
|
if 100 * abs(someprice - b) / b < per:
|
|
return 1
|
|
else:
|
|
return 0
|
|
|
|
countmap = list(map(compare15, junjia_anst))
|
|
######print('countmap=',countmap)
|
|
countmap1 = pandas.Series(countmap)
|
|
countmap2 = countmap1[countmap1 > 0]
|
|
# #####print('countmap2=',countmap2)
|
|
|
|
numb = len(countmap2)
|
|
# #####print('len(countmap2)=',len(countmap2))
|
|
if numb > ctr:
|
|
bb.append([someprice, numb])
|
|
###print('bb=',bb)
|
|
return bb
|
|
|
|
def middle_operate(junjia_lzf, junjia_anst):
|
|
|
|
# #####print('%%%',ppd3['total_price'])
|
|
|
|
# #####print('junjia_lzf=',junjia_lzf)
|
|
# #####print('junjia_anst=',junjia_anst)
|
|
|
|
def mid_out_1(junjia_lzf,per):
|
|
|
|
if len(junjia_lzf) > 1:
|
|
# #####print('aa1',aa)
|
|
bb = group(junjia_lzf, junjia_lzf, 1,per)
|
|
# #####print('bb=',bb)
|
|
if len(bb) > 0:
|
|
bb1 = pandas.DataFrame(bb, columns=['price', 'count'])
|
|
bb2 = bb1.sort_values(by='count', ascending=False)
|
|
######print(bb2)
|
|
return bb2.iloc[0]['price']
|
|
else:
|
|
return None
|
|
else:
|
|
return None
|
|
|
|
def mid_out_2(junjia_lzf, junjia_anst,per):
|
|
|
|
if len(junjia_lzf) > 0 and len(junjia_anst) > 0:
|
|
|
|
bb = group(junjia_lzf, junjia_anst, 0,per)
|
|
# #####print('bb1=',bb)
|
|
if len(bb) > 0:
|
|
bb1 = pandas.DataFrame(bb, columns=['price', 'count'])
|
|
bb2 = bb1.sort_values(by='count', ascending=False)
|
|
# #####print(bb2)
|
|
return bb2.iloc[0]['price']
|
|
else:
|
|
return None
|
|
else:
|
|
return None
|
|
|
|
#a11 = mid_out_1(junjia_lzf,8)
|
|
|
|
######print('a11=',a11)
|
|
#a22 = mid_out_2(junjia_lzf, junjia_anst,8)
|
|
######print('a22=',a22)
|
|
#a33 = mid_out_1(junjia_anst,8)
|
|
a44 = mid_out_1(junjia_lzf,12)
|
|
a55 = mid_out_2(junjia_lzf, junjia_anst,12)
|
|
a66 = mid_out_1(junjia_anst,12)
|
|
######print('a33=',a33)
|
|
#aa1_6 = pandas.Series([a11, a22, a33,a44,a55,a66]).dropna()
|
|
aa1_6 = pandas.Series([ a44,a55,a66]).dropna()
|
|
|
|
######print('aa1_6=',aa1_6)
|
|
if len(aa1_6) > 0:
|
|
return aa1_6.iloc[0]
|
|
else:
|
|
return None
|
|
|
|
def devide_group(al, bl, junjia):
|
|
|
|
alls = set(junjia.index)
|
|
|
|
def set_div(ds, alls):
|
|
if len(set(ds) & alls) > 0:
|
|
return junjia[ds].dropna()
|
|
else:
|
|
return pandas.Series()
|
|
|
|
lzf = set_div(al, alls)
|
|
anst = set_div(bl, alls)
|
|
|
|
return lzf, anst
|
|
|
|
def dglg_out1(junjia, neighb):
|
|
|
|
"""
|
|
alist=['anjuke_esf_price','ganji_esf_price',
|
|
'lianjia_esf_price','soufang_esf_price',
|
|
'tuboshi_esf_price','zhugezhaofang_esf_price',
|
|
'yungujia_esf_price','fanggugu_esf_price'
|
|
]
|
|
"""
|
|
|
|
al = [
|
|
'lianjia_esf_price',
|
|
'zhugezhaofang_esf_price',
|
|
'fanggugu_esf_price',
|
|
'yungujia_esf_price'
|
|
]
|
|
bl = ['anjuke_esf_price',
|
|
'soufang_esf_price',
|
|
'tuboshi_esf_price',
|
|
'ganji_esf_price'
|
|
|
|
]
|
|
|
|
mmax=junjia.max()
|
|
def junjia_max(mmax,dg12):
|
|
if dg12!=None and mmax!=None:
|
|
|
|
return dg12
|
|
'''
|
|
ratiog=dg12/mmax
|
|
if 0.5<ratiog<0.7:
|
|
return mmax
|
|
else:
|
|
return dg12
|
|
'''
|
|
else:
|
|
return dg12
|
|
|
|
|
|
# ppd3dp=ppd3['total_price'].dropna()
|
|
if type(neighb) != str and neighb != None:
|
|
|
|
|
|
def cut_junjia(junjia,neighb):
|
|
|
|
if len(junjia)>3:
|
|
return junjia[junjia > 0.49* neighb]
|
|
else:
|
|
return junjia[junjia > 0.6* neighb]
|
|
|
|
|
|
junjia=cut_junjia(junjia,neighb)
|
|
'''
|
|
junjia_lzf=junjia[al].dropna()
|
|
junjia_anst=junjia[bl].dropna()
|
|
'''
|
|
alldi = devide_group(al, bl, junjia)
|
|
|
|
junjia_lzf = alldi[0]
|
|
junjia_anst = alldi[1]
|
|
dg11 = middle_operate(junjia_lzf, junjia_anst)
|
|
##print('dg11=',dg11)
|
|
return junjia_max(mmax,dg11)
|
|
|
|
|
|
else:
|
|
alldi = devide_group(al, bl, junjia)
|
|
|
|
junjia_lzf = alldi[0]
|
|
junjia_anst = alldi[1]
|
|
|
|
dg22 = middle_operate(junjia_lzf, junjia_anst)
|
|
return junjia_max(mmax,dg22)
|
|
|
|
#由于链家和诸葛找房的数据准确度比较高,增加链家和诸葛找房数据的权重,即放大20倍.
|
|
def add_weight(s1):
|
|
|
|
choose_columns=['url','area', 'district_name', 'name', 'p_date', 'region',
|
|
'total_price', 'room']
|
|
|
|
all_set=set(s1.columns)
|
|
|
|
|
|
if set(choose_columns).issubset(all_set):
|
|
|
|
s2=s1[choose_columns]
|
|
|
|
s3=s2.values
|
|
|
|
d=s3.tolist()
|
|
for k in range(len(d)):
|
|
###print(k)
|
|
if type( d[k][0])==str:
|
|
if 'lianjia.com' in d[k][0] or 'zhuge' in d[k][0]:
|
|
x1=[d[k]]*20
|
|
###print(x1)
|
|
###print(d)
|
|
d=d+x1
|
|
##print(d)
|
|
d1=pandas.DataFrame(np.array(d),columns=s2.columns)
|
|
d1[['area']] = d1[['area']].astype(float)
|
|
d1[['total_price']] = d1[['total_price']].astype(float)
|
|
d1[['p_date']] = d1[['p_date']].astype(float)
|
|
##print('s3.shape',s3.shape)
|
|
##print('d1.shape',d1.shape)
|
|
return d1
|
|
else:
|
|
return s1
|
|
#在第三方 均价中安居客的价格和赶集网的价格大概率的相同,在采用第三方均价计算价格时如果
|
|
#赶集和安居客几乎相同并且和其他数据不同,就drop赶集网 的均价
|
|
def drop_ganji(junjia):
|
|
a = {'ganji_esf_price', 'anjuke_esf_price'}
|
|
# #####print('a=',a)
|
|
b = set(list(junjia.index))
|
|
# #####print('b=',b)
|
|
c = a & b
|
|
u=list(b-a)
|
|
if len(c) == 2 and len(junjia)>2:
|
|
gj=junjia['ganji_esf_price']
|
|
ajk=junjia['anjuke_esf_price']
|
|
ratio=abs((gj-ajk)/min(gj,ajk))
|
|
other=junjia[u[0]]
|
|
ott=(other-max(gj,ajk))/min(gj,ajk)
|
|
if ratio<0.1 and ott>0.3:
|
|
return junjia.drop('ganji_esf_price')
|
|
else:
|
|
return junjia
|
|
else:
|
|
return junjia
|
|
lk = p[0]
|
|
|
|
#本小区的经纬度
|
|
location = take_location(ct, rl, p)
|
|
|
|
######print('location',location)
|
|
|
|
# ttp=time.time()
|
|
#本小区的周边价格(向量回归或者聚类)
|
|
neighb = no_source_price(ct, rl, location, p)
|
|
|
|
# #####print(time.time()-ttp,'m'*20)
|
|
#本小区的第三方均价
|
|
junjia = dsfjujia(ct, rl, lk)
|
|
print('junjia=',junjia)
|
|
#print('neighb=',neighb)
|
|
|
|
'''
|
|
dglg=dsplg(junjia,neighb,2)
|
|
dggg=dsplg(junjia,neighb,1)
|
|
'''
|
|
|
|
#在第三方 均价中安居客的价格和赶集网的价格大概率的相同,在采用第三方均价计算价格时如果
|
|
#赶集和安居客几乎相同并且和其他数据不同,就drop赶集网 的均价
|
|
|
|
judp=drop_ganji(junjia)
|
|
dglg = dglg_out1(judp, neighb[0])
|
|
print('dglg=',dglg)
|
|
# dggg=dglg_out1(junjia,neighb[0])
|
|
|
|
# ####print('dglg=',dglg)
|
|
# #####print('dggg=',dggg)
|
|
|
|
#tt=time.time()
|
|
data = consider_villa(ct, rl, p)
|
|
|
|
#data.to_excel('data.xls')
|
|
##print('data=',data)
|
|
|
|
#data.to_excel('ddw.xls')
|
|
###print(time.time()-tt,'<>'*26)
|
|
#######################################################
|
|
##print('rrrr')
|
|
##print(data.shape[0])
|
|
data=add_weight(data)
|
|
##print('data=',data)
|
|
if data.shape[0] > 3:
|
|
|
|
paa = add_block(data)
|
|
ppd = paa[['total_price', 'area', 'region', 'district_name', 'url', 'block', 'p_date']]
|
|
# #####print('ppd##############')
|
|
# #####print('ppd=',ppd)
|
|
ppd = ppd.fillna(method='ffill')
|
|
|
|
ppd1 = ppd.dropna()
|
|
|
|
# t3=time.time()
|
|
##print(ppd1[['total_price','area']])
|
|
##print(type(ppd1['total_price'][1]))
|
|
##print(type(ppd1['area'][1]))
|
|
#ppd1[['total_price','area']].to_excel('ppd1.xls')
|
|
ppd1['total_price'] = ppd1['total_price'] / ppd1['area']
|
|
|
|
|
|
def take_region_price(ct,rl):
|
|
|
|
|
|
def before_step_time(day):
|
|
now = datetime.datetime.now()
|
|
front = now - datetime.timedelta(days=day)
|
|
return front
|
|
aaw=before_step_time(14)
|
|
query1 = {"city": ct, "region": rl,'weekend':{'$gt':aaw}}
|
|
lf = fangjia.find_region_price(query1)
|
|
##print('lf================',lf)
|
|
|
|
if lf.shape[0]>0:
|
|
return lf['calc_value'].iloc[-1]
|
|
else:
|
|
return -1
|
|
|
|
if ppd1.shape[0] > 0:
|
|
|
|
def guolv_data(ppd2, neighb):
|
|
if type(neighb) is not str:
|
|
ph = ppd2[ppd2['total_price'] > 0.7 * neighb]
|
|
return ph
|
|
else:
|
|
sss=take_region_price(ct,rl)
|
|
if sss>0:
|
|
ph1 = ppd2[ppd2['total_price'] > 0.5 *sss]
|
|
return ph1
|
|
else:
|
|
return ppd2
|
|
|
|
# ppd3=ppd2[ppd2['total_price']>0.6*neighb]
|
|
|
|
######################################
|
|
ppd3 = guolv_data(ppd1, neighb[0])
|
|
|
|
#ppd3.to_excel('ppd4.xls')
|
|
|
|
######################################
|
|
ldf = label(ppd3, lianjia)
|
|
# #####print('ldf=',ldf)
|
|
|
|
ldf1 = label(ppd3, zhuge)
|
|
|
|
ldf2 = label(ppd3, wawj)
|
|
|
|
ldf3 = label(ppd3, iwjw)
|
|
ldf4 = label(ppd3, fangdd)
|
|
|
|
ldf5 = label(ppd3, wkzf)
|
|
l_and_z = ldf | ldf1
|
|
# #####print(l_and_z)
|
|
# #####print('l_and_z=',l_and_z)
|
|
allf = l_and_z | ldf3 | ldf4 | ldf5 | ldf2
|
|
# #####print('l_and_z_and_i=',l_and_z_and_i)
|
|
# mm=0
|
|
length = len(l_and_z)
|
|
length1 = len(allf)
|
|
##print('#########')
|
|
#a=datetime.datetime.now()
|
|
#timeArray = a.strftime("%Y%m%d")
|
|
|
|
##print('timeArray=',timeArray)
|
|
|
|
#front_step_time(360)
|
|
def ss1(junjia,dglg,ppd3):
|
|
sample_size = len(junjia)
|
|
if dglg != None:
|
|
# #####print('junjia.shape',len(junjia))
|
|
dglg_dd=ppd3[(ppd3['total_price']>0.85*dglg) & (ppd3['total_price']<1.2*dglg)]
|
|
#dglg_dd.to_excel('dglg_dd.xls')
|
|
##print(dglg_dd['p_date'])
|
|
|
|
if dglg_dd.shape[0]>2:
|
|
dglg1= k_means_clus(dglg_dd['total_price'].dropna(),3)
|
|
return dglg1, float('%.2f' % credit['0.85']), value_type['第三方'], sample_size
|
|
else:
|
|
return dglg,float('%.2f' % credit['0.85']), value_type['第三方'], sample_size
|
|
else:
|
|
return 0,float('%.2f' % credit['0.85']), value_type['第三方'], sample_size
|
|
|
|
def consider_time_ss1(junjia,dglg,ppd3,day):
|
|
sample_size = len(junjia)
|
|
if dglg != None:
|
|
# #####print('junjia.shape',len(junjia))
|
|
dglg_dd=ppd3[(ppd3['total_price']>0.85*dglg) & (ppd3['total_price']<1.2*dglg)]
|
|
#dglg_dd.to_excel('dglg_dd.xls')
|
|
#print(dglg_dd[['p_date','total_price','url']])
|
|
dglg_dd=dglg_dd[dglg_dd['p_date']>=day]
|
|
if dglg_dd.shape[0]>2:
|
|
dglg1= k_means_clus(dglg_dd['total_price'].dropna(),3)
|
|
return dglg1, float('%.2f' % credit['0.85']), value_type['第三方'], sample_size
|
|
else:
|
|
return -2, float('%.2f' % credit['0.85']), value_type['第三方'], sample_size
|
|
else:
|
|
return -2,float('%.2f' % credit['0.85']), value_type['第三方'], sample_size
|
|
def map_time_price(junjia,dglg,ppd3):
|
|
|
|
day_15=front_step_time(weekend,15)
|
|
day_30=front_step_time(weekend,30)
|
|
day_60=front_step_time(weekend,60)
|
|
price_15= consider_time_ss1(junjia,dglg,ppd3,day_15)
|
|
price_30= consider_time_ss1(junjia,dglg,ppd3,day_30)
|
|
price_60= consider_time_ss1(junjia,dglg,ppd3,day_60)
|
|
|
|
if price_15[0]>0:
|
|
#print('first_15')
|
|
return price_15
|
|
elif price_30[0]>0:
|
|
#print('first_30')
|
|
return price_30
|
|
elif price_60[0]>0:
|
|
#print('first_60')
|
|
return price_60
|
|
else:
|
|
return ss1(junjia,dglg,ppd3)
|
|
|
|
dggg=map_time_price(junjia,dglg,ppd3)
|
|
|
|
|
|
if len(junjia)>4 and dglg!=None:
|
|
sample_size = len(junjia)
|
|
return dglg, float('%.2f' % credit['0.88']), value_type['第三方'], sample_size
|
|
|
|
|
|
|
|
elif dggg[0]>0 :
|
|
return dggg
|
|
|
|
|
|
elif length > 3:
|
|
# #####print('ldf=',l_and_z_and_i)
|
|
pppk = ppd3.loc[l_and_z, :]
|
|
|
|
ppp = pppk[['total_price', 'p_date']]
|
|
# #####print(ppp)
|
|
ppp1 = ppp.sort_values(['p_date'], ascending=False)
|
|
|
|
gdf = ppp1['total_price'][:36].dropna()
|
|
##print('链家和诸葛找房合并后的数据集=',gdf)
|
|
sample_size = len(gdf)
|
|
gm = k_means_clus(gdf, 2)
|
|
######print('链猪= ', gm)
|
|
return int(gm), float('%.2f' % credit['0.8']), value_type['自身样本'], sample_size # 优质样本
|
|
|
|
|
|
elif data.shape[0] > 0:
|
|
data['avg'] = data['total_price'] / data['area']
|
|
|
|
def guolv_data11(data, neighb):
|
|
if type(neighb) is not str:
|
|
|
|
######print('周边价格=', neighb)
|
|
ddd11 = data[data['avg'] > 0.65 * neighb]
|
|
return ddd11
|
|
else:
|
|
sss=take_region_price(ct,rl)
|
|
if sss>0:
|
|
ddd22 = data[data['avg'] > 0.5 * sss]
|
|
return ddd22
|
|
else:
|
|
return data
|
|
|
|
ddd11 = guolv_data11(data, neighb[0])
|
|
|
|
#print('ddd11=',ddd11[['total_price','area','url']])
|
|
|
|
if ddd11.shape[0] > 2:
|
|
|
|
if ddd11.shape[0] > 3:
|
|
|
|
from zhuge import zhuge_consist
|
|
|
|
s = zhuge_consist.zhuge_out(ddd11, 3, 1)
|
|
|
|
# #####print('s=',s)
|
|
|
|
dd22=ddd11['total_price']/ddd11['area']
|
|
|
|
def compelex(ddd11, s, neighb, junjia):
|
|
#cc1 = ddd11['avg'].dropna()
|
|
|
|
# #####print('cc1=',cc1)
|
|
##print(ppd3['total_price'],'ppppppppppppppp')
|
|
##print('dd22=',dd22)
|
|
kmzhuge = k_means_clus(dd22.dropna(), 3)
|
|
##print(ppd3['total_price'])
|
|
junjia['zhuge_logic'] = kmzhuge
|
|
|
|
junjia['kmzg'] = kmzhuge
|
|
|
|
if type(neighb) != str and (neighb) != None:
|
|
junjia['neighb'] = neighb
|
|
# #####print('junjia',junjia)
|
|
|
|
junjia_front = junjia[['zhuge_logic', 'kmzg']]
|
|
|
|
def drop_ganji(junjia):
|
|
a = {'ganji_esf_price', 'anjuke_esf_price'}
|
|
# #####print('a=',a)
|
|
b = set(list(junjia.index))
|
|
# #####print('b=',b)
|
|
c = a & b
|
|
u=list(b-a)
|
|
if len(c) == 2 and len(junjia)>2:
|
|
gj=junjia['ganji_esf_price']
|
|
ajk=junjia['anjuke_esf_price']
|
|
ratio=abs((gj-ajk)/min(gj,ajk))
|
|
other=junjia[u[0]]
|
|
ott=(other-max(gj,ajk))/min(gj,ajk)
|
|
if ratio<0.1 and ott>0.3:
|
|
return junjia.drop('ganji_esf_price')
|
|
else:
|
|
return junjia
|
|
else:
|
|
return junjia
|
|
|
|
junjia1 = drop_ganji(junjia)
|
|
cc = group(junjia_front, junjia1, 1,10)
|
|
if len(cc) > 0:
|
|
cc1 = pandas.DataFrame(cc, columns=['price', 'count'])
|
|
cc2 = cc1.sort_values(by='count', ascending=False)
|
|
##print('cc2=',cc2)
|
|
|
|
def two_one_choose(cc2):
|
|
qwrt=set(cc2['count'])
|
|
|
|
if len(qwrt)>1:
|
|
return cc2['price'].iloc[0]
|
|
else :
|
|
return junjia['kmzg']
|
|
|
|
sample_size = ddd11.shape[0]
|
|
return two_one_choose(cc2) * 1.0, float('%.2f' % credit['0.7']), value_type[
|
|
'第三方'], sample_size
|
|
else:
|
|
##print('777777')
|
|
sample_size = ddd11.shape[0]
|
|
##print('junjia=',junjia)
|
|
return junjia['kmzg'] * 1.0, float('%.2f' % credit['0.6']), value_type[
|
|
'自身样本'], sample_size
|
|
##print('ddd11=',ddd11)
|
|
junjia_km = compelex(ddd11, s, neighb[0], junjia)
|
|
|
|
return junjia_km
|
|
|
|
|
|
|
|
else:
|
|
#####print('*' * 20, '采用周边算法')
|
|
if ddd11.shape[0] > 3:
|
|
cc1 = ddd11['avg'].dropna()
|
|
kmzhuge = k_means_clus(cc1, 2)
|
|
sample_size = ddd11.shape[0]
|
|
|
|
##print('#########')
|
|
return kmzhuge, float('%.2f' % credit['0.6']), value_type['自身样本'], sample_size
|
|
else:
|
|
return neighb[0], float('%.2f' % credit['0.5']), value_type['周边楼盘'], neighb[1]
|
|
|
|
elif dglg != None:
|
|
#####print('优质第三方均价2 = ', dglg)
|
|
sample_size = len(junjia)
|
|
return dglg, float('%.2f' % credit['0.8']), value_type['第三方'], sample_size # '优质样本'
|
|
|
|
else:
|
|
|
|
##print('lgggggg=')
|
|
|
|
##print('neighb[0]=',neighb[0])
|
|
return neighb[0], float('%.2f' % credit['0.5']), value_type['周边楼盘'], neighb[1]
|
|
|
|
elif dglg != None:
|
|
#####print('优质第三方均价3 = ', dglg)
|
|
sample_size = len(junjia)
|
|
return dglg, float('%.2f' % credit['0.8']), value_type['第三方'] # '优质样本'
|
|
|
|
else:
|
|
|
|
return neighb[0], float('%.2f' % credit['0.5']), value_type['周边楼盘'], neighb[1]
|
|
|
|
|
|
elif dglg != None:
|
|
##print('优质第三方均价4 = ', dglg)
|
|
##print('neighb[0]',neighb[0])
|
|
|
|
def dgneib(dglg,neighb):
|
|
|
|
if type(neighb[0])!=str:
|
|
|
|
if dglg/neighb[0]>0.75:
|
|
return dglg, float('%.2f' % credit['0.75']), value_type['第三方'], len(junjia) # '优质样本'
|
|
else:
|
|
return neighb[0], float('%.2f' % credit['0.5']), value_type['周边楼盘'], neighb[1]
|
|
else:
|
|
return dglg,float('%.2f' % credit['0.75']), value_type['第三方'], len(junjia) # '优质样本'
|
|
return dgneib(dglg,neighb)
|
|
|
|
|
|
else:
|
|
return neighb[0], float('%.2f' % credit['0.5']), value_type['周边楼盘'], neighb[1]
|
|
|
|
|
|
|
|
def add_alias(ppd):
|
|
if 'alias' not in ppd.columns:
|
|
ppd['alias'] = '无'
|
|
return ppd
|
|
|
|
|
|
def ghhh(kkg):
|
|
kkg = kkg.fillna('无')
|
|
a = list()
|
|
for i in kkg.index:
|
|
b = list(kkg.loc[[i], :].values.flatten())
|
|
a.append(b)
|
|
return a
|
|
|
|
|
|
stat = fangjia.get_collection("fangjia", "district_stat", config.get_district_stat_host(), config.get_district_stat_port())
|
|
#pt = fangjia.get_collection("fangjia_log", "calc_command", config.get_log_mongodb_host(), config.get_log_mongodb_port())
|
|
pt = fangjia.get_collection("fangjia_log", "problem_command", config.get_log_mongodb_host(), config.get_log_mongodb_port())
|
|
|
|
|
|
|
|
#stat = fangjia.get_collection("fangjia", "Copy_of_district_stat_shanghai_april_ten", '192.168.10.182', 27017)
|
|
#stat = fangjia.get_collection("fangjia", config.get_save_db(), '192.168.10.182', 27017)
|
|
#pt = fangjia.get_collection("fangjia_log", "problem_command", '192.168.10.182', 27017)
|
|
#################################################################
|
|
|
|
def region_district_list_lk(ct, rl, lk):
|
|
# import pandas
|
|
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
|
# db1 = client1.fangjia
|
|
# seaweed1 = db1.seaweed
|
|
|
|
query1 = {"status": 0, "cat": "district", "city": ct, "region": rl, "name": lk}
|
|
fields1 = {"lat2": 1, "lng2": 1, "city": 1, "region": 1,
|
|
"cat": 1, "name": 1, 'estate_type2': 1, 'alias': 1}
|
|
|
|
# lct= list(seaweed136.find(query1, fields1))
|
|
# lf=pandas.DataFrame(lct)
|
|
lf = fangjia.find_seaweed(query1, fields1)
|
|
return lf
|
|
|
|
|
|
def last_sunday(weekend):
|
|
|
|
######print('weekend_up',weekend)
|
|
# import datetime
|
|
|
|
last_sunday = weekend - datetime.timedelta(days=7)
|
|
#d1 = future_time.strftime('%Y%m%d')
|
|
#weekend1 = datetime.datetime.strptime(d1, '%Y%m%d')
|
|
#weekend = weekend1 + datetime.timedelta(hours=-8)
|
|
|
|
######print('weekend',last_sunday)
|
|
return last_sunday
|
|
|
|
|
|
# last_sunday=last_sunday(weekend)
|
|
|
|
def make_up_price(ct, rl, lk, weekend):
|
|
|
|
#####print('lg=',ct,rl,lk)
|
|
# last_sunday=last_sunday(weekend)南湖国际中心二期
|
|
# save_lg=client136.fangjia.district_stat
|
|
if lk != None:
|
|
|
|
# cc=last_sunday
|
|
|
|
ly = last_sunday(weekend)
|
|
# stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017)
|
|
#stat = stat
|
|
query = {"city": ct, "region": rl, "district_name": lk, 'weekend': ly
|
|
}
|
|
|
|
pos1 = list(stat.find(query).limit(30))
|
|
|
|
pos = pandas.DataFrame(pos1)
|
|
|
|
# pos=timestamp_to_int(pos)
|
|
# pos=pos[pos['week1']>cc-5]
|
|
# pos=pos.sort_values(by='week1',ascending=False)
|
|
|
|
# #####print('5'*10)
|
|
# #####print(pos)
|
|
|
|
if pos.shape[0] > 0:
|
|
# pp1"""
|
|
pp1 = pos['avg_price'].iloc[0]
|
|
if type(pp1) != None and type(pp1) != str:
|
|
|
|
return int(pp1)
|
|
else:
|
|
return None
|
|
else:
|
|
return None
|
|
|
|
else:
|
|
|
|
# stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017)
|
|
|
|
query = {"city": ct, "region": rl, "district_name": lk, 'weekend': last_sunday
|
|
}
|
|
|
|
pos1 = list(stat.find(query).limit(30))
|
|
|
|
pos = pandas.DataFrame(pos1)
|
|
|
|
# pos=timestamp_to_int(pos)
|
|
# pos=pos[pos['week1']>cc-5]
|
|
# pos=pos.sort_values(by='week1',ascending=False)
|
|
|
|
# #####print('5'*10)
|
|
# #####print(pos)
|
|
|
|
if pos.shape[0] > 0:
|
|
# pp1
|
|
pp1 = pos['avg_price'].iloc[0]
|
|
if type(pp1) != None and type(pp1) != str:
|
|
|
|
return int(pp1)
|
|
else:
|
|
return None
|
|
else:
|
|
return None
|
|
|
|
|
|
# weekend=weekend00()
|
|
|
|
def out_estate_type(estate_type):
|
|
if estate_type is not None and "别墅" in estate_type:
|
|
return '别墅','别墅'
|
|
else:
|
|
return '小区',"普通住宅"
|
|
|
|
@timer
|
|
def pt_down(ct,rl,p,weekend,c_date,cmd_estate_type,mm):
|
|
pt.replace_one(
|
|
|
|
{"city": ct, "region": rl, "name": p[0], "calc_date": weekend},
|
|
|
|
{"city": ct, "region": rl, "name": p[0], "c_date": c_date, 'status': int(3),
|
|
'remark': 'week', 'category': 'sell', "calc_date": weekend, 'priority': int(0),
|
|
'problem': mm[0], 'estate_type': cmd_estate_type, "calc_type": int(0)
|
|
}, True
|
|
)
|
|
|
|
@timer
|
|
def stat_save(ct, rl, p, weekend, mm, c_date, m_date, make_up, estate_type,avg):
|
|
stat.replace_one(
|
|
|
|
{"city": ct, "region": rl, "district_name": p[0], "weekend": weekend},
|
|
|
|
{"city": ct, "region": rl, "district_name": p[0], 'credit': mm[1],
|
|
"calc_value": int(mm[0]), "weekend": weekend, "c_date": c_date, 'avg_price':int(avg),
|
|
"m_date": m_date, 'value_type': int(mm[2]), 'weekend_markup': make_up,
|
|
'changed_markup': make_up, 'markup': make_up, 'sample_size': mm[3],
|
|
'estate_type': estate_type}, True
|
|
)
|
|
|
|
@timer
|
|
def pt_up(ct, rl, p, weekend, c_date, cmd_estate_type):
|
|
pt.replace_one(
|
|
|
|
{"city": ct, "region": rl, "name": p[0], "calc_date": weekend},
|
|
|
|
{"city": ct, "region": rl, "name": p[0], "c_date": c_date, 'status': int(102),
|
|
'remark': 'week', 'category': 'sell', "calc_date": weekend, 'priority': int(0),
|
|
'estate_type': cmd_estate_type, 'problem': None, "calc_type": int(0)
|
|
}, True
|
|
)
|
|
|
|
|
|
@timer
|
|
def out_put_price(ct, rl, lk, weekend):
|
|
lf = region_district_list_lk(ct, rl, lk)
|
|
|
|
if lf.shape[0]>0:
|
|
lf = add_alias(lf)
|
|
p = ghhh(lf[['name', 'alias', 'estate_type2']])[0]
|
|
|
|
######print('p=', p)
|
|
#t1=time.time()
|
|
mm = commonprice(ct, rl, p,weekend)
|
|
##print('mmjisuantime=',time.time()-t1,'yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy')
|
|
######print('mm=',mm)
|
|
c_date = datetime.datetime.utcnow()
|
|
m_date = datetime.datetime.utcnow()
|
|
estate_type = lf['estate_type2'].iloc[0]
|
|
|
|
|
|
######print('estate_type',estate_type)
|
|
|
|
|
|
estate_type,cmd_estate_type = out_estate_type(estate_type)
|
|
|
|
print('mm=',mm,'$'*20)
|
|
###print(type(mm[0]))
|
|
if type(mm[0]) != str:
|
|
|
|
|
|
# #####print(lf['estate_type2'].iloc[0],'*'*10)
|
|
|
|
|
|
# last_sunday=last_sunday()
|
|
# t1=time.time()
|
|
last_sunday_price = make_up_price(ct, rl, lk, weekend)
|
|
|
|
# #####print(time.time()-t1,'*'*20)
|
|
|
|
#print('last_sunday_price=',last_sunday_price)
|
|
# #####print('mm[0]',mm[0])
|
|
|
|
def make_up(avg,last_sunday_price):
|
|
if type(last_sunday_price) != str and last_sunday_price != None and mm[0] != None and type(mm[0]) != str:
|
|
|
|
make_up = 10000 * (avg - last_sunday_price) / last_sunday_price
|
|
return make_up
|
|
else:
|
|
return 0
|
|
|
|
|
|
|
|
######print('make_up',make_up,'*'*20)
|
|
######print('credic',mm[1])
|
|
|
|
def avg_compare(ct,rl,lk,mm,weekend):
|
|
avg=fix.price_fix.update(ct,rl,lk,int(mm[0]),weekend)
|
|
##print('nihe=',avg)
|
|
##print('mm[0]',mm[0])
|
|
if avg>0:
|
|
return avg
|
|
else:
|
|
return int(mm[0])
|
|
#s1=time.time()
|
|
|
|
|
|
avg=avg_compare(ct,rl,lk,mm,weekend)
|
|
|
|
#avg=int(mm[0])
|
|
|
|
|
|
make_up = int(make_up(avg,last_sunday_price))
|
|
##print('s1_time=',time.time()-s1)
|
|
#avg=mm[0]
|
|
#print('mm=',mm)
|
|
#print(ct, rl, lk, avg, 'avg=',avg,'$' * 20)
|
|
try:
|
|
|
|
stat_save(ct,rl,p,weekend,mm,c_date,m_date,make_up,estate_type,avg)
|
|
#pt_up(ct,rl,p,weekend,c_date,cmd_estate_type)
|
|
|
|
'''
|
|
stat.replace_one(
|
|
|
|
{"city": ct, "region": rl, "district_name": p[0], "weekend": weekend},
|
|
|
|
{"city": ct, "region": rl, "district_name": p[0], 'credit': mm[1],
|
|
"calc_value": int(mm[0]), "weekend": weekend, "c_date": c_date, 'avg_price': int(mm[0]),
|
|
"m_date": m_date, 'value_type': int(mm[2]), 'weekend_markup': make_up,
|
|
'changed_markup': make_up, 'markup': make_up, 'sample_size': mm[3],
|
|
'estate_type': estate_type}, True
|
|
)
|
|
|
|
|
|
|
|
pt.replace_one(
|
|
|
|
{"city": ct, "region": rl, "name": p[0], "calc_date": weekend},
|
|
|
|
{"city": ct, "region": rl, "name": p[0], "c_date": c_date, 'status': int(102),
|
|
'remark': 'week', 'category': 'sell', "calc_date": weekend, 'priority': int(0),
|
|
'estate_type': cmd_estate_type, 'problem': None, "calc_type": int(0)
|
|
}, True
|
|
)
|
|
'''
|
|
|
|
|
|
except:
|
|
stat_save(ct,rl,p,weekend,mm,c_date,m_date,make_up,estate_type,avg)
|
|
|
|
|
|
|
|
|
|
else:
|
|
|
|
pt_down(ct,rl,p,weekend,c_date,cmd_estate_type,mm)
|
|
'''
|
|
#
|
|
pt.replace_one(
|
|
|
|
{"city": ct, "region": rl, "name": p[0], "calc_date": weekend},
|
|
|
|
{"city": ct, "region": rl, "name": p[0], "c_date": c_date, 'status': int(3),
|
|
'remark': 'week', 'category': 'sell', "calc_date": weekend, 'priority': int(0),
|
|
'problem': mm[0], 'estate_type': cmd_estate_type, "calc_type": int(0)
|
|
}, True
|
|
)
|
|
'''
|
|
|
|
|