From 4d91003f726fe65af864ac2eb0c2879e51e4fba8 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Wed, 7 Mar 2018 13:13:14 +0800 Subject: [PATCH] =?UTF-8?q?=E7=A7=BB=E9=99=A4=E6=97=A0=E7=94=A8=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 + PriceSystem/CalcSpeedMonitor.py | 5 +- PriceSystem/config/__init__.py | 4 + PriceSystem/fangjia/__init__.py | 4 +- PriceSystem/test.py | 15 +- PriceSystem/zhuge0118A.py | 1973 +++++++++++++++---------------- 6 files changed, 960 insertions(+), 1045 deletions(-) diff --git a/.gitignore b/.gitignore index d4f4c0c..0eb2882 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ /PriceSystem/config/__pycache__ /PriceSystem/PriceSystem/.idea +*/__pycache__ .idea +/PriceSystem/fangjia/__pycache__ +/PriceSystem/sell_price/__pycache__ +/PriceSystem/zhuge/__pycache__ diff --git a/PriceSystem/CalcSpeedMonitor.py b/PriceSystem/CalcSpeedMonitor.py index 996b309..9d3cfbd 100644 --- a/PriceSystem/CalcSpeedMonitor.py +++ b/PriceSystem/CalcSpeedMonitor.py @@ -2,9 +2,10 @@ import json import multiprocessing import time import redis +import config QUEUE = "calc_cmd" -redis_pool = redis.ConnectionPool(host='192.168.0.144', port=6379, db=0) +redis_pool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=0) def monitor(pname): @@ -12,7 +13,7 @@ def monitor(pname): while True: pre_counter = client.llen(QUEUE) time.sleep(60) - print(pname + "#####################################:".zfill(100) + str( + print(pname + "##################:".zfill(20) + str( pre_counter - client.llen(QUEUE)) + "/min") diff --git a/PriceSystem/config/__init__.py b/PriceSystem/config/__init__.py index 63c6053..ffdd97d 100644 --- a/PriceSystem/config/__init__.py +++ b/PriceSystem/config/__init__.py @@ -33,3 +33,7 @@ def get_log_mongodb_port(): def get_timeout(): return 0.3 + + +def get_seawater_host(): + return "seawater.db.fangjia.com" diff --git a/PriceSystem/fangjia/__init__.py b/PriceSystem/fangjia/__init__.py index debe270..6f0a154 100644 --- a/PriceSystem/fangjia/__init__.py +++ b/PriceSystem/fangjia/__init__.py @@ -66,7 +66,7 @@ def get_seawater_index_name(city, region): @timer def findHouse(query, fields=None): # s = time.time() - seawater105 = get_collection("fangjia", "seawater", "192.168.0.105", 27777) + seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777) for err_count in range(0, 10): try: @@ -135,7 +135,7 @@ def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, @timer def find_sample_pool(query, fields=None): # s = time.time() - seawater105 = get_collection("fangjia", "seawater", "192.168.0.105 ", 27777) + seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777) for err_count in range(0, 10): try: diff --git a/PriceSystem/test.py b/PriceSystem/test.py index 4d09d21..f256869 100644 --- a/PriceSystem/test.py +++ b/PriceSystem/test.py @@ -1,12 +1,13 @@ import zhuge0118A import time +import datetime + s = time.time() -ct='上海' -rl='奉贤' -lk='杨王村公寓' +ct = '上海' +rl = '奉贤' +lk = '杨王村公寓' +weekend = datetime.datetime.strptime("2018-03-11", "%Y-%m-%d") - -zhuge0118A.out_put_price(ct, rl, lk) -zhuge0118A.out_put_price(ct, rl, lk) +zhuge0118A.out_put_price(ct, rl, lk, weekend) e = time.time() -print(e -s) +print(e - s) diff --git a/PriceSystem/zhuge0118A.py b/PriceSystem/zhuge0118A.py index f0a5870..880a240 100644 --- a/PriceSystem/zhuge0118A.py +++ b/PriceSystem/zhuge0118A.py @@ -1,4 +1,4 @@ - # -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- """ Created on Tue Jan 2 13:24:07 2018 @@ -13,46 +13,39 @@ Created on Fri Sep 22 18:18:15 2017 """ import os -print( os.getcwd()) -import pymongo -import numpy as np -import pandas as pd import random -import collections -import pymongo +import collections +import pymongo import sklearn.cluster import pandas import numpy -import sys, time -import pp -import zhuge +import time import fangjia -import sklearn.svm +import sklearn.svm import datetime -#import yesterday + +# import yesterday def front_step_time(day): now = datetime.datetime.now() - front=now-datetime.timedelta(days=day) - d1= front.strftime('%Y%m%d') + front = now - datetime.timedelta(days=day) + d1 = front.strftime('%Y%m%d') return int(d1) +# 提取房源的数据时间上限,如从20170301之后的数据 +some_district_information_time = front_step_time(360) -#提取房源的数据时间上限,如从20170301之后的数据 -some_district_information_time=front_step_time(360) +# 提周边小区均价的数据时间上限,如从20170801之后的数据 +zhoubian_xiaoqu_time = front_step_time(200) -#提周边小区均价的数据时间上限,如从20170801之后的数据 -zhoubian_xiaoqu_time=front_step_time(200) - - -#第三方均价的数据时间上限,如从201703之后的数据 -dsfjujia_time=front_step_time(430)//100 +# 第三方均价的数据时间上限,如从201703之后的数据 +dsfjujia_time = front_step_time(430) // 100 ############################################################ ############################################################ -#存到数据库的日期 +# 存到数据库的日期 """ @@ -78,568 +71,538 @@ def weekend00(): """ - -#date="2018-02-04" +# date="2018-02-04" ############################################################ ############################################################ -credit=pandas.Series([0.9,0.8,0.7,0.6,0.5],index=['0.9','0.8','0.7','0.6','0.5']) +credit = pandas.Series([0.9, 0.8, 0.7, 0.6, 0.5], index=['0.9', '0.8', '0.7', '0.6', '0.5']) -value_type=pandas.Series([1,2,3,4,5,6,7], - index=['指导价','自身样本','第三方','交易数据','周边楼盘','推导','优质样本']) +value_type = pandas.Series([1, 2, 3, 4, 5, 6, 7], + index=['指导价', '自身样本', '第三方', '交易数据', '周边楼盘', '推导', '优质样本']) -#1:指导价 2:自身样本 3:第三方 4:交易数据 5:周边楼盘, 6:推导 - -def commonprice(ct,rl,p): - +# 1:指导价 2:自身样本 3:第三方 4:交易数据 5:周边楼盘, 6:推导 - + +def commonprice(ct, rl, p): def add_block(ppd): if 'block' not in ppd.columns: - ppd['block']='没有' - return ppd + ppd['block'] = '没有' + return ppd def add_room(ppd): if 'room' not in ppd.columns: - ppd['room']= 2 + ppd['room'] = 2 return ppd - - def some_district_information(ct,rl,lk): - - - query = {"city":ct,"cat":"sell","region":rl, - "district_name":{"$in":lk}, - "p_date":{"$gt":some_district_information_time}} - + def some_district_information(ct, rl, lk): + + query = {"city": ct, "cat": "sell", "region": rl, + "district_name": {"$in": lk}, + "p_date": {"$gt": some_district_information_time}} + return fangjia.findHouse(query) """ return fangjia.find_house_from_elastic_search(ct, rl, lk, "lianjia.com", some_district_information_time) - """ - - def disanganf_price(ct,rl,lk,tt): - #db1 = client1.fangjia - #dsf = db1.average_statistics - query = {"city":ct,"region":rl, - "name":lk,"s_date":{"$gt":tt}} - #pos = list(dsf136.find(query)) - - #data1=pandas.DataFrame(pos) - data1=fangjia.find_disanganf_price(query) - - if data1.shape[0]>1: - a_set={'fangjia_price'} - b_set=set(data1.columns) - if a_set & b_set==a_set: - - data=data1[data1['credit']>0.7] - if data.shape[0]>=1: - choose_class=['city','region','name','fangjia_price','credit', - 's_date'] - dc=data[choose_class] + """ + + def disanganf_price(ct, rl, lk, tt): + # db1 = client1.fangjia + # dsf = db1.average_statistics + query = {"city": ct, "region": rl, + "name": lk, "s_date": {"$gt": tt}} + # pos = list(dsf136.find(query)) + + # data1=pandas.DataFrame(pos) + data1 = fangjia.find_disanganf_price(query) + + if data1.shape[0] > 1: + a_set = {'fangjia_price'} + b_set = set(data1.columns) + if a_set & b_set == a_set: + + data = data1[data1['credit'] > 0.7] + if data.shape[0] >= 1: + choose_class = ['city', 'region', 'name', 'fangjia_price', 'credit', + 's_date'] + dc = data[choose_class] return dc else: return numpy.array([]) else: - + return numpy.array([]) else: return numpy.array([]) - - - - def take_location(ct,rl,p): - - dirlt=p[0] - - #client1 = pymongo.MongoClient('192.168.0.136',27017) - #db1 = client1.fangjia - #seaweed1 = db1.seaweed - seaweed136=fangjia.get_collection("fangjia","seaweed", '192.168.0.136',27017) - query1 = {"status":0,"cat":"district","city":ct, - "region":rl, "name":dirlt} - + def take_location(ct, rl, p): + + dirlt = p[0] + + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # seaweed1 = db1.seaweed + seaweed136 = fangjia.get_collection("fangjia", "seaweed", '192.168.0.136', 27017) + query1 = {"status": 0, "cat": "district", "city": ct, + "region": rl, "name": dirlt} + for err_count in range(0, 10): - - try: - ltt= seaweed136.count(query1) - if ltt>0: - fields1 = {"lat2":1,"lng2":1, "city":1,"region":1,"cat":1,"name":1} - lct= seaweed136.find(query1, fields1) - lf=pandas.DataFrame(list(lct)) - - #lf=fangjia.find_seaweed(query1, fields1) - le=lf - le.index=le['name'] - cluns=lf.columns - if ('lng2' in cluns )and ('lat2' in cluns): - lr=le[['lng2','lat2']] - del lf,le,cluns + + try: + ltt = seaweed136.count(query1) + if ltt > 0: + fields1 = {"lat2": 1, "lng2": 1, "city": 1, "region": 1, "cat": 1, "name": 1} + lct = seaweed136.find(query1, fields1) + lf = pandas.DataFrame(list(lct)) + + # lf=fangjia.find_seaweed(query1, fields1) + le = lf + le.index = le['name'] + cluns = lf.columns + if ('lng2' in cluns) and ('lat2' in cluns): + lr = le[['lng2', 'lat2']] + del lf, le, cluns return lr else: - del lf,le,cluns + del lf, le, cluns return numpy.array([]) - + else: return numpy.array([]) except: - - ltt= seaweed136.count(query1) - if ltt>0: - fields1 = {"lat2":1,"lng2":1, "city":1,"region":1,"cat":1,"name":1} - lct= seaweed136.find(query1, fields1) - lf=pandas.DataFrame(list(lct)) - - #lf=fangjia.find_seaweed(query1, fields1) - le=lf - le.index=le['name'] - cluns=lf.columns - if ('lng2' in cluns )and ('lat2' in cluns): - lr=le[['lng2','lat2']] - del lf,le,cluns + + ltt = seaweed136.count(query1) + if ltt > 0: + fields1 = {"lat2": 1, "lng2": 1, "city": 1, "region": 1, "cat": 1, "name": 1} + lct = seaweed136.find(query1, fields1) + lf = pandas.DataFrame(list(lct)) + + # lf=fangjia.find_seaweed(query1, fields1) + le = lf + le.index = le['name'] + cluns = lf.columns + if ('lng2' in cluns) and ('lat2' in cluns): + lr = le[['lng2', 'lat2']] + del lf, le, cluns return lr else: - del lf,le,cluns + del lf, le, cluns return numpy.array([]) - + else: return numpy.array([]) - + def lianjia(ur): - #print(ur) + # print(ur) if 'lianjia' in ur: return 0 - else: + else: return 1 - - + def zhuge(ur): if 'zhuge' in ur: return 0 - else: + else: return 1 - + def wawj(ur): if '5i5j' in ur: return 0 - else: + else: return 1 - + def iwjw(ur): if 'iwjw' in ur: return 0 - else: + else: return 1 - + def fangdd(ur): if 'fangdd' in ur: return 0 - else: + else: return 1 - + def wkzf(ur): if 'wkzf' in ur: return 0 - else: - return 1 - - - - def label(dd,f): - aa=list() - + else: + return 1 + + def label(dd, f): + aa = list() + for i in dd.index: - d2f=dd['url'][i] - if type(d2f)==str: - if f(dd['url'][i])==0: - aa.append(i) + d2f = dd['url'][i] + if type(d2f) == str: + if f(dd['url'][i]) == 0: + aa.append(i) return set(aa) - - - + def new_data(ds): - bb=set(ds['name']) - bb1=list(bb) - x1=pandas.Series() + bb = set(ds['name']) + bb1 = list(bb) + x1 = pandas.Series() for i in bb1: - a1=ds[ds['name']==i] - a2=a1.sort_values(by='s_date',ascending=False) - x1[i]=a2.iloc[0]['fangjia_price'] + a1 = ds[ds['name'] == i] + a2 = a1.sort_values(by='s_date', ascending=False) + x1[i] = a2.iloc[0]['fangjia_price'] return x1 + def zhoubian_xiaoqu(ct, rl, dirst): - def zhoubian_xiaoqu(ct,rl,dirst): - - #提取周边小聚的均价,本方法不用房源,直接用上一批数据的价格来计算无房源小区 + # 提取周边小聚的均价,本方法不用房源,直接用上一批数据的价格来计算无房源小区 ################### ################### - tii=zhoubian_xiaoqu_time + tii = zhoubian_xiaoqu_time ################### ################### - - tt=tii//100-1 - #client1 = pymongo.MongoClient('192.168.0.136',27017) - #db1 = client1.fangjia - #dsf = db1.average_statistics - query = {"city":ct,"region":rl, - "name":{"$in":dirst},"s_date":{"$gt":tt}} - #pos = list( dsf136.find(query)) - - #data=pandas.DataFrame(pos) - data=fangjia.find_disanganf_price(query) - #print(data) - if data.shape[0]>=1: - #data=add_block(data) - data=add_room(data) - - a_set={'city','region','name','fangjia_price','credit', - 's_date'} - b_set=set(data.columns) - - if a_set & b_set==a_set: - - choose_class=['city','region','name','fangjia_price','credit', - 's_date'] - dc=data[choose_class] + + tt = tii // 100 - 1 + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # dsf = db1.average_statistics + query = {"city": ct, "region": rl, + "name": {"$in": dirst}, "s_date": {"$gt": tt}} + # pos = list( dsf136.find(query)) + + # data=pandas.DataFrame(pos) + data = fangjia.find_disanganf_price(query) + # print(data) + if data.shape[0] >= 1: + # data=add_block(data) + data = add_room(data) + + a_set = {'city', 'region', 'name', 'fangjia_price', 'credit', + 's_date'} + b_set = set(data.columns) + + if a_set & b_set == a_set: + + choose_class = ['city', 'region', 'name', 'fangjia_price', 'credit', + 's_date'] + dc = data[choose_class] return dc else: return numpy.array([]) return numpy.array([]) else: - + return numpy.array([]) - - def hh(ppp,df): - ppp['price']=None - aa=list(df.index) - #print('aa=',aa) + def hh(ppp, df): + ppp['price'] = None + aa = list(df.index) + # print('aa=',aa) for i in ppp.index: - kk=ppp['district_name'][i] - #print('kk=',kk) + kk = ppp['district_name'][i] + # print('kk=',kk) if kk in aa: - #ppp['price'][i]=df[kk] - ppp.loc[i,'price'] = df[kk] - #print('df[kk]',df[kk]) + # ppp['price'][i]=df[kk] + ppp.loc[i, 'price'] = df[kk] + # print('df[kk]',df[kk]) return ppp - - - def neighbor(lng,lat,scale,ct,rl,p): + def neighbor(lng, lat, scale, ct, rl, p): ################### ################### - #timm=20171122 + # timm=20171122 ################### ################### - estype=p[2] - #client1 = pymongo.MongoClient('192.168.0.136',27017) - #db1 = client1.fangjia - #seaweed1 = db1.seaweed - kkk=scale/1000 - longtitude=lng - latitude=lat - - query1 = {"status":0,"cat":"district", - "lat2":{"$gt":latitude-kkk*10**10/0.86,"$lt":latitude+kkk*10**10/0.86}, - "lng2":{"$gt":longtitude-kkk*10**10,"$lt":longtitude+kkk*10**10}} - #fields1 = {"lat2":1,"lng2":1, "city":1,"region":1,"cat":1,"name":1} - - fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, - "cat":1,"name":1,'estate_type2':1} - - #lct= list( seaweed136.find(query1, fields1)) - - #lf=pandas.DataFrame(lct) - lf=fangjia.find_seaweed(query1, fields1) - - le=le=lf[lf['estate_type2']==estype] - - le.index=le['name'] - - lr=le[['lng2','lat2']] - - - dirst1=list(lr.index) - + estype = p[2] + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # seaweed1 = db1.seaweed + kkk = scale / 1000 + longtitude = lng + latitude = lat + + query1 = {"status": 0, "cat": "district", + "lat2": {"$gt": latitude - kkk * 10 ** 10 / 0.86, "$lt": latitude + kkk * 10 ** 10 / 0.86}, + "lng2": {"$gt": longtitude - kkk * 10 ** 10, "$lt": longtitude + kkk * 10 ** 10}} + # fields1 = {"lat2":1,"lng2":1, "city":1,"region":1,"cat":1,"name":1} + + fields1 = {"lat2": 1, "lng2": 1, "city": 1, "region": 1, + "cat": 1, "name": 1, 'estate_type2': 1} + + # lct= list( seaweed136.find(query1, fields1)) + + # lf=pandas.DataFrame(lct) + lf = fangjia.find_seaweed(query1, fields1) + + le = le = lf[lf['estate_type2'] == estype] + + le.index = le['name'] + + lr = le[['lng2', 'lat2']] + + dirst1 = list(lr.index) + random.shuffle(dirst1) - - dirst=dirst1[:80] - #print('dirst=',dirst) - - dirst_plus=dirst+[p[0]] - - query_feature = {"city":ct,"region":rl, - "district_name":{"$in":dirst_plus}} - - ppp=fangjia.dis_feature(query_feature) - #print('ox'*10,ppp) - - #print('小区个数= ',len(dirst)) - df1=zhoubian_xiaoqu(ct,rl,dirst) - #print('df1=',df1,'*'*20) - if df1.shape[0]>1: - df=new_data(df1) - #print('df=',df) - vv=hh(ppp,df) - #print('#'*36) - #print('vv=',vv) + + dirst = dirst1[:80] + # print('dirst=',dirst) + + dirst_plus = dirst + [p[0]] + + query_feature = {"city": ct, "region": rl, + "district_name": {"$in": dirst_plus}} + + ppp = fangjia.dis_feature(query_feature) + # print('ox'*10,ppp) + + # print('小区个数= ',len(dirst)) + df1 = zhoubian_xiaoqu(ct, rl, dirst) + # print('df1=',df1,'*'*20) + if df1.shape[0] > 1: + df = new_data(df1) + # print('df=',df) + vv = hh(ppp, df) + # print('#'*36) + # print('vv=',vv) return vv else: return numpy.array([]) + def svr(vv, lk): + # 提取要计算小区的特征 + vk = vv[vv['district_name'] == lk] + # 删除要计算小区的空特征 + # print('vk=',vk) + if vk.shape[0] > 0: - def svr(vv,lk): - #提取要计算小区的特征 - vk=vv[vv['district_name']==lk] - #删除要计算小区的空特征 - #print('vk=',vk) - if vk.shape[0]>0: - - vk1=vk.iloc[0].dropna() - - #将要计算小区的特征提取出来 - - vk2=list( set(vk1.index)-{'_id','district_name','region','city'}) - - #print('vk2=',vk2) - if len(vk2)>1: - vvo=vv[vv['district_name']!=lk] - vk3=vk2+['price'] - vk31=list(set(vk3)) - vvo1=vvo[vk31].dropna() - - x1 =vvo1.drop('price',axis=1) - #print('x1=',x1) + vk1 = vk.iloc[0].dropna() + + # 将要计算小区的特征提取出来 + + vk2 = list(set(vk1.index) - {'_id', 'district_name', 'region', 'city'}) + + # print('vk2=',vk2) + if len(vk2) > 1: + vvo = vv[vv['district_name'] != lk] + vk3 = vk2 + ['price'] + vk31 = list(set(vk3)) + vvo1 = vvo[vk31].dropna() + + x1 = vvo1.drop('price', axis=1) + # print('x1=',x1) y = vvo1['price'] - #print('y',y) - - if len(x1)>2: - - clf = sklearn.svm.SVR() - clf.fit(x1, y) - pp1= vk[vk31].drop('price',axis=1) - result = clf.predict(pp1 ) - #print (result) + # print('y',y) + + if len(x1) > 2: + + clf = sklearn.svm.SVR() + clf.fit(x1, y) + pp1 = vk[vk31].drop('price', axis=1) + result = clf.predict(pp1) + # print (result) return int(result[0]) else: - km=vv['price'].dropna() + km = vv['price'].dropna() return int(k_means(km)) - - + + else: - km=vv['price'].dropna() + km = vv['price'].dropna() return int(k_means(km)) - + else: - km=vv['price'].dropna() - return int(k_means(km)) - - - - def no_source_price(ct,rl,location,p): - lk=p[0] - hg=location.shape[0] - - if hg==0: - return '没有经纬度数据',0 - elif hg>1: - return '纬度数据有误,有多个经纬度',0 + km = vv['price'].dropna() + return int(k_means(km)) + + def no_source_price(ct, rl, location, p): + lk = p[0] + hg = location.shape[0] + + if hg == 0: + return '没有经纬度数据', 0 + elif hg > 1: + return '纬度数据有误,有多个经纬度', 0 else: - #t1=time.time() - dd=neighbor(location['lng2'][0],location['lat2'][0],5,ct,rl,p) - #print(time.time()-t1,'t1'*10) - #print('dd=',dd,'dd'*20) - if dd.shape[0]>=20: - #print('$'*50) - #dd1=neighbor(location['lng2'][0],location['lat2'][0],5) - #print('dd',dd) - #svr_time=time.time() - other=svr(dd,lk) - #print(time.time()-svr_time,'svr'*10) - print('other= ' ,other,'9'*20) - return other,dd.shape[0] + # t1=time.time() + dd = neighbor(location['lng2'][0], location['lat2'][0], 5, ct, rl, p) + # print(time.time()-t1,'t1'*10) + # print('dd=',dd,'dd'*20) + if dd.shape[0] >= 20: + # print('$'*50) + # dd1=neighbor(location['lng2'][0],location['lat2'][0],5) + # print('dd',dd) + # svr_time=time.time() + other = svr(dd, lk) + # print(time.time()-svr_time,'svr'*10) + print('other= ', other, '9' * 20) + return other, dd.shape[0] else: - #t2=time.time() - dd1=neighbor(location['lng2'][0],location['lat2'][0],10,ct,rl,p) - #print(time.time()-t2,'t2'*10) - #print('dd1=',dd1) - if dd1.shape[0]>=20: - #print('$'*50) - - - #svr_time=time.time() - other1=svr(dd1,lk) - #print(time.time()-svr_time,'svr'*10) - #print('other1=',other1) - return other1,dd1.shape[0] - else: - #t3=time.time() - dd2=neighbor(location['lng2'][0],location['lat2'][0],50,ct,rl,p) - #print(time.time()-t3,'t3'*10) - print('dd2.shape',dd2.shape[0],'@'*60) - if dd2.shape[0]>=20: - print('$'*50) - svr_time=time.time() - other1=svr(dd2,lk) - print(time.time()-svr_time,'e'*10) - #print('other1=',other1) - return other1,dd2.shape[0] - else: - dd3=neighbor(location['lng2'][0],location['lat2'][0],100,ct,rl,p) - #print('dd2=',dd2,'@'*60) - if dd3.shape[0]>=20: - #svr_time=time.time() - other1=svr(dd3,lk) - #print(time.time()-svr_time,'svr'*10) - #print('other1=',other1) - return other1,dd3.shape[0] - else: - - return '没有周围小区数据' ,0 + # t2=time.time() + dd1 = neighbor(location['lng2'][0], location['lat2'][0], 10, ct, rl, p) + # print(time.time()-t2,'t2'*10) + # print('dd1=',dd1) + if dd1.shape[0] >= 20: + # print('$'*50) - - def region_district_list(ct,rl): - #import pandas - #client1 = pymongo.MongoClient('192.168.0.136',27017) - #db1 = client1.fangjia - #seaweed1 = db1.seaweed - - query1 = {"status":0,"cat":"district","city":ct,"region":rl} - fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, - "cat":1,"name":1,'estate_type2':1,'alias':1} - - #lct= list(seaweed136.find(query1, fields1)) - #lf=pandas.DataFrame(lct) - lf=fangjia.find_seaweed(query1, fields1) + # svr_time=time.time() + other1 = svr(dd1, lk) + # print(time.time()-svr_time,'svr'*10) + # print('other1=',other1) + return other1, dd1.shape[0] + else: + # t3=time.time() + dd2 = neighbor(location['lng2'][0], location['lat2'][0], 50, ct, rl, p) + # print(time.time()-t3,'t3'*10) + print('dd2.shape', dd2.shape[0], '@' * 60) + if dd2.shape[0] >= 20: + print('$' * 50) + svr_time = time.time() + other1 = svr(dd2, lk) + print(time.time() - svr_time, 'e' * 10) + # print('other1=',other1) + return other1, dd2.shape[0] + else: + dd3 = neighbor(location['lng2'][0], location['lat2'][0], 100, ct, rl, p) + # print('dd2=',dd2,'@'*60) + if dd3.shape[0] >= 20: + # svr_time=time.time() + other1 = svr(dd3, lk) + # print(time.time()-svr_time,'svr'*10) + # print('other1=',other1) + return other1, dd3.shape[0] + else: + + return '没有周围小区数据', 0 + + def region_district_list(ct, rl): + # import pandas + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # seaweed1 = db1.seaweed + + query1 = {"status": 0, "cat": "district", "city": ct, "region": rl} + fields1 = {"lat2": 1, "lng2": 1, "city": 1, "region": 1, + "cat": 1, "name": 1, 'estate_type2': 1, 'alias': 1} + + # lct= list(seaweed136.find(query1, fields1)) + # lf=pandas.DataFrame(lct) + lf = fangjia.find_seaweed(query1, fields1) return lf - - - def k_means(pp1): from sklearn.cluster import KMeans - pv=list(pp1) - if len(set(pv))>3: - gf=numpy.array([pv]).T - estimator = KMeans(n_clusters=3)#构造聚类器 - estimator.fit(gf)#聚类 - label_pred = estimator.labels_ #获取聚类标签 - aa=collections.Counter(label_pred) - v=pandas.Series(aa) - gg=list(v) - index_max=gg.index(max(gg)) - - centroids = estimator.cluster_centers_ #获取聚类中心 - #inertia = estimator.inertia_ # 获取聚类准则的总和 - center=centroids[index_max][0] + pv = list(pp1) + if len(set(pv)) > 3: + gf = numpy.array([pv]).T + estimator = KMeans(n_clusters=3) # 构造聚类器 + estimator.fit(gf) # 聚类 + label_pred = estimator.labels_ # 获取聚类标签 + aa = collections.Counter(label_pred) + v = pandas.Series(aa) + gg = list(v) + index_max = gg.index(max(gg)) + + centroids = estimator.cluster_centers_ # 获取聚类中心 + # inertia = estimator.inertia_ # 获取聚类准则的总和 + center = centroids[index_max][0] return (int(center)) else: return int(pp1.mean()) - - def region_district_list_name(ct,rl,lkk): - #import pandas - #client1 = pymongo.MongoClient('192.168.0.136',27017) - #db1 = client1.fangjia - #seaweed1 = db1.seaweed - - query1 = {"status":0,"cat":"district","city":ct,"region":rl,"name":lkk} - fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, - "cat":1,"name":1,'estate_type2':1,'alias':1} - - #lct= list( seaweed136.find(query1, fields1)) - - - #lf=pandas.DataFrame(lct) - lf=fangjia.find_seaweed(query1, fields1 ) + def region_district_list_name(ct, rl, lkk): + # import pandas + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # seaweed1 = db1.seaweed + + query1 = {"status": 0, "cat": "district", "city": ct, "region": rl, "name": lkk} + fields1 = {"lat2": 1, "lng2": 1, "city": 1, "region": 1, + "cat": 1, "name": 1, 'estate_type2': 1, 'alias': 1} + + # lct= list( seaweed136.find(query1, fields1)) + + # lf=pandas.DataFrame(lct) + lf = fangjia.find_seaweed(query1, fields1) return lf - - - def strip_villa(ct,rl,p): - lw=p[0] - #estype=p[2] - alias=p[1] - - #print('alias=',alias) - + + def strip_villa(ct, rl, p): + lw = p[0] + # estype=p[2] + alias = p[1] + + # print('alias=',alias) + def filt(x): - - return type(x)==str and len(x)>1 - - bb=list(filter(filt,alias)) - #print('bb=',bb) + + return type(x) == str and len(x) > 1 + + bb = list(filter(filt, alias)) + # print('bb=',bb) bb.extend([lw]) - #lk1=list(set(lk)) - - #print('本小区名=',lw) - - #print('type(lw)=',type(bb)) - - #print('小区名列表=',bb) - - cc1= some_district_information(ct,rl,bb) - - cc1= add_room(cc1) - - print('cc1本小区数据条数=',cc1.shape[0]) - - if cc1.shape[0]>2: - - del lw,alias,bb + # lk1=list(set(lk)) + + # print('本小区名=',lw) + + # print('type(lw)=',type(bb)) + + # print('小区名列表=',bb) + + cc1 = some_district_information(ct, rl, bb) + + cc1 = add_room(cc1) + + print('cc1本小区数据条数=', cc1.shape[0]) + + if cc1.shape[0] > 2: + + del lw, alias, bb return cc1 else: - + if '别墅' in lw: - - lkk=lw.strip('别墅') - cc=region_district_list_name(ct,rl,lkk) - #print('cc=',cc) - if cc.shape[0]>1: - - cc1=list(cc['alias'].dropna()) - cc2=list(filter(filt,alias)) + + lkk = lw.strip('别墅') + cc = region_district_list_name(ct, rl, lkk) + # print('cc=',cc) + if cc.shape[0] > 1: + + cc1 = list(cc['alias'].dropna()) + cc2 = list(filter(filt, alias)) bb.extend([lkk]) - #clk1=list(set(clk)) - cc2=some_district_information(ct,rl,bb) - if cc2.shape[0]>2: - del lw,alias,bb,cc1,cc + # clk1=list(set(clk)) + cc2 = some_district_information(ct, rl, bb) + if cc2.shape[0] > 2: + del lw, alias, bb, cc1, cc return cc2 else: - del lw,alias,bb,cc1,cc - return cc1 + del lw, alias, bb, cc1, cc + return cc1 else: return cc1 else: return cc1 - - def consider_villa(ct,rl,p): - - #print('villa p=',p) - - estype=p[2] - df=strip_villa(ct,rl,p) - if df.shape[0]>1: - - if estype=='普通住宅': - all_index=set(df.index) - vv=fetch_villa_index(df) - ww=all_index-vv - - d1=df.loc[ww,:] - if d1.shape[0]>3: + def consider_villa(ct, rl, p): + + # print('villa p=',p) + + estype = p[2] + df = strip_villa(ct, rl, p) + if df.shape[0] > 1: + + if estype == '普通住宅': + all_index = set(df.index) + vv = fetch_villa_index(df) + ww = all_index - vv + + d1 = df.loc[ww, :] + if d1.shape[0] > 3: return d1 else: return df - elif estype=='别墅': - vv=fetch_villa_index(df) - d1=df.loc[vv,:] - if d1.shape[0]>3: + elif estype == '别墅': + vv = fetch_villa_index(df) + d1 = df.loc[vv, :] + if d1.shape[0] > 3: return d1 else: return df @@ -647,78 +610,69 @@ def commonprice(ct,rl,p): return df else: return df - - + def fetch_villa_index(dd): - - #dd=dd.dropna() - d1=dd[dd['area']>150] - - #print(dd) - - - villa_label=["独栋", "联排", "叠加", "双拼", - "花园面积", "泳池", "别墅", "带花园"] - bb=list() + + # dd=dd.dropna() + d1 = dd[dd['area'] > 150] + + # print(dd) + + villa_label = ["独栋", "联排", "叠加", "双拼", + "花园面积", "泳池", "别墅", "带花园"] + bb = list() for i in d1.index: - we=d1['name'][i] + we = d1['name'][i] for j in villa_label: if j in we: bb.append(i) - d21=dd[dd['room']>=6] - d2=d21[d21['area']>=360] - - d3=dd[dd['area']>500] - - villa=set(bb)|set(d2.index)|set(d3.index) - - return villa - - - + d21 = dd[dd['room'] >= 6] + d2 = d21[d21['area'] >= 360] + + d3 = dd[dd['area'] > 500] + + villa = set(bb) | set(d2.index) | set(d3.index) + + return villa + + def dsfjujia(ct, rl, lk): + + query = {"city": ct, "region": rl, + "name": lk, "s_date": {"$gt": dsfjujia_time}} + + data = fangjia.find_disanganf_price(query) + + alist = ['anjuke_esf_price', 'ganji_esf_price', + 'lianjia_esf_price', 'soufang_esf_price', + 'tuboshi_esf_price', 'zhugezhaofang_esf_price', + 'yungujia_esf_price', 'fanggugu_esf_price' + ] + a1list = set(alist) + blist = set(data.columns) + + clist = a1list & blist + + ddq = list(clist) + + data1 = data[ddq] + + ser = pandas.Series() - - - def dsfjujia(ct,rl,lk): - - query = {"city":ct,"region":rl, - "name":lk,"s_date":{"$gt":dsfjujia_time}} - - data=fangjia.find_disanganf_price(query) - - alist=['anjuke_esf_price','ganji_esf_price', - 'lianjia_esf_price','soufang_esf_price', - 'tuboshi_esf_price','zhugezhaofang_esf_price', - 'yungujia_esf_price','fanggugu_esf_price' - ] - a1list=set(alist) - blist=set(data.columns) - - clist=a1list & blist - - ddq=list(clist) - - data1=data[ddq] - - ser=pandas.Series() - for i in ddq: - pdd=data1[i].dropna() - #ser[i]=k_means(pdd) - ser[i]=list(pdd)[-1] - #print('采用抓取的第三方均价') + pdd = data1[i].dropna() + # ser[i]=k_means(pdd) + ser[i] = list(pdd)[-1] + # print('采用抓取的第三方均价') return ser - - - - - def sfjj(ll_zz,aa_zz,dddf): - - dxx=pandas.Series() + + def sfjj(ll_zz, aa_zz, dddf): + + dxx = pandas.Series() for i in ll_zz: if i in aa_zz: - dxx[i]=dddf[i] + dxx[i] = dddf[i] return int(dxx.mean()) + """ tt=timm//100 @@ -727,7 +681,7 @@ def commonprice(ct,rl,p): data= consider_villa(ct,rl,p,timm) """ - + ####################################################### """ junjia=dsfjujia(ct,rl,lk) @@ -735,118 +689,119 @@ def commonprice(ct,rl,p): aa_zz=set(junjia.index) jiaji_lz= list(ll_zz & aa_zz) """ - - def k_means_clus(pp1,clus): - + + def k_means_clus(pp1, clus): + from sklearn.cluster import KMeans - pv=list(pp1) - if len(set(pv))>clus: - gf=numpy.array([pv]).T - estimator = KMeans(n_clusters=clus)#构造聚类器 - estimator.fit(gf)#聚类 - label_pred = estimator.labels_ #获取聚类标签 - aa=collections.Counter(label_pred) - v=pandas.Series(aa) - gg=list(v) - index_max=gg.index(max(gg)) - - centroids = estimator.cluster_centers_ #获取聚类中心 - #inertia = estimator.inertia_ # 获取聚类准则的总和 - center=centroids[index_max][0] + pv = list(pp1) + if len(set(pv)) > clus: + gf = numpy.array([pv]).T + estimator = KMeans(n_clusters=clus) # 构造聚类器 + estimator.fit(gf) # 聚类 + label_pred = estimator.labels_ # 获取聚类标签 + aa = collections.Counter(label_pred) + v = pandas.Series(aa) + gg = list(v) + index_max = gg.index(max(gg)) + + centroids = estimator.cluster_centers_ # 获取聚类中心 + # inertia = estimator.inertia_ # 获取聚类准则的总和 + center = centroids[index_max][0] return (int(center)) else: return int(pp1.mean()) - - def group(junjia_lzf,junjia_anst,ctr): - - bb=[] + + def group(junjia_lzf, junjia_anst, ctr): + + bb = [] for i in junjia_lzf.index: - #print(junjia_lzf) - someprice=junjia_lzf[i] + # print(junjia_lzf) + someprice = junjia_lzf[i] + def compare15(b): - if 100*abs(someprice-b)/b<16: + if 100 * abs(someprice - b) / b < 16: return 1 else: return 0 - countmap=list(map(compare15,junjia_anst)) - #print('countmap=',countmap) - countmap1=pandas.Series(countmap) - countmap2=countmap1[countmap1>0] - #print('countmap2=',countmap2) - - numb=len(countmap2) - #print('len(countmap2)=',len(countmap2)) - if numb>ctr: - bb.append([someprice,numb]) + + countmap = list(map(compare15, junjia_anst)) + # print('countmap=',countmap) + countmap1 = pandas.Series(countmap) + countmap2 = countmap1[countmap1 > 0] + # print('countmap2=',countmap2) + + numb = len(countmap2) + # print('len(countmap2)=',len(countmap2)) + if numb > ctr: + bb.append([someprice, numb]) return bb - - def middle_operate(junjia_lzf,junjia_anst): - - #print('%%%',ppd3['total_price']) - - #print('junjia_lzf=',junjia_lzf) - #print('junjia_anst=',junjia_anst) - - + + def middle_operate(junjia_lzf, junjia_anst): + + # print('%%%',ppd3['total_price']) + + # print('junjia_lzf=',junjia_lzf) + # print('junjia_anst=',junjia_anst) + def mid_out_1(junjia_lzf): - - if len(junjia_lzf)>1: - #print('aa1',aa) - bb=group(junjia_lzf,junjia_lzf,1) - #print('bb=',bb) - if len(bb)>0: - bb1=pandas.DataFrame(bb,columns=['price','count']) - bb2=bb1.sort_values(by='count',ascending=False) - #print(bb2) + + if len(junjia_lzf) > 1: + # print('aa1',aa) + bb = group(junjia_lzf, junjia_lzf, 1) + # print('bb=',bb) + if len(bb) > 0: + bb1 = pandas.DataFrame(bb, columns=['price', 'count']) + bb2 = bb1.sort_values(by='count', ascending=False) + # print(bb2) return bb2.iloc[0]['price'] else: return None else: return None - - - def mid_out_2(junjia_lzf,junjia_anst): - - if len(junjia_lzf)>0 and len(junjia_anst)>0: - - bb=group(junjia_lzf,junjia_anst,0) - #print('bb1=',bb) - if len(bb)>0: - bb1=pandas.DataFrame(bb,columns=['price','count']) - bb2=bb1.sort_values(by='count',ascending=False) - #print(bb2) + + def mid_out_2(junjia_lzf, junjia_anst): + + if len(junjia_lzf) > 0 and len(junjia_anst) > 0: + + bb = group(junjia_lzf, junjia_anst, 0) + # print('bb1=',bb) + if len(bb) > 0: + bb1 = pandas.DataFrame(bb, columns=['price', 'count']) + bb2 = bb1.sort_values(by='count', ascending=False) + # print(bb2) return bb2.iloc[0]['price'] else: return None else: - return None - a11=mid_out_1(junjia_lzf) - a22=mid_out_2(junjia_lzf,junjia_anst) - a33=mid_out_1(junjia_anst) - aa1_3=pandas.Series([a11,a22,a33]).dropna() - #print(a11,a22,a33) - if len(aa1_3)>0: + return None + + a11 = mid_out_1(junjia_lzf) + a22 = mid_out_2(junjia_lzf, junjia_anst) + a33 = mid_out_1(junjia_anst) + aa1_3 = pandas.Series([a11, a22, a33]).dropna() + # print(a11,a22,a33) + if len(aa1_3) > 0: return aa1_3.iloc[0] else: return None - - - def devide_group(al,bl,junjia): - - alls=set(junjia.index) - def set_div(ds,alls): - if len(set(ds) & alls)>0: - return junjia[ds].dropna() - else: - return pandas.Series() - lzf=set_div(al,alls) - anst=set_div(bl,alls) - - return lzf , anst - - def dglg_out1(junjia,neighb): - + def devide_group(al, bl, junjia): + + alls = set(junjia.index) + + def set_div(ds, alls): + if len(set(ds) & alls) > 0: + return junjia[ds].dropna() + else: + return pandas.Series() + + lzf = set_div(al, alls) + anst = set_div(bl, alls) + + return lzf, anst + + def dglg_out1(junjia, neighb): + """ alist=['anjuke_esf_price','ganji_esf_price', 'lianjia_esf_price','soufang_esf_price', @@ -854,380 +809,355 @@ def commonprice(ct,rl,p): 'yungujia_esf_price','fanggugu_esf_price' ] """ - - - al=[ + + al = [ 'lianjia_esf_price', 'zhugezhaofang_esf_price', 'fanggugu_esf_price' - ] - bl=['anjuke_esf_price', - 'soufang_esf_price', - 'tuboshi_esf_price', - ] - - #ppd3dp=ppd3['total_price'].dropna() - if type(neighb)!=str and neighb!=None : - - junjia=junjia[junjia>0.55*neighb] + ] + bl = ['anjuke_esf_price', + 'soufang_esf_price', + 'tuboshi_esf_price', + ] + + # ppd3dp=ppd3['total_price'].dropna() + if type(neighb) != str and neighb != None: + + junjia = junjia[junjia > 0.55 * neighb] ''' junjia_lzf=junjia[al].dropna() junjia_anst=junjia[bl].dropna() ''' - alldi=devide_group(al,bl,junjia) - - junjia_lzf=alldi[0] - junjia_anst=alldi[1] - dg11=middle_operate(junjia_lzf,junjia_anst) + alldi = devide_group(al, bl, junjia) + + junjia_lzf = alldi[0] + junjia_anst = alldi[1] + dg11 = middle_operate(junjia_lzf, junjia_anst) return dg11 - - + + else: - alldi=devide_group(al,bl,junjia) - - junjia_lzf=alldi[0] - junjia_anst=alldi[1] - - dg22=middle_operate(junjia_lzf,junjia_anst) + alldi = devide_group(al, bl, junjia) + + junjia_lzf = alldi[0] + junjia_anst = alldi[1] + + dg22 = middle_operate(junjia_lzf, junjia_anst) return dg22 - - - lk=p[0] - location=take_location(ct,rl,p) - - #ttp=time.time() - neighb=no_source_price(ct,rl,location,p) - - #print(time.time()-ttp,'m'*20) - - junjia=dsfjujia(ct,rl,lk) - #print('junjia=',junjia) - + + lk = p[0] + location = take_location(ct, rl, p) + + # ttp=time.time() + neighb = no_source_price(ct, rl, location, p) + + # print(time.time()-ttp,'m'*20) + + junjia = dsfjujia(ct, rl, lk) + # print('junjia=',junjia) + ''' dglg=dsplg(junjia,neighb,2) dggg=dsplg(junjia,neighb,1) ''' - dglg=dglg_out1(junjia,neighb[0]) - #dggg=dglg_out1(junjia,neighb[0]) - - #print('dglg=',dglg) - #print('dggg=',dggg) - - #tt=time.time() - data= consider_villa(ct,rl,p) - #print(time.time()-tt,'<>'*26) - ####################################################### - - if data.shape[0]>3: - - paa=add_block(data) - ppd=paa[['total_price','area','region','district_name','url','block','p_date']] - #print('ppd##############') - #print('ppd=',ppd) - ppd=ppd.fillna(method='ffill') - - ppd1=ppd.dropna() - - #t3=time.time() - - ppd1['total_price']=ppd1['total_price']/ppd1['area'] - #print('ppd1time',time.time()-t3,'o'*36) - - #ppd1.loc['total_price']=ppd1['total_price']/ppd1['area'] - - #ppd2=ppd1 - + dglg = dglg_out1(junjia, neighb[0]) + # dggg=dglg_out1(junjia,neighb[0]) - - #if type(neighb) is not str: - - if ppd1.shape[0]>0: - - def guolv_data(ppd2,neighb): - if type(neighb) is not str: - ph=ppd2[ppd2['total_price']>0.5*neighb] + # print('dglg=',dglg) + # print('dggg=',dggg) + + # tt=time.time() + data = consider_villa(ct, rl, p) + # print(time.time()-tt,'<>'*26) + ####################################################### + + if data.shape[0] > 3: + + paa = add_block(data) + ppd = paa[['total_price', 'area', 'region', 'district_name', 'url', 'block', 'p_date']] + # print('ppd##############') + # print('ppd=',ppd) + ppd = ppd.fillna(method='ffill') + + ppd1 = ppd.dropna() + + # t3=time.time() + + ppd1['total_price'] = ppd1['total_price'] / ppd1['area'] + # print('ppd1time',time.time()-t3,'o'*36) + + # ppd1.loc['total_price']=ppd1['total_price']/ppd1['area'] + + # ppd2=ppd1 + + # if type(neighb) is not str: + + if ppd1.shape[0] > 0: + + def guolv_data(ppd2, neighb): + if type(neighb) is not str: + ph = ppd2[ppd2['total_price'] > 0.5 * neighb] return ph else: - return ppd2 - - #ppd3=ppd2[ppd2['total_price']>0.6*neighb] - ###################################### - ppd3=guolv_data(ppd1,neighb[0]) - - ###################################### - ldf=label(ppd3,lianjia) - #print('ldf=',ldf) - - ldf1=label(ppd3,zhuge) - - ldf2=label(ppd3,wawj) - - ldf3=label(ppd3,iwjw) - ldf4=label(ppd3,fangdd) - - ldf5=label(ppd3,wkzf) - l_and_z=ldf|ldf1 - #print(l_and_z) - #print('l_and_z=',l_and_z) - allf=l_and_z|ldf3|ldf4|ldf5|ldf2 - #print('l_and_z_and_i=',l_and_z_and_i) - #mm=0 - length=len(l_and_z) - length1=len(allf) + return ppd2 + + # ppd3=ppd2[ppd2['total_price']>0.6*neighb] + + ###################################### + ppd3 = guolv_data(ppd1, neighb[0]) + + ###################################### + ldf = label(ppd3, lianjia) + # print('ldf=',ldf) + + ldf1 = label(ppd3, zhuge) + + ldf2 = label(ppd3, wawj) + + ldf3 = label(ppd3, iwjw) + ldf4 = label(ppd3, fangdd) + + ldf5 = label(ppd3, wkzf) + l_and_z = ldf | ldf1 + # print(l_and_z) + # print('l_and_z=',l_and_z) + allf = l_and_z | ldf3 | ldf4 | ldf5 | ldf2 + # print('l_and_z_and_i=',l_and_z_and_i) + # mm=0 + length = len(l_and_z) + length1 = len(allf) - if dglg != None: - - #print('junjia.shape',len(junjia)) - sample_size=len(junjia) - return dglg,float('%.2f' % credit['0.8']),value_type['第三方'],sample_size - - elif length>3: - #print('ldf=',l_and_z_and_i) - pppk=ppd3.loc[l_and_z,:] - - ppp=pppk[['total_price','p_date']] - #print(ppp) - ppp1=ppp.sort_values(['p_date'], ascending=False) - - gdf=ppp1['total_price'][:36].dropna() - #print('链家和诸葛找房合并后的数据集=',gdf) - sample_size=len(gdf) - gm=k_means_clus(gdf,2) - print('链猪= ',gm) - return int(gm),float('%.2f' % credit['0.8']),value_type['自身样本'] ,sample_size #优质样本 - - - elif data.shape[0]>0: - + + # print('junjia.shape',len(junjia)) + sample_size = len(junjia) + return dglg, float('%.2f' % credit['0.8']), value_type['第三方'], sample_size + + elif length > 3: + # print('ldf=',l_and_z_and_i) + pppk = ppd3.loc[l_and_z, :] + + ppp = pppk[['total_price', 'p_date']] + # print(ppp) + ppp1 = ppp.sort_values(['p_date'], ascending=False) + + gdf = ppp1['total_price'][:36].dropna() + # print('链家和诸葛找房合并后的数据集=',gdf) + sample_size = len(gdf) + gm = k_means_clus(gdf, 2) + print('链猪= ', gm) + return int(gm), float('%.2f' % credit['0.8']), value_type['自身样本'], sample_size # 优质样本 + + + elif data.shape[0] > 0: + print('诸葛找房算法') from zhuge import zhuge_consist - data['avg']=data['total_price']/data['area'] - - def guolv_data11(data,neighb): - if type(neighb) is not str: - - print('周边价格=',neighb) - ddd11=data[data['avg']>0.5*neighb] + data['avg'] = data['total_price'] / data['area'] + + def guolv_data11(data, neighb): + if type(neighb) is not str: + + print('周边价格=', neighb) + ddd11 = data[data['avg'] > 0.5 * neighb] return ddd11 else: return data - - ddd11=guolv_data11(data,neighb[0]) - - if ddd11.shape[0]>2: - - - s=zhuge_consist.url_lable(ddd11) - s1=zhuge_consist.drop_rubbish(s) - #print('s1=',s1) - s2=set(s1) - #print('s2=',s2) - if len(s2)>3: - + + ddd11 = guolv_data11(data, neighb[0]) + + if ddd11.shape[0] > 2: + + s = zhuge_consist.url_lable(ddd11) + s1 = zhuge_consist.drop_rubbish(s) + # print('s1=',s1) + s2 = set(s1) + # print('s2=',s2) + if len(s2) > 3: + from zhuge import zhuge_consist - s=zhuge_consist.zhuge_out(ddd11) - - def compelex(ddd11,s,neighb,junjia): - cc1=ddd11['avg'].dropna() - - - kmzhuge=k_means(cc1) - - junjia['zhuge_logic']=s - - junjia['kmzg']=kmzhuge + s = zhuge_consist.zhuge_out(ddd11) - if type(neighb)!=str and (neighb)!=None: - - junjia['neighb']=neighb - #print('junjia',junjia) - - junjia_front=junjia[['zhuge_logic','kmzg']] - cc=group(junjia_front,junjia,1) - #junjia_km=k_means(junjia) - #print('cc=',cc) + def compelex(ddd11, s, neighb, junjia): + cc1 = ddd11['avg'].dropna() + + kmzhuge = k_means(cc1) + + junjia['zhuge_logic'] = s + + junjia['kmzg'] = kmzhuge + + if type(neighb) != str and (neighb) != None: + junjia['neighb'] = neighb + # print('junjia',junjia) + + junjia_front = junjia[['zhuge_logic', 'kmzg']] + cc = group(junjia_front, junjia, 1) + # junjia_km=k_means(junjia) + # print('cc=',cc) ############ - if len(cc)>0: - cc1=pandas.DataFrame(cc,columns=['price','count']) - cc2=cc1.sort_values(by='count',ascending=False) - #print(bb2) - sample_size=ddd11.shape[0] - return cc2.iloc[0]['price']*1.0,float('%.2f' % credit['0.7']),value_type['第三方'],sample_size + if len(cc) > 0: + cc1 = pandas.DataFrame(cc, columns=['price', 'count']) + cc2 = cc1.sort_values(by='count', ascending=False) + # print(bb2) + sample_size = ddd11.shape[0] + return cc2.iloc[0]['price'] * 1.0, float('%.2f' % credit['0.7']), value_type[ + '第三方'], sample_size else: - sample_size=ddd11.shape[0] - return junjia['kmzg']*1.0,float('%.2f' % credit['0.6']),value_type['自身样本'],sample_size + sample_size = ddd11.shape[0] + return junjia['kmzg'] * 1.0, float('%.2f' % credit['0.6']), value_type[ + '自身样本'], sample_size - - junjia_km=compelex(ddd11,s,neighb[0],junjia) + junjia_km = compelex(ddd11, s, neighb[0], junjia) return junjia_km - - + + else: - print('*'*20,'采用周边算法') - if ddd11.shape[0]>3: - cc1=ddd11['avg'].dropna() - kmzhuge=k_means_clus(cc1,2) - sample_size=ddd11.shape[0] - return kmzhuge,float('%.2f' % credit['0.6']),value_type['自身样本'],sample_size + print('*' * 20, '采用周边算法') + if ddd11.shape[0] > 3: + cc1 = ddd11['avg'].dropna() + kmzhuge = k_means_clus(cc1, 2) + sample_size = ddd11.shape[0] + return kmzhuge, float('%.2f' % credit['0.6']), value_type['自身样本'], sample_size else: - return neighb[0],float('%.2f' % credit['0.5']),value_type['周边楼盘'],neighb[1] + return neighb[0], float('%.2f' % credit['0.5']), value_type['周边楼盘'], neighb[1] elif dglg != None: - print('优质第三方均价2 = ',dglg) - sample_size=len(junjia) - return dglg,float('%.2f' % credit['0.8']),value_type['第三方'] ,sample_size #'优质样本' + print('优质第三方均价2 = ', dglg) + sample_size = len(junjia) + return dglg, float('%.2f' % credit['0.8']), value_type['第三方'], sample_size # '优质样本' else: - return neighb[0],float('%.2f' % credit['0.5']),value_type['周边楼盘'],neighb[1] + return neighb[0], float('%.2f' % credit['0.5']), value_type['周边楼盘'], neighb[1] elif dglg != None: - print('优质第三方均价3 = ',dglg) - sample_size=len(junjia) - return dglg,float('%.2f' % credit['0.8']),value_type['第三方'] #'优质样本' - + print('优质第三方均价3 = ', dglg) + sample_size = len(junjia) + return dglg, float('%.2f' % credit['0.8']), value_type['第三方'] # '优质样本' + else: - - return neighb[0],float('%.2f' % credit['0.5']),value_type['周边楼盘'],neighb[1] + + return neighb[0], float('%.2f' % credit['0.5']), value_type['周边楼盘'], neighb[1] elif dglg != None: - print('优质第三方均价4 = ',dglg) - return dglg,float('%.2f' % credit['0.8']),value_type['第三方'] ,len(junjia) #'优质样本' + print('优质第三方均价4 = ', dglg) + return dglg, float('%.2f' % credit['0.8']), value_type['第三方'], len(junjia) # '优质样本' else: - - #print('*'*100) - #print(neighb) - return neighb[0],float('%.2f' % credit['0.5']),value_type['周边楼盘'],neighb[1] + + # print('*'*100) + # print(neighb) + return neighb[0], float('%.2f' % credit['0.5']), value_type['周边楼盘'], neighb[1] - - -client65 = pymongo.MongoClient('192.168.0.105',27777) +client65 = pymongo.MongoClient('192.168.0.105', 27777) db65 = client65.fangjia seawater65 = db65.seawater -client136 = pymongo.MongoClient('192.168.0.136',27017) +client136 = pymongo.MongoClient('192.168.0.136', 27017) db136 = client136.fangjia dsf136 = db136.average_statistics -seaweed136 = db136.seaweed +seaweed136 = db136.seaweed + +dsf = db136.district_stat -dsf = db136.district_stat - -#client136 = pymongo.MongoClient('192.168.0.136',27017) -#db136 = client136.fangjia +# client136 = pymongo.MongoClient('192.168.0.136',27017) +# db136 = client136.fangjia +def region_district_list(ct, rl): + # import pandas + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # seaweed1 = db1.seaweed + query1 = {"status": 0, "cat": "district", "city": ct, "region": rl} + fields1 = {"lat2": 1, "lng2": 1, "city": 1, "region": 1, + "cat": 1, "name": 1, 'estate_type2': 1, 'alias': 1} - -def region_district_list(ct,rl): - #import pandas - #client1 = pymongo.MongoClient('192.168.0.136',27017) - #db1 = client1.fangjia - #seaweed1 = db1.seaweed - - query1 = {"status":0,"cat":"district","city":ct,"region":rl} - fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, - "cat":1,"name":1,'estate_type2':1,'alias':1} - - #lct= list(seaweed136.find(query1, fields1)) - #lf=pandas.DataFrame(lct) - lf=fangjia.find_seaweed(query1,fields1) + # lct= list(seaweed136.find(query1, fields1)) + # lf=pandas.DataFrame(lct) + lf = fangjia.find_seaweed(query1, fields1) return lf - + + def city_list(): - #client = pymongo.MongoClient('192.168.0.136',27017) #'192.168.0.65',27777 - #db = client.fangjia + # client = pymongo.MongoClient('192.168.0.136',27017) #'192.168.0.65',27777 + # db = client.fangjia city = db136.server_area - #seawater.find_one() - - - pos= city.find({"display":{"$ne":"全国"}}, - {"display":1,"city_grade":1,"_id":1}).sort([("city_grade",1),("_id",1)]) - - + # seawater.find_one() - data=pandas.DataFrame(list(pos)) - #print(data) + pos = city.find({"display": {"$ne": "全国"}}, + {"display": 1, "city_grade": 1, "_id": 1}).sort([("city_grade", 1), ("_id", 1)]) + + data = pandas.DataFrame(list(pos)) + # print(data) return data['display'] - -#print(some_district_information()) +# print(some_district_information()) def region_list(city): - #client = pymongo.MongoClient('192.168.0.136',27017) - #db = client.fangjia + # client = pymongo.MongoClient('192.168.0.136',27017) + # db = client.fangjia region = db136.region_block - #seawater.find_one() - query = {"city":city,"category":"region"} - fields1 = {"name":1} - pos = list(region.find(query,fields1)) - - - data=pandas.DataFrame(pos) + # seawater.find_one() + query = {"city": city, "category": "region"} + fields1 = {"name": 1} + pos = list(region.find(query, fields1)) + + data = pandas.DataFrame(pos) return data['name'] - - - - - - def add_alias(ppd): if 'alias' not in ppd.columns: - ppd['alias']='无' + ppd['alias'] = '无' return ppd - + + def ghhh(kkg): - kkg=kkg.fillna('无') - a=list() + kkg = kkg.fillna('无') + a = list() for i in kkg.index: - b=list(kkg.loc[[i],:].values.flatten()) + b = list(kkg.loc[[i], :].values.flatten()) a.append(b) - return a + return a - - -client1 = pymongo.MongoClient('192.168.10.182',27017) +client1 = pymongo.MongoClient('192.168.10.182', 27017) db1 = client1.fangjia stat = db1.distrit_stat -pt=client1.fangjia_log.calc_command +pt = client1.fangjia_log.calc_command ################################################################# -def region_district_list_lk(ct,rl,lk): - #import pandas - #client1 = pymongo.MongoClient('192.168.0.136',27017) - #db1 = client1.fangjia - #seaweed1 = db1.seaweed - - query1 = {"status":0,"cat":"district","city":ct,"region":rl,"name":lk} - fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, - "cat":1,"name":1,'estate_type2':1,'alias':1} - - #lct= list(seaweed136.find(query1, fields1)) - #lf=pandas.DataFrame(lct) - lf=fangjia.find_seaweed(query1,fields1) +def region_district_list_lk(ct, rl, lk): + # import pandas + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # seaweed1 = db1.seaweed + + query1 = {"status": 0, "cat": "district", "city": ct, "region": rl, "name": lk} + fields1 = {"lat2": 1, "lng2": 1, "city": 1, "region": 1, + "cat": 1, "name": 1, 'estate_type2': 1, 'alias': 1} + + # lct= list(seaweed136.find(query1, fields1)) + # lf=pandas.DataFrame(lct) + lf = fangjia.find_seaweed(query1, fields1) return lf - - - def last_sunday(weekend): - #import datetime + # import datetime """ today = datetime.datetime.now() @@ -1236,188 +1166,163 @@ def last_sunday(weekend): now_time=datetime.datetime(today.year, today.month, today.day, 0, 0, 0) week_time = now_time.weekday() """ - future_time = weekend-datetime.timedelta(days=7-1) - d1= future_time.strftime('%Y%m%d') - weekend1= datetime.datetime.strptime(d1,'%Y%m%d') - weekend=weekend1+datetime.timedelta(hours=-8) + future_time = weekend - datetime.timedelta(days=7 - 1) + d1 = future_time.strftime('%Y%m%d') + weekend1 = datetime.datetime.strptime(d1, '%Y%m%d') + weekend = weekend1 + datetime.timedelta(hours=-8) return weekend +# last_sunday=last_sunday(weekend) +def make_up_price(ct, rl, lk, weekend): + # last_sunday=last_sunday(weekend) + # save_lg=client136.fangjia.district_stat + if lk != None: + # cc=last_sunday + ly = last_sunday(weekend) + # stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) + stat = dsf + query = {"city": ct, "region": rl, "district_name": lk, 'weekend': ly + } + pos1 = list(stat.find(query).limit(30)) -#last_sunday=last_sunday(weekend) + pos = pandas.DataFrame(pos1) -def make_up_price(ct,rl,lk,weekend): - + # pos=timestamp_to_int(pos) + # pos=pos[pos['week1']>cc-5] + # pos=pos.sort_values(by='week1',ascending=False) - #last_sunday=last_sunday(weekend) - #save_lg=client136.fangjia.district_stat - if lk!=None: + # print('5'*10) + # print(pos) - #cc=last_sunday - - - ly=last_sunday(weekend) - #stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) - stat=dsf - query = {"city":ct,"region":rl,"district_name":lk,'weekend':ly - } + if pos.shape[0] > 0: + # pp1""" + pp1 = pos['avg_price'].iloc[0] + if type(pp1) != None and type(pp1) != str: - - pos1=list(stat.find(query).limit(30)) - - pos=pandas.DataFrame(pos1) - - #pos=timestamp_to_int(pos) - #pos=pos[pos['week1']>cc-5] - #pos=pos.sort_values(by='week1',ascending=False) - - #print('5'*10) - #print(pos) - - if pos.shape[0]>0: - #pp1""" - pp1=pos['avg_price'].iloc[0] - if type(pp1)!=None and type(pp1)!=str: - - - return int(pp1) else: - return None + return None else: return None - + else: - #stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) - stat=dsf - query = {"city":ct,"region":rl,"district_name":lk,'weekend':last_sunday - } + # stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) + stat = dsf + query = {"city": ct, "region": rl, "district_name": lk, 'weekend': last_sunday + } - - pos1=list(stat.find(query).limit(30)) - - pos=pandas.DataFrame(pos1) - - #pos=timestamp_to_int(pos) - #pos=pos[pos['week1']>cc-5] - #pos=pos.sort_values(by='week1',ascending=False) - - #print('5'*10) - #print(pos) - - if pos.shape[0]>0: - #pp1 - pp1=pos['avg_price'].iloc[0] - if type(pp1)!=None and type(pp1)!=str: + pos1 = list(stat.find(query).limit(30)) + + pos = pandas.DataFrame(pos1) + + # pos=timestamp_to_int(pos) + # pos=pos[pos['week1']>cc-5] + # pos=pos.sort_values(by='week1',ascending=False) + + # print('5'*10) + # print(pos) + + if pos.shape[0] > 0: + # pp1 + pp1 = pos['avg_price'].iloc[0] + if type(pp1) != None and type(pp1) != str: - - return int(pp1) else: - return None + return None else: return None - - -#weekend=weekend00() +# weekend=weekend00() @fangjia.timer -def out_put_price(ct,rl,lk,weekend): - lf=region_district_list_lk(ct,rl,lk) - lf=add_alias(lf) - - p=ghhh(lf[['name','alias','estate_type2']])[0] - - mm=commonprice(ct,rl,p) - - c_date = datetime.datetime.utcnow() - m_date=datetime.datetime.utcnow() - - if type(mm[0])!=str: - - print(ct,rl,lk,mm,'$'*20) - #print(lf['estate_type2'].iloc[0],'*'*10) - estate_type=lf['estate_type2'].iloc[0] - +def out_put_price(ct, rl, lk, weekend): + lf = region_district_list_lk(ct, rl, lk) + lf = add_alias(lf) + + p = ghhh(lf[['name', 'alias', 'estate_type2']])[0] + + mm = commonprice(ct, rl, p) + + c_date = datetime.datetime.utcnow() + m_date = datetime.datetime.utcnow() + + if type(mm[0]) != str: + + print(ct, rl, lk, mm, '$' * 20) + # print(lf['estate_type2'].iloc[0],'*'*10) + estate_type = lf['estate_type2'].iloc[0] + + # last_sunday=last_sunday() + # t1=time.time() + last_sunday_price = make_up_price(ct, rl, lk, weekend) + + # print(time.time()-t1,'*'*20) + + # print('last_sunday_price',last_sunday_price) + # print('mm[0]',mm[0]) - - #last_sunday=last_sunday() - #t1=time.time() - last_sunday_price=make_up_price(ct,rl,lk,weekend) - #print(time.time()-t1,'*'*20) - - #print('last_sunday_price',last_sunday_price) - #print('mm[0]',mm[0]) - def make_up(): - if type(last_sunday_price) != str and last_sunday_price != None and mm[0]!=None and type(mm[0])!=str: - - make_up=10000*(mm[0]-last_sunday_price)/last_sunday_price + if type(last_sunday_price) != str and last_sunday_price != None and mm[0] != None and type(mm[0]) != str: + + make_up = 10000 * (mm[0] - last_sunday_price) / last_sunday_price return make_up else: return 0 - - make_up=int(make_up()) - - + + make_up = int(make_up()) + try: - + stat.replace_one( - - {"city":ct,"region":rl,"district_name":p[0],"weekend":weekend}, - - {"city":ct,"region":rl,"district_name":p[0],'credit':mm[1], - "calc_value":int(mm[0]), "weekend":weekend,"c_date":c_date,'avg_price':int(mm[0]), - "m_date":m_date,'value_type':int(mm[2]),'weekend_markup':make_up, - 'changed_markup':make_up,'markup':make_up,'sample_size':mm[3], - 'estate_type':estate_type},True - ) - - - - pt.replace_one( - - {"city":ct,"region":rl,"district_name":p[0],"weekend":weekend}, - - - {"city":ct,"region":rl,"district_name":p[0],"c_date":c_date,'status':int(0), - 'remark':'week','category':'sell','weekend':weekend,'priority':int(0), - 'problem':None, - },True - ) - - - except: - - stat.replace_one( - - {"city":ct,"region":rl,"district_name":p[0],"weekend":weekend}, - - {"city":ct,"region":rl,"district_name":p[0],'credit':mm[1], - "calc_value":int(mm[0]), "weekend":weekend,"c_date":c_date,'avg_price':int(mm[0]), - "m_date":m_date,'value_type':int(mm[2]),'weekend_markup':make_up, - 'changed_markup':make_up,'markup':make_up,'sample_size':mm[3], - 'estate_type':estate_type},True - ) - - else: - pt.replace_one( - - {"city":ct,"region":rl,"district_name":p[0],"weekend":weekend}, - - - {"city":ct,"region":rl,"district_name":p[0],"c_date":c_date,'status':int(1), - 'remark':'week','category':'sell','weekend':weekend,'priority':int(0), - 'problem':mm[0], - },True + + {"city": ct, "region": rl, "district_name": p[0], "weekend": weekend}, + + {"city": ct, "region": rl, "district_name": p[0], 'credit': mm[1], + "calc_value": int(mm[0]), "weekend": weekend, "c_date": c_date, 'avg_price': int(mm[0]), + "m_date": m_date, 'value_type': int(mm[2]), 'weekend_markup': make_up, + 'changed_markup': make_up, 'markup': make_up, 'sample_size': mm[3], + 'estate_type': estate_type}, True ) - - \ No newline at end of file + pt.replace_one( + + {"city": ct, "region": rl, "district_name": p[0], "weekend": weekend}, + + {"city": ct, "region": rl, "district_name": p[0], "c_date": c_date, 'status': int(0), + 'remark': 'week', 'category': 'sell', 'weekend': weekend, 'priority': int(0), + 'problem': None, + }, True + ) + + + except: + + stat.replace_one( + + {"city": ct, "region": rl, "district_name": p[0], "weekend": weekend}, + + {"city": ct, "region": rl, "district_name": p[0], 'credit': mm[1], + "calc_value": int(mm[0]), "weekend": weekend, "c_date": c_date, 'avg_price': int(mm[0]), + "m_date": m_date, 'value_type': int(mm[2]), 'weekend_markup': make_up, + 'changed_markup': make_up, 'markup': make_up, 'sample_size': mm[3], + 'estate_type': estate_type}, True + ) + + else: + pt.replace_one( + + {"city": ct, "region": rl, "district_name": p[0], "weekend": weekend}, + + {"city": ct, "region": rl, "district_name": p[0], "c_date": c_date, 'status': int(1), + 'remark': 'week', 'category': 'sell', 'weekend': weekend, 'priority': int(0), + 'problem': mm[0], + }, True + )