From 933f2d7aececdfc19406b2fe1ca678dc4b1db6f0 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Wed, 7 Mar 2018 09:59:22 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E5=9D=87=E4=BB=B7?= =?UTF-8?q?=E7=B3=BB=E7=BB=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 + PriceSystem/CalcCmdReceiver.py | 53 + PriceSystem/CalcCmdSender.py | 49 + PriceSystem/CalcSpeedMonitor.py | 27 + PriceSystem/config/__init__.py | 35 + PriceSystem/fangjia/ElasticsearchClient.py | 23 + PriceSystem/fangjia/__init__.py | 281 ++++ PriceSystem/fangjia/mongoClients.py | 33 + PriceSystem/fangjia/seaweed.py | 8 + PriceSystem/sell_price/__init__.py | 5 + PriceSystem/sell_price/zhuge0118A.py | 1424 ++++++++++++++++++++ PriceSystem/test.py | 12 + PriceSystem/zhuge/__init__.py | 1 + PriceSystem/zhuge/zhuge2.py | 487 +++++++ PriceSystem/zhuge/zhuge_consist.py | 486 +++++++ PriceSystem/zhuge0118A.py | 1423 +++++++++++++++++++ 16 files changed, 4350 insertions(+) create mode 100644 .gitignore create mode 100644 PriceSystem/CalcCmdReceiver.py create mode 100644 PriceSystem/CalcCmdSender.py create mode 100644 PriceSystem/CalcSpeedMonitor.py create mode 100644 PriceSystem/config/__init__.py create mode 100644 PriceSystem/fangjia/ElasticsearchClient.py create mode 100644 PriceSystem/fangjia/__init__.py create mode 100644 PriceSystem/fangjia/mongoClients.py create mode 100644 PriceSystem/fangjia/seaweed.py create mode 100644 PriceSystem/sell_price/__init__.py create mode 100644 PriceSystem/sell_price/zhuge0118A.py create mode 100644 PriceSystem/test.py create mode 100644 PriceSystem/zhuge/__init__.py create mode 100644 PriceSystem/zhuge/zhuge2.py create mode 100644 PriceSystem/zhuge/zhuge_consist.py create mode 100644 PriceSystem/zhuge0118A.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d4f4c0c --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/PriceSystem/config/__pycache__ +/PriceSystem/PriceSystem/.idea +.idea diff --git a/PriceSystem/CalcCmdReceiver.py b/PriceSystem/CalcCmdReceiver.py new file mode 100644 index 0000000..82586e2 --- /dev/null +++ b/PriceSystem/CalcCmdReceiver.py @@ -0,0 +1,53 @@ +import json +import multiprocessing +import time +import redis +import config +import sell_price +import fangjia +import datetime + +QUEUE = "calc_cmd" +redis_pool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=1) +logger = fangjia.get_logger("CalcCmdReceiver") + + +def worker(): + client = redis.Redis(connection_pool=redis_pool) + while True: + try: + cmd = client.lpop(QUEUE) + except: + cmd = None + if cmd is None: + time.sleep(3) + logger.info("等待接受计算命令") + else: + cmd = format_cmd(cmd) + start_time = time.clock() + try: + sell_price.calc_sell_price(cmd) + end_time = time.clock() + logger.info("Calc :" + str(cmd) + "\t计算成功,耗时:" + str(end_time - start_time)) + except Exception as ex: + print(ex) + logger.exception(ex) + end_time = time.clock() + logger.error("Calc :" + str(cmd) + "\t计算失败,耗时:" + (end_time - start_time), ex) + + +def format_cmd(cmd): + cmd = json.loads(cmd) + cmd["weekend"] = datetime.datetime.strptime(cmd["weekend"], "%Y-%m-%d %H:%M:%S") + return cmd + + +if __name__ == "__main__": + + pro_num = config.get_worker_num() + pool = multiprocessing.Pool(processes=pro_num) + for pid in range(1, pro_num): + pool.apply_async(worker, ()) + pool.close() + logger.info("均价系统启动成功") + pool.join() diff --git a/PriceSystem/CalcCmdSender.py b/PriceSystem/CalcCmdSender.py new file mode 100644 index 0000000..3b2f40e --- /dev/null +++ b/PriceSystem/CalcCmdSender.py @@ -0,0 +1,49 @@ +import json +import redis +import fangjia +import config +from datetime import datetime, date + + +class CJsonEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, datetime): + return obj.strftime('%Y-%m-%d %H:%M:%S') + elif isinstance(obj, date): + return obj.strftime('%Y-%m-%d') + else: + return json.JSONEncoder.default(self, obj) + + +QUEUE = "calc_cmd" +redisPool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=1) +client = redis.Redis(connection_pool=redisPool) + + +def get_district(): + seaweeds = [] + keys = {"city": 1, "region": 1, "name": 1, "_id": 0} + for dis in fangjia.get_collection("fangjia", "seaweed").find({"cat": "district", "status": 0, "city": "上海"}, keys): + dis["weekend"] = datetime.strptime("2018-03-11", "%Y-%m-%d") + seaweeds.append(dis) + + return seaweeds + + +def send_cmd(seaweed): + """ + cmd = {} + cmd["city"] = seaweed["city"] + cmd["region"] = seaweed["region"] + cmd["name"] = seaweed["name"] + """ + + json_cmd = json.dumps(seaweed, ensure_ascii=False, cls=CJsonEncoder) + client.rpush(QUEUE, json_cmd) + print("命令发送成功:" + json_cmd) + + +if __name__ == "__main__": + for seaweed in get_district(): + if len(seaweed) > 2: + send_cmd(seaweed) diff --git a/PriceSystem/CalcSpeedMonitor.py b/PriceSystem/CalcSpeedMonitor.py new file mode 100644 index 0000000..996b309 --- /dev/null +++ b/PriceSystem/CalcSpeedMonitor.py @@ -0,0 +1,27 @@ +import json +import multiprocessing +import time +import redis + +QUEUE = "calc_cmd" +redis_pool = redis.ConnectionPool(host='192.168.0.144', port=6379, db=0) + + +def monitor(pname): + client = redis.Redis(connection_pool=redis_pool) + while True: + pre_counter = client.llen(QUEUE) + time.sleep(60) + print(pname + "#####################################:".zfill(100) + str( + pre_counter - client.llen(QUEUE)) + "/min") + + +def format_cmd(cmd): + return json.loads(cmd) + + +if __name__ == "__main__": + pool = multiprocessing.Pool(processes=1) + pool.apply_async(monitor, ("Monitor-001",)) + pool.close() + pool.join() diff --git a/PriceSystem/config/__init__.py b/PriceSystem/config/__init__.py new file mode 100644 index 0000000..63c6053 --- /dev/null +++ b/PriceSystem/config/__init__.py @@ -0,0 +1,35 @@ +logging_config = { + +} + + +def get_logging_config(): + return logging_config + + +def get_worker_num(): + return 20 + + +def get_redis_host(): + return "192.168.0.144" + + +def get_mongodb_host(): + return "192.168.10.182" + + +def get_mongodb_port(): + return 27017 + + +def get_log_mongodb_host(): + return "192.168.0.136" + + +def get_log_mongodb_port(): + return 52005 + + +def get_timeout(): + return 0.3 diff --git a/PriceSystem/fangjia/ElasticsearchClient.py b/PriceSystem/fangjia/ElasticsearchClient.py new file mode 100644 index 0000000..3cbe793 --- /dev/null +++ b/PriceSystem/fangjia/ElasticsearchClient.py @@ -0,0 +1,23 @@ +from elasticsearch import Elasticsearch +from pypinyin import lazy_pinyin +from fangjia import mongoClients as mongoClients + +city_province_map = {} +es = Elasticsearch(['192.168.0.18']) + +server_area = mongoClients.MongoClients("fangjia", "server_area", "mongo.db.fangjia.com").collection +for area in server_area.find({"display": {'$ne': "全国"}}, {"display": 1, "province": 1}): + city_province_map[area["display"]] = area["province"] + + +def to_pinyin(hans): + return "".join(lazy_pinyin(hans)) + + +def get_index_name(city, region): + big_city = ["北京", "上海", "广州", "深圳", "天津", "杭州", "成都", "苏州", "重庆", "苏州", "南京", "武汉", "厦门"] + if big_city.count(city) > 0: + return "seawater-" + to_pinyin(city) + "-" + to_pinyin(region) + if list(city_province_map.keys()).count(city) == 0: + return "seawater-unknown" + return "seawater-" + to_pinyin(city) diff --git a/PriceSystem/fangjia/__init__.py b/PriceSystem/fangjia/__init__.py new file mode 100644 index 0000000..debe270 --- /dev/null +++ b/PriceSystem/fangjia/__init__.py @@ -0,0 +1,281 @@ +import pandas +import time +import logging +import config +from fangjia import mongoClients as mongoClients +from fangjia import ElasticsearchClient as es + +formatter = logging.Formatter('%(processName)s %(asctime)s %(name)-6s %(levelname)-6s %(message)s') + + +def get_logger(name): + logger = logging.getLogger(name) + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + + file_handler = logging.FileHandler("/home/goojia/logs/price.log") + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + logger.setLevel("INFO") + + return logger + + +def get_performance_logger(name): + logger = logging.getLogger(name) + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + + file_handler = logging.FileHandler("/home/goojia/logs/price.performance.log") + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + logger.setLevel("INFO") + + return logger + + +performance_logger = get_performance_logger("performance") + + +def timer(func): + def _timer(*args, **kw): + begin = time.clock() + rs = func(*args, **kw) + end = time.clock() + elapsed = end - begin + if elapsed > config.get_timeout(): + arg_list = list() + for arg in args: + arg_list.append(str(arg)) + performance_logger.info("函数名:(%s) 参数:(%s) 耗时:%f" % (func.__name__, arg_list, end - begin)) + return rs + + return _timer + + +def get_collection(db, name, host="192.168.0.136", port=27017): + return mongoClients.MongoClients(db, name, host, port).collection + + +def get_seawater_index_name(city, region): + return es.get_index_name(city, region) + + +@timer +def findHouse(query, fields=None): + # s = time.time() + seawater105 = get_collection("fangjia", "seawater", "192.168.0.105", 27777) + for err_count in range(0, 10): + + try: + + # lt = seawater105.count(query) + + # print('flag1') + lt = 1000 + # seawater.find(query,{"p_date":-1}).sort([("p_date",-1)]) + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + # print("==================>", time.time() - s) + return data + except: + lt = seawater105.count(query) + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + return data + + raise Exception("样本查询失败!") + + +@timer +def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, cat="sell", limit=100): + index_name = get_seawater_index_name(city, region) + + routing_set = [cat + "_" + city + "_" + region + "_" + alia for alia in alias] + must = [ + { + "term": {"city": city} + }, { + "term": {"cat": cat} + }, { + "term": {"format_region": region} + }, { + "terms": {"district_name.raw": alias} + } + + ] + if roots is not None: + must.append({ + "terms": {"root": roots} + }) + + if pdate is not None: + must.append({"range": {"p_date": {"gt": pdate}}}) + + body = { + + "query": { + "bool": { + "must": must + } + }, + "from": 0, + "size": limit, + "sort": [{"p_date": "desc"}], + "aggs": {} + } + response = es.es.search(index_name, "doc", routing=",".join(routing_set), body=body) + return pandas.DataFrame([doc["_source"] for doc in response["hits"]["hits"]]) + + +@timer +def find_sample_pool(query, fields=None): + # s = time.time() + seawater105 = get_collection("fangjia", "seawater", "192.168.0.105 ", 27777) + for err_count in range(0, 10): + + try: + + # lt = seawater105.count(query) + lt = 300 + # print('flag2') + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + # print("==================>", time.time() - s) + return data + + except: + + # err_count = err_count + 1 + # lt = seawater105.count(query) + lt = 300 + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + return data + + raise Exception("样本查询失败!") + + +def find_disanganf_price(query, fields=None): + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # dsf = db1.average_statistics + # s = time.time() + seawater105 = get_collection("fangjia", "average_statistics", "192.168.0.136", 27017) + for err_count in range(0, 10): + + try: + # print('flag3') + lt = seawater105.count(query) + + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + # print("==================>", time.time() - s) + return data + + except: + + # err_count = err_count + 1 + lt = seawater105.count(query) + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + return data + + raise Exception("yang ben cha xun shi bai1") + + +def dis_feature(query, fields=None): + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # dsf = db1.average_statistics + # s = time.time() + seawater105 = get_collection("luogan", "distric_feature_0119", "192.168.0.136", 27017) + for err_count in range(0, 10): + + try: + # print('flag4') + lt = seawater105.count(query) + + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + # print("==================>", time.time() - s) + return data + + except: + + # err_count = err_count + 1 + lt = seawater105.count(query) + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + return data + + raise Exception("yang ben cha xun shi bai1") + +@timer +def find_seaweed(query, fields=None): + # dirlt=p[0] + + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # seaweed1 = db1.seaweed + # s = time.time() + seawater105 = get_collection("fangjia", "seaweed", "192.168.0.136", 27017) + for err_count in range(0, 10): + + try: + # print('flag5') + lt = seawater105.count(query) + + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + # print("==================>", time.time() - s) + return data + + except: + + # err_count = err_count + 1 + lt = seawater105.count(query) + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + return data + + raise Exception("yang ben cha xun shi bai1") + + +def find_region_district_list_name(query, fields=None): + # import pandas + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # seaweed1 = db1.seaweed + seawater105 = get_collection("fangjia", "seaweed", "192.168.0.136", 27017) + for err_count in range(0, 10): + + try: + + lt = seawater105.count(query) + + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + return data + + except: + + # err_count = err_count + 1 + lt = seawater105.count(query) + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + return data diff --git a/PriceSystem/fangjia/mongoClients.py b/PriceSystem/fangjia/mongoClients.py new file mode 100644 index 0000000..3b08fce --- /dev/null +++ b/PriceSystem/fangjia/mongoClients.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Nov 15 22:45:06 2017 + +@author: wenyan +""" + +from pymongo import MongoClient + + +class MongoClients: + + cache = {} + + def __init__(self, db, name, host="127.0.0.0", port=27017): + + key = db + "_" + name + "_" + host + "_" + str(port) + if key in MongoClients.cache.keys(): + self.collection = MongoClients.cache[key] + else: + print("init " + key) + collection = MongoClient(host, port).get_database(db).get_collection(name) + MongoClients.cache[key] = collection + self.collection = MongoClients.cache[key] + """ + def find(self, *args, **kwargs): + return self.collection.find(*args, **kwargs) + + def find2(self, query, fields, call): + + for item in self.collection.find(query): + call(item) + """ diff --git a/PriceSystem/fangjia/seaweed.py b/PriceSystem/fangjia/seaweed.py new file mode 100644 index 0000000..91e7207 --- /dev/null +++ b/PriceSystem/fangjia/seaweed.py @@ -0,0 +1,8 @@ +class CalcCommand: + + def __init__(self, city, region, name, weekend): + self.city = city + self.region = region + self.name = name + self.weekend = weekend + diff --git a/PriceSystem/sell_price/__init__.py b/PriceSystem/sell_price/__init__.py new file mode 100644 index 0000000..5088ba1 --- /dev/null +++ b/PriceSystem/sell_price/__init__.py @@ -0,0 +1,5 @@ +import sell_price.zhuge0118A as sellPrice + + +def calc_sell_price(cmd): + sellPrice.out_put_price(cmd["city"], cmd["region"], cmd["name"], cmd["weekend"]) diff --git a/PriceSystem/sell_price/zhuge0118A.py b/PriceSystem/sell_price/zhuge0118A.py new file mode 100644 index 0000000..f9c6750 --- /dev/null +++ b/PriceSystem/sell_price/zhuge0118A.py @@ -0,0 +1,1424 @@ + # -*- coding: utf-8 -*- +""" +Created on Tue Jan 2 13:24:07 2018 + +@author: Administrator +""" + +# -*- coding: utf-8 -*- +""" +Created on Fri Sep 22 18:18:15 2017 + +@author: Administrator +""" + +import os +print( os.getcwd()) +import pymongo +import numpy as np +import pandas as pd +import random +import collections +import pymongo +import sklearn.cluster +import pandas +import numpy +import sys, time +import pp +import zhuge +import fangjia +import sklearn.svm +import datetime +from fangjia import timer + +#import yesterday + +def front_step_time(day): + now = datetime.datetime.now() + front=now-datetime.timedelta(days=day) + d1= front.strftime('%Y%m%d') + return int(d1) + + + +#提取房源的数据时间上限,如从20170301之后的数据 +some_district_information_time=front_step_time(360) + +#提周边小区均价的数据时间上限,如从20170801之后的数据 +zhoubian_xiaoqu_time=front_step_time(200) + + +#第三方均价的数据时间上限,如从201703之后的数据 +dsfjujia_time=front_step_time(430)//100 + +############################################################ +############################################################ +#存到数据库的日期 + + +""" +def weekend00(): + + today = datetime.datetime.now() + now_time=datetime.datetime(today.year, today.month, today.day, 0, 0, 0) + week_time = now_time.weekday() + + + if week_time>=4: + future_time = now_time-datetime.timedelta(days=week_time+1-7) + d1= future_time.strftime('%Y%m%d') + weekend1= datetime.datetime.strptime(d1,'%Y%m%d') + weekend=weekend1+datetime.timedelta(hours=-8) + return weekend + else: + future_time = now_time-datetime.timedelta(days=week_time+1) + d1= future_time.strftime('%Y%m%d') + weekend1= datetime.datetime.strptime(d1,'%Y%m%d') + weekend=weekend1+datetime.timedelta(hours=-8) + return weekend + +""" + + +#date="2018-02-04" +############################################################ +############################################################ + +credit=pandas.Series([0.9,0.8,0.7,0.6,0.5],index=['0.9','0.8','0.7','0.6','0.5']) + +value_type=pandas.Series([1,2,3,4,5,6,7], + index=['指导价','自身样本','第三方','交易数据','周边楼盘','推导','优质样本']) + +#1:指导价 2:自身样本 3:第三方 4:交易数据 5:周边楼盘, 6:推导 + +@timer +def commonprice(ct,rl,p): + + + + def add_block(ppd): + if 'block' not in ppd.columns: + ppd['block']='没有' + return ppd + + def add_room(ppd): + if 'room' not in ppd.columns: + ppd['room']= 2 + return ppd + + @timer + def some_district_information(ct,rl,lk): + + + query = {"city":ct,"cat":"sell","region":rl, + "district_name":{"$in":lk}, + "p_date":{"$gt":some_district_information_time}} + + return fangjia.findHouse(query) + """ + return fangjia.find_house_from_elastic_search(ct, rl, lk, + "lianjia.com", some_district_information_time) + """ + + def disanganf_price(ct,rl,lk,tt): + #db1 = client1.fangjia + #dsf = db1.average_statistics + query = {"city":ct,"region":rl, + "name":lk,"s_date":{"$gt":tt}} + #pos = list(dsf136.find(query)) + + #data1=pandas.DataFrame(pos) + data1=fangjia.find_disanganf_price(query) + + if data1.shape[0]>1: + a_set={'fangjia_price'} + b_set=set(data1.columns) + if a_set & b_set==a_set: + + data=data1[data1['credit']>0.7] + if data.shape[0]>=1: + choose_class=['city','region','name','fangjia_price','credit', + 's_date'] + dc=data[choose_class] + return dc + else: + return numpy.array([]) + else: + + return numpy.array([]) + else: + return numpy.array([]) + + + + + def take_location(ct,rl,p): + + dirlt=p[0] + + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + seaweed136=fangjia.get_collection("fangjia","seaweed", '192.168.0.136',27017) + query1 = {"status":0,"cat":"district","city":ct, + "region":rl, "name":dirlt} + + for err_count in range(0, 10): + + try: + ltt= seaweed136.count(query1) + if ltt>0: + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1,"cat":1,"name":1} + lct= seaweed136.find(query1, fields1) + lf=pandas.DataFrame(list(lct)) + + #lf=fangjia.find_seaweed(query1, fields1) + le=lf + le.index=le['name'] + cluns=lf.columns + if ('lng2' in cluns )and ('lat2' in cluns): + lr=le[['lng2','lat2']] + del lf,le,cluns + return lr + else: + del lf,le,cluns + return numpy.array([]) + + else: + return numpy.array([]) + except: + + ltt= seaweed136.count(query1) + if ltt>0: + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1,"cat":1,"name":1} + lct= seaweed136.find(query1, fields1) + lf=pandas.DataFrame(list(lct)) + + #lf=fangjia.find_seaweed(query1, fields1) + le=lf + le.index=le['name'] + cluns=lf.columns + if ('lng2' in cluns )and ('lat2' in cluns): + lr=le[['lng2','lat2']] + del lf,le,cluns + return lr + else: + del lf,le,cluns + return numpy.array([]) + + else: + return numpy.array([]) + + def lianjia(ur): + #print(ur) + if 'lianjia' in ur: + return 0 + else: + return 1 + + + def zhuge(ur): + if 'zhuge' in ur: + return 0 + else: + return 1 + + def wawj(ur): + if '5i5j' in ur: + return 0 + else: + return 1 + + def iwjw(ur): + if 'iwjw' in ur: + return 0 + else: + return 1 + + def fangdd(ur): + if 'fangdd' in ur: + return 0 + else: + return 1 + + def wkzf(ur): + if 'wkzf' in ur: + return 0 + else: + return 1 + + + + def label(dd,f): + aa=list() + + for i in dd.index: + d2f=dd['url'][i] + if type(d2f)==str: + if f(dd['url'][i])==0: + aa.append(i) + return set(aa) + + + + def new_data(ds): + bb=set(ds['name']) + bb1=list(bb) + x1=pandas.Series() + for i in bb1: + a1=ds[ds['name']==i] + a2=a1.sort_values(by='s_date',ascending=False) + x1[i]=a2.iloc[0]['fangjia_price'] + return x1 + + + def zhoubian_xiaoqu(ct,rl,dirst): + + #提取周边小聚的均价,本方法不用房源,直接用上一批数据的价格来计算无房源小区 + ################### + ################### + tii=zhoubian_xiaoqu_time + ################### + ################### + + tt=tii//100-1 + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #dsf = db1.average_statistics + query = {"city":ct,"region":rl, + "name":{"$in":dirst},"s_date":{"$gt":tt}} + #pos = list( dsf136.find(query)) + + #data=pandas.DataFrame(pos) + data=fangjia.find_disanganf_price(query) + #print(data) + if data.shape[0]>=1: + #data=add_block(data) + data=add_room(data) + + a_set={'city','region','name','fangjia_price','credit', + 's_date'} + b_set=set(data.columns) + + if a_set & b_set==a_set: + + choose_class=['city','region','name','fangjia_price','credit', + 's_date'] + dc=data[choose_class] + return dc + else: + return numpy.array([]) + return numpy.array([]) + else: + + return numpy.array([]) + + + def hh(ppp,df): + ppp['price']=None + aa=list(df.index) + #print('aa=',aa) + for i in ppp.index: + kk=ppp['district_name'][i] + #print('kk=',kk) + if kk in aa: + #ppp['price'][i]=df[kk] + ppp.loc[i,'price'] = df[kk] + #print('df[kk]',df[kk]) + return ppp + + + @timer + def neighbor(lng,lat,scale,ct,rl,p): + + ################### + ################### + #timm=20171122 + ################### + ################### + + estype=p[2] + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + kkk=scale/1000 + longtitude=lng + latitude=lat + + query1 = {"status":0,"cat":"district", + "lat2":{"$gt":latitude-kkk*10**10/0.86,"$lt":latitude+kkk*10**10/0.86}, + "lng2":{"$gt":longtitude-kkk*10**10,"$lt":longtitude+kkk*10**10}} + #fields1 = {"lat2":1,"lng2":1, "city":1,"region":1,"cat":1,"name":1} + + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, + "cat":1,"name":1,'estate_type2':1} + + #lct= list( seaweed136.find(query1, fields1)) + + #lf=pandas.DataFrame(lct) + lf=fangjia.find_seaweed(query1, fields1) + + le=le=lf[lf['estate_type2']==estype] + + le.index=le['name'] + + lr=le[['lng2','lat2']] + + + dirst1=list(lr.index) + + random.shuffle(dirst1) + + dirst=dirst1[:80] + #print('dirst=',dirst) + + dirst_plus=dirst+[p[0]] + + query_feature = {"city":ct,"region":rl, + "district_name":{"$in":dirst_plus}} + + ppp=fangjia.dis_feature(query_feature) + #print('ox'*10,ppp) + + #print('小区个数= ',len(dirst)) + df1=zhoubian_xiaoqu(ct,rl,dirst) + #print('df1=',df1,'*'*20) + if df1.shape[0]>1: + df=new_data(df1) + #print('df=',df) + vv=hh(ppp,df) + #print('#'*36) + #print('vv=',vv) + return vv + else: + return numpy.array([]) + + + def svr(vv,lk): + #提取要计算小区的特征 + vk=vv[vv['district_name']==lk] + #删除要计算小区的空特征 + #print('vk=',vk) + if vk.shape[0]>0: + + vk1=vk.iloc[0].dropna() + + #将要计算小区的特征提取出来 + + vk2=list( set(vk1.index)-{'_id','district_name','region','city'}) + + #print('vk2=',vk2) + if len(vk2)>1: + vvo=vv[vv['district_name']!=lk] + vk3=vk2+['price'] + vk31=list(set(vk3)) + vvo1=vvo[vk31].dropna() + + x1 =vvo1.drop('price',axis=1) + #print('x1=',x1) + y = vvo1['price'] + #print('y',y) + + if len(x1)>2: + + clf = sklearn.svm.SVR() + clf.fit(x1, y) + pp1= vk[vk31].drop('price',axis=1) + result = clf.predict(pp1 ) + #print (result) + return int(result[0]) + else: + km=vv['price'].dropna() + return int(k_means(km)) + + + else: + km=vv['price'].dropna() + return int(k_means(km)) + + else: + km=vv['price'].dropna() + return int(k_means(km)) + + + @timer + def no_source_price(ct,rl,location,p): + lk=p[0] + hg=location.shape[0] + + if hg==0: + return '没有经纬度数据',0 + elif hg>1: + return '纬度数据有误,有多个经纬度',0 + else: + #t1=time.time() + dd=neighbor(location['lng2'][0],location['lat2'][0],5,ct,rl,p) + #print(time.time()-t1,'t1'*10) + #print('dd=',dd,'dd'*20) + if dd.shape[0]>=20: + #print('$'*50) + #dd1=neighbor(location['lng2'][0],location['lat2'][0],5) + #print('dd',dd) + #svr_time=time.time() + other=svr(dd,lk) + #print(time.time()-svr_time,'svr'*10) + print('other= ' ,other,'9'*20) + return other,dd.shape[0] + else: + #t2=time.time() + dd1=neighbor(location['lng2'][0],location['lat2'][0],10,ct,rl,p) + #print(time.time()-t2,'t2'*10) + #print('dd1=',dd1) + if dd1.shape[0]>=20: + #print('$'*50) + + + #svr_time=time.time() + other1=svr(dd1,lk) + #print(time.time()-svr_time,'svr'*10) + #print('other1=',other1) + return other1,dd1.shape[0] + else: + #t3=time.time() + dd2=neighbor(location['lng2'][0],location['lat2'][0],50,ct,rl,p) + #print(time.time()-t3,'t3'*10) + print('dd2.shape',dd2.shape[0],'@'*60) + if dd2.shape[0]>=20: + print('$'*50) + svr_time=time.time() + other1=svr(dd2,lk) + print(time.time()-svr_time,'e'*10) + #print('other1=',other1) + return other1,dd2.shape[0] + else: + dd3=neighbor(location['lng2'][0],location['lat2'][0],100,ct,rl,p) + #print('dd2=',dd2,'@'*60) + if dd3.shape[0]>=20: + #svr_time=time.time() + other1=svr(dd3,lk) + #print(time.time()-svr_time,'svr'*10) + #print('other1=',other1) + return other1,dd3.shape[0] + else: + + return '没有周围小区数据' ,0 + + + def region_district_list(ct,rl): + #import pandas + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + + query1 = {"status":0,"cat":"district","city":ct,"region":rl} + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, + "cat":1,"name":1,'estate_type2':1,'alias':1} + + #lct= list(seaweed136.find(query1, fields1)) + #lf=pandas.DataFrame(lct) + lf=fangjia.find_seaweed(query1, fields1) + return lf + + + + + def k_means(pp1): + from sklearn.cluster import KMeans + pv=list(pp1) + if len(set(pv))>3: + gf=numpy.array([pv]).T + estimator = KMeans(n_clusters=3)#构造聚类器 + estimator.fit(gf)#聚类 + label_pred = estimator.labels_ #获取聚类标签 + aa=collections.Counter(label_pred) + v=pandas.Series(aa) + gg=list(v) + index_max=gg.index(max(gg)) + + centroids = estimator.cluster_centers_ #获取聚类中心 + #inertia = estimator.inertia_ # 获取聚类准则的总和 + center=centroids[index_max][0] + return (int(center)) + else: + return int(pp1.mean()) + + + def region_district_list_name(ct,rl,lkk): + #import pandas + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + + query1 = {"status":0,"cat":"district","city":ct,"region":rl,"name":lkk} + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, + "cat":1,"name":1,'estate_type2':1,'alias':1} + + #lct= list( seaweed136.find(query1, fields1)) + + + #lf=pandas.DataFrame(lct) + lf=fangjia.find_seaweed(query1, fields1 ) + return lf + + + def strip_villa(ct,rl,p): + lw=p[0] + #estype=p[2] + alias=p[1] + + #print('alias=',alias) + + def filt(x): + + return type(x)==str and len(x)>1 + + bb=list(filter(filt,alias)) + #print('bb=',bb) + bb.extend([lw]) + #lk1=list(set(lk)) + + #print('本小区名=',lw) + + #print('type(lw)=',type(bb)) + + #print('小区名列表=',bb) + + cc1= some_district_information(ct,rl,bb) + + cc1= add_room(cc1) + + print('cc1本小区数据条数=',cc1.shape[0]) + + if cc1.shape[0]>2: + + del lw,alias,bb + return cc1 + else: + + if '别墅' in lw: + + lkk=lw.strip('别墅') + cc=region_district_list_name(ct,rl,lkk) + #print('cc=',cc) + if cc.shape[0]>1: + + cc1=list(cc['alias'].dropna()) + cc2=list(filter(filt,alias)) + bb.extend([lkk]) + #clk1=list(set(clk)) + cc2=some_district_information(ct,rl,bb) + if cc2.shape[0]>2: + del lw,alias,bb,cc1,cc + return cc2 + else: + del lw,alias,bb,cc1,cc + return cc1 + else: + return cc1 + else: + return cc1 + + + def consider_villa(ct,rl,p): + + #print('villa p=',p) + + estype=p[2] + df=strip_villa(ct,rl,p) + if df.shape[0]>1: + + if estype=='普通住宅': + all_index=set(df.index) + vv=fetch_villa_index(df) + ww=all_index-vv + + d1=df.loc[ww,:] + if d1.shape[0]>3: + return d1 + else: + return df + elif estype=='别墅': + vv=fetch_villa_index(df) + d1=df.loc[vv,:] + if d1.shape[0]>3: + return d1 + else: + return df + else: + return df + else: + return df + + + def fetch_villa_index(dd): + + #dd=dd.dropna() + d1=dd[dd['area']>150] + + #print(dd) + + + villa_label=["独栋", "联排", "叠加", "双拼", + "花园面积", "泳池", "别墅", "带花园"] + bb=list() + for i in d1.index: + we=d1['name'][i] + for j in villa_label: + if j in we: + bb.append(i) + d21=dd[dd['room']>=6] + d2=d21[d21['area']>=360] + + d3=dd[dd['area']>500] + + villa=set(bb)|set(d2.index)|set(d3.index) + + return villa + + + + + + + def dsfjujia(ct,rl,lk): + + query = {"city":ct,"region":rl, + "name":lk,"s_date":{"$gt":dsfjujia_time}} + + data=fangjia.find_disanganf_price(query) + + alist=['anjuke_esf_price','ganji_esf_price', + 'lianjia_esf_price','soufang_esf_price', + 'tuboshi_esf_price','zhugezhaofang_esf_price', + 'yungujia_esf_price','fanggugu_esf_price' + ] + a1list=set(alist) + blist=set(data.columns) + + clist=a1list & blist + + ddq=list(clist) + + data1=data[ddq] + + ser=pandas.Series() + + for i in ddq: + pdd=data1[i].dropna() + #ser[i]=k_means(pdd) + ser[i]=list(pdd)[-1] + #print('采用抓取的第三方均价') + return ser + + + + + def sfjj(ll_zz,aa_zz,dddf): + + dxx=pandas.Series() + for i in ll_zz: + if i in aa_zz: + dxx[i]=dddf[i] + return int(dxx.mean()) + """ + tt=timm//100 + + lk=p[0] + dsfp1=disanganf_price(ct,rl,lk,tt) + + data= consider_villa(ct,rl,p,timm) + """ + + ####################################################### + """ + junjia=dsfjujia(ct,rl,lk) + ll_zz={'zhugezhaofang_esf_price','lianjia_esf_price'} + aa_zz=set(junjia.index) + jiaji_lz= list(ll_zz & aa_zz) + """ + + def k_means_clus(pp1,clus): + + from sklearn.cluster import KMeans + pv=list(pp1) + if len(set(pv))>clus: + gf=numpy.array([pv]).T + estimator = KMeans(n_clusters=clus)#构造聚类器 + estimator.fit(gf)#聚类 + label_pred = estimator.labels_ #获取聚类标签 + aa=collections.Counter(label_pred) + v=pandas.Series(aa) + gg=list(v) + index_max=gg.index(max(gg)) + + centroids = estimator.cluster_centers_ #获取聚类中心 + #inertia = estimator.inertia_ # 获取聚类准则的总和 + center=centroids[index_max][0] + return (int(center)) + else: + return int(pp1.mean()) + + def group(junjia_lzf,junjia_anst,ctr): + + bb=[] + for i in junjia_lzf.index: + #print(junjia_lzf) + someprice=junjia_lzf[i] + def compare15(b): + if 100*abs(someprice-b)/b<16: + return 1 + else: + return 0 + countmap=list(map(compare15,junjia_anst)) + #print('countmap=',countmap) + countmap1=pandas.Series(countmap) + countmap2=countmap1[countmap1>0] + #print('countmap2=',countmap2) + + numb=len(countmap2) + #print('len(countmap2)=',len(countmap2)) + if numb>ctr: + bb.append([someprice,numb]) + return bb + + def middle_operate(junjia_lzf,junjia_anst): + + #print('%%%',ppd3['total_price']) + + #print('junjia_lzf=',junjia_lzf) + #print('junjia_anst=',junjia_anst) + + + def mid_out_1(junjia_lzf): + + if len(junjia_lzf)>1: + #print('aa1',aa) + bb=group(junjia_lzf,junjia_lzf,1) + #print('bb=',bb) + if len(bb)>0: + bb1=pandas.DataFrame(bb,columns=['price','count']) + bb2=bb1.sort_values(by='count',ascending=False) + #print(bb2) + return bb2.iloc[0]['price'] + else: + return None + else: + return None + + + def mid_out_2(junjia_lzf,junjia_anst): + + if len(junjia_lzf)>0 and len(junjia_anst)>0: + + bb=group(junjia_lzf,junjia_anst,0) + #print('bb1=',bb) + if len(bb)>0: + bb1=pandas.DataFrame(bb,columns=['price','count']) + bb2=bb1.sort_values(by='count',ascending=False) + #print(bb2) + return bb2.iloc[0]['price'] + else: + return None + else: + return None + a11=mid_out_1(junjia_lzf) + a22=mid_out_2(junjia_lzf,junjia_anst) + a33=mid_out_1(junjia_anst) + aa1_3=pandas.Series([a11,a22,a33]).dropna() + #print(a11,a22,a33) + if len(aa1_3)>0: + return aa1_3.iloc[0] + else: + return None + + + def devide_group(al,bl,junjia): + + alls=set(junjia.index) + def set_div(ds,alls): + if len(set(ds) & alls)>0: + return junjia[ds].dropna() + else: + return pandas.Series() + lzf=set_div(al,alls) + anst=set_div(bl,alls) + + return lzf , anst + + @fangjia.timer + def dglg_out1(junjia,neighb): + + """ + alist=['anjuke_esf_price','ganji_esf_price', + 'lianjia_esf_price','soufang_esf_price', + 'tuboshi_esf_price','zhugezhaofang_esf_price', + 'yungujia_esf_price','fanggugu_esf_price' + ] + """ + + + al=[ + 'lianjia_esf_price', + 'zhugezhaofang_esf_price', + 'fanggugu_esf_price' + ] + bl=['anjuke_esf_price', + 'soufang_esf_price', + 'tuboshi_esf_price', + ] + + #ppd3dp=ppd3['total_price'].dropna() + if type(neighb)!=str and neighb!=None : + + junjia=junjia[junjia>0.55*neighb] + ''' + junjia_lzf=junjia[al].dropna() + junjia_anst=junjia[bl].dropna() + ''' + alldi=devide_group(al,bl,junjia) + + junjia_lzf=alldi[0] + junjia_anst=alldi[1] + dg11=middle_operate(junjia_lzf,junjia_anst) + return dg11 + + + else: + alldi=devide_group(al,bl,junjia) + + junjia_lzf=alldi[0] + junjia_anst=alldi[1] + + dg22=middle_operate(junjia_lzf,junjia_anst) + return dg22 + + + lk=p[0] + location=take_location(ct,rl,p) + + #ttp=time.time() + neighb=no_source_price(ct,rl,location,p) + + #print(time.time()-ttp,'m'*20) + + junjia=dsfjujia(ct,rl,lk) + #print('junjia=',junjia) + + ''' + dglg=dsplg(junjia,neighb,2) + dggg=dsplg(junjia,neighb,1) + ''' + dglg=dglg_out1(junjia,neighb[0]) + #dggg=dglg_out1(junjia,neighb[0]) + + #print('dglg=',dglg) + #print('dggg=',dggg) + + #tt=time.time() + data= consider_villa(ct,rl,p) + #print(time.time()-tt,'<>'*26) + ####################################################### + + if data.shape[0]>3: + + paa=add_block(data) + ppd=paa[['total_price','area','region','district_name','url','block','p_date']] + #print('ppd##############') + #print('ppd=',ppd) + ppd=ppd.fillna(method='ffill') + + ppd1=ppd.dropna() + + #t3=time.time() + + ppd1['total_price']=ppd1['total_price']/ppd1['area'] + #print('ppd1time',time.time()-t3,'o'*36) + + #ppd1.loc['total_price']=ppd1['total_price']/ppd1['area'] + + #ppd2=ppd1 + + + + #if type(neighb) is not str: + + if ppd1.shape[0]>0: + + def guolv_data(ppd2,neighb): + if type(neighb) is not str: + ph=ppd2[ppd2['total_price']>0.5*neighb] + return ph + else: + return ppd2 + + #ppd3=ppd2[ppd2['total_price']>0.6*neighb] + ###################################### + ppd3=guolv_data(ppd1,neighb[0]) + + ###################################### + ldf=label(ppd3,lianjia) + #print('ldf=',ldf) + + ldf1=label(ppd3,zhuge) + + ldf2=label(ppd3,wawj) + + ldf3=label(ppd3,iwjw) + ldf4=label(ppd3,fangdd) + + ldf5=label(ppd3,wkzf) + l_and_z=ldf|ldf1 + #print(l_and_z) + #print('l_and_z=',l_and_z) + allf=l_and_z|ldf3|ldf4|ldf5|ldf2 + #print('l_and_z_and_i=',l_and_z_and_i) + #mm=0 + length=len(l_and_z) + length1=len(allf) + + + if dglg != None: + + #print('junjia.shape',len(junjia)) + sample_size=len(junjia) + return dglg,float('%.2f' % credit['0.8']),value_type['第三方'],sample_size + + elif length>3: + #print('ldf=',l_and_z_and_i) + pppk=ppd3.loc[l_and_z,:] + + ppp=pppk[['total_price','p_date']] + #print(ppp) + ppp1=ppp.sort_values(['p_date'], ascending=False) + + gdf=ppp1['total_price'][:36].dropna() + #print('链家和诸葛找房合并后的数据集=',gdf) + sample_size=len(gdf) + gm=k_means_clus(gdf,2) + print('链猪= ',gm) + return int(gm),float('%.2f' % credit['0.8']),value_type['自身样本'] ,sample_size #优质样本 + + + elif data.shape[0]>0: + + print('诸葛找房算法') + from zhuge import zhuge_consist + data['avg']=data['total_price']/data['area'] + + def guolv_data11(data,neighb): + if type(neighb) is not str: + + print('周边价格=',neighb) + ddd11=data[data['avg']>0.5*neighb] + return ddd11 + else: + return data + + ddd11=guolv_data11(data,neighb[0]) + + if ddd11.shape[0]>2: + + + s=zhuge_consist.url_lable(ddd11) + s1=zhuge_consist.drop_rubbish(s) + #print('s1=',s1) + s2=set(s1) + #print('s2=',s2) + if len(s2)>3: + + from zhuge import zhuge_consist + + s=zhuge_consist.zhuge_out(ddd11) + + def compelex(ddd11,s,neighb,junjia): + cc1=ddd11['avg'].dropna() + + + kmzhuge=k_means(cc1) + + junjia['zhuge_logic']=s + + junjia['kmzg']=kmzhuge + + if type(neighb)!=str and (neighb)!=None: + + junjia['neighb']=neighb + #print('junjia',junjia) + + junjia_front=junjia[['zhuge_logic','kmzg']] + cc=group(junjia_front,junjia,1) + #junjia_km=k_means(junjia) + #print('cc=',cc) + ############ + if len(cc)>0: + cc1=pandas.DataFrame(cc,columns=['price','count']) + cc2=cc1.sort_values(by='count',ascending=False) + #print(bb2) + sample_size=ddd11.shape[0] + return cc2.iloc[0]['price']*1.0,float('%.2f' % credit['0.7']),value_type['第三方'],sample_size + else: + sample_size=ddd11.shape[0] + return junjia['kmzg']*1.0,float('%.2f' % credit['0.6']),value_type['自身样本'],sample_size + + + junjia_km=compelex(ddd11,s,neighb[0],junjia) + + return junjia_km + + + + else: + print('*'*20,'采用周边算法') + if ddd11.shape[0]>3: + cc1=ddd11['avg'].dropna() + kmzhuge=k_means_clus(cc1,2) + sample_size=ddd11.shape[0] + return kmzhuge,float('%.2f' % credit['0.6']),value_type['自身样本'],sample_size + else: + return neighb[0],float('%.2f' % credit['0.5']),value_type['周边楼盘'],neighb[1] + + elif dglg != None: + print('优质第三方均价2 = ',dglg) + sample_size=len(junjia) + return dglg,float('%.2f' % credit['0.8']),value_type['第三方'] ,sample_size #'优质样本' + + else: + return neighb[0],float('%.2f' % credit['0.5']),value_type['周边楼盘'],neighb[1] + + elif dglg != None: + print('优质第三方均价3 = ',dglg) + sample_size=len(junjia) + return dglg,float('%.2f' % credit['0.8']),value_type['第三方'] #'优质样本' + + else: + + return neighb[0],float('%.2f' % credit['0.5']),value_type['周边楼盘'],neighb[1] + + + elif dglg != None: + print('优质第三方均价4 = ',dglg) + return dglg,float('%.2f' % credit['0.8']),value_type['第三方'] ,len(junjia) #'优质样本' + + else: + + #print('*'*100) + #print(neighb) + return neighb[0],float('%.2f' % credit['0.5']),value_type['周边楼盘'],neighb[1] + + + + +client65 = pymongo.MongoClient('192.168.0.105',27777) +db65 = client65.fangjia +seawater65 = db65.seawater + +client136 = pymongo.MongoClient('192.168.0.136',27017) +db136 = client136.fangjia +dsf136 = db136.average_statistics +seaweed136 = db136.seaweed + + +dsf = db136.district_stat + +#client136 = pymongo.MongoClient('192.168.0.136',27017) +#db136 = client136.fangjia + + + + + +def region_district_list(ct,rl): + #import pandas + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + + query1 = {"status":0,"cat":"district","city":ct,"region":rl} + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, + "cat":1,"name":1,'estate_type2':1,'alias':1} + + #lct= list(seaweed136.find(query1, fields1)) + #lf=pandas.DataFrame(lct) + lf=fangjia.find_seaweed(query1,fields1) + return lf + +def city_list(): + #client = pymongo.MongoClient('192.168.0.136',27017) #'192.168.0.65',27777 + #db = client.fangjia + city = db136.server_area + #seawater.find_one() + + + pos= city.find({"display":{"$ne":"全国"}}, + {"display":1,"city_grade":1,"_id":1}).sort([("city_grade",1),("_id",1)]) + + + + data=pandas.DataFrame(list(pos)) + #print(data) + return data['display'] + + +#print(some_district_information()) + + + +def region_list(city): + #client = pymongo.MongoClient('192.168.0.136',27017) + #db = client.fangjia + region = db136.region_block + #seawater.find_one() + query = {"city":city,"category":"region"} + fields1 = {"name":1} + pos = list(region.find(query,fields1)) + + + data=pandas.DataFrame(pos) + return data['name'] + + + + + + + + +def add_alias(ppd): + if 'alias' not in ppd.columns: + ppd['alias']='无' + return ppd + +def ghhh(kkg): + kkg=kkg.fillna('无') + a=list() + for i in kkg.index: + b=list(kkg.loc[[i],:].values.flatten()) + a.append(b) + return a + + + + +client1 = pymongo.MongoClient('192.168.10.182',27017) +db1 = client1.fangjia +stat = db1.distrit_stat + +pt=client1.fangjia_log.calc_command + + +################################################################# +@timer +def region_district_list_lk(ct,rl,lk): + #import pandas + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + + query1 = {"status":0,"cat":"district","city":ct,"region":rl,"name":lk} + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, + "cat":1,"name":1,'estate_type2':1,'alias':1} + + #lct= list(seaweed136.find(query1, fields1)) + #lf=pandas.DataFrame(lct) + lf=fangjia.find_seaweed(query1,fields1) + return lf + + + + + +def last_sunday(weekend): + #import datetime + """ + today = datetime.datetime.now() + + today=today+datetime.timedelta(days=-4) + + now_time=datetime.datetime(today.year, today.month, today.day, 0, 0, 0) + week_time = now_time.weekday() + """ + future_time = weekend-datetime.timedelta(days=7-1) + d1= future_time.strftime('%Y%m%d') + weekend1= datetime.datetime.strptime(d1,'%Y%m%d') + weekend=weekend1+datetime.timedelta(hours=-8) + return weekend + + + + + + + +#last_sunday=last_sunday(weekend) + +def make_up_price(ct,rl,lk,weekend): + + + #last_sunday=last_sunday(weekend) + #save_lg=client136.fangjia.district_stat + if lk!=None: + + #cc=last_sunday + + + ly=last_sunday(weekend) + #stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) + stat=dsf + query = {"city":ct,"region":rl,"district_name":lk,'weekend':ly + } + + + pos1=list(stat.find(query).limit(30)) + + pos=pandas.DataFrame(pos1) + + #pos=timestamp_to_int(pos) + #pos=pos[pos['week1']>cc-5] + #pos=pos.sort_values(by='week1',ascending=False) + + #print('5'*10) + #print(pos) + + if pos.shape[0]>0: + #pp1""" + pp1=pos['avg_price'].iloc[0] + if type(pp1)!=None and type(pp1)!=str: + + + + return int(pp1) + else: + return None + else: + return None + + else: + + #stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) + stat=dsf + query = {"city":ct,"region":rl,"district_name":lk,'weekend':last_sunday + } + + + pos1=list(stat.find(query).limit(30)) + + pos=pandas.DataFrame(pos1) + + #pos=timestamp_to_int(pos) + #pos=pos[pos['week1']>cc-5] + #pos=pos.sort_values(by='week1',ascending=False) + + #print('5'*10) + #print(pos) + + if pos.shape[0]>0: + #pp1 + pp1=pos['avg_price'].iloc[0] + if type(pp1)!=None and type(pp1)!=str: + + + + return int(pp1) + else: + return None + else: + return None + + + + +#weekend=weekend00() + +@fangjia.timer +def out_put_price(ct,rl,lk,weekend): + lf=region_district_list_lk(ct,rl,lk) + lf=add_alias(lf) + + p=ghhh(lf[['name','alias','estate_type2']])[0] + + mm=commonprice(ct,rl,p) + + c_date = datetime.datetime.utcnow() + m_date=datetime.datetime.utcnow() + + if type(mm[0])!=str: + + print(ct,rl,lk,mm,'$'*20) + #print(lf['estate_type2'].iloc[0],'*'*10) + estate_type=lf['estate_type2'].iloc[0] + + + + #last_sunday=last_sunday() + #t1=time.time() + last_sunday_price=make_up_price(ct,rl,lk,weekend) + #print(time.time()-t1,'*'*20) + + #print('last_sunday_price',last_sunday_price) + #print('mm[0]',mm[0]) + + def make_up(): + if type(last_sunday_price) != str and last_sunday_price != None and mm[0]!=None and type(mm[0])!=str: + + make_up=10000*(mm[0]-last_sunday_price)/last_sunday_price + return make_up + else: + return 0 + + make_up=int(make_up()) + + + try: + + stat.replace_one( + + {"city":ct,"region":rl,"district_name":p[0],"weekend":weekend}, + + {"city":ct,"region":rl,"district_name":p[0],'credit':mm[1], + "calc_value":int(mm[0]), "weekend":weekend,"c_date":c_date,'avg_price':int(mm[0]), + "m_date":m_date,'value_type':int(mm[2]),'weekend_markup':make_up, + 'changed_markup':make_up,'markup':make_up,'sample_size':mm[3], + 'estate_type':estate_type},True + ) + + + + pt.replace_one( + + {"city":ct,"region":rl,"district_name":p[0],"weekend":weekend}, + + + {"city":ct,"region":rl,"district_name":p[0],"c_date":c_date,'status':int(0), + 'remark':'week','category':'sell','weekend':weekend,'priority':int(0), + 'problem':None, + },True + ) + + + except: + + stat.replace_one( + + {"city":ct,"region":rl,"district_name":p[0],"weekend":weekend}, + + {"city":ct,"region":rl,"district_name":p[0],'credit':mm[1], + "calc_value":int(mm[0]), "weekend":weekend,"c_date":c_date,'avg_price':int(mm[0]), + "m_date":m_date,'value_type':int(mm[2]),'weekend_markup':make_up, + 'changed_markup':make_up,'markup':make_up,'sample_size':mm[3], + 'estate_type':estate_type},True + ) + + else: + pt.replace_one( + + {"city":ct,"region":rl,"district_name":p[0],"weekend":weekend}, + + + {"city":ct,"region":rl,"district_name":p[0],"c_date":c_date,'status':int(1), + 'remark':'week','category':'sell','weekend':weekend,'priority':int(0), + 'problem':mm[0], + },True + ) + + + \ No newline at end of file diff --git a/PriceSystem/test.py b/PriceSystem/test.py new file mode 100644 index 0000000..4d09d21 --- /dev/null +++ b/PriceSystem/test.py @@ -0,0 +1,12 @@ +import zhuge0118A +import time +s = time.time() +ct='上海' +rl='奉贤' +lk='杨王村公寓' + + +zhuge0118A.out_put_price(ct, rl, lk) +zhuge0118A.out_put_price(ct, rl, lk) +e = time.time() +print(e -s) diff --git a/PriceSystem/zhuge/__init__.py b/PriceSystem/zhuge/__init__.py new file mode 100644 index 0000000..4d78a3e --- /dev/null +++ b/PriceSystem/zhuge/__init__.py @@ -0,0 +1 @@ +from zhuge import zhuge2 \ No newline at end of file diff --git a/PriceSystem/zhuge/zhuge2.py b/PriceSystem/zhuge/zhuge2.py new file mode 100644 index 0000000..ec61f18 --- /dev/null +++ b/PriceSystem/zhuge/zhuge2.py @@ -0,0 +1,487 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue Nov 28 13:15:49 2017 + +@author: Administrator +""" +import pymongo +import pandas +from collections import Counter +import numpy +import collections +import re + +def url_lable(dmi1): + + company_list=['lianjia','zhuge','5i5j','iwjw','fangdd','centanet', + 'cityhouse','esf','baixing','ganji','anjuke','58'] + + for p in company_list: + #print('p=',p) + for k in dmi1.index: + #print('k=',k) + #print(dmi1['url'][k]) + ff=dmi1['url'][k] + if type(ff)==str: + + if p in dmi1['url'][k]: + dmi1.loc[k,'url']=p + return dmi1 + + +def drop_rubbish(dd1): + + for i in dd1.index: + pp=dd1['url'][i] + + # print(pp) + + if type(pp)==str: + + if len(pp)>12 : + dd1=dd1.drop(i) + return dd1 + + +#dd=drop_rubbish(dd1) + + + +def zhuge_out(ddd): + + #print('我在诸葛文件内',ddd) + + + def url_lable(company_list,dmi1): + + for p in company_list: + #print('p=',p) + for k in dmi1.index: + #print('k=',k) + #print(dmi1['url'][k]) + ff=dmi1['url'][k] + if type(ff)==str: + + if p in dmi1['url'][k]: + dmi1['url'][k]=p + return dmi1 + + def drop_rubbish(dd1): + + for i in dd1.index: + pp=dd1['url'][i] + + # print(pp) + + if type(pp)==str: + + if len(pp)>12 : + dd1=dd1.drop(i) + return dd1 + + + + + + def k_means(pp,clus): + from sklearn.cluster import KMeans + pv=list(pp) + if len(set(pv))>3: + gf=numpy.array([pv]).T + estimator = KMeans(n_clusters=clus)#构造聚类器 + estimator.fit(gf)#聚类 + label_pred = estimator.labels_ #获取聚类标签 + + aa=collections.Counter(label_pred) + + #print('aa=',aa) + v=pandas.Series(aa) + gg=list(v) + index_max=gg.index(max(gg)) + + centroids = estimator.cluster_centers_ #获取聚类中心 + #inertia = estimator.inertia_ # 获取聚类准则的总和 + center=centroids[index_max][0] + return int(center) + else: + return int(pp.mean()) + + def spilit_number_from_string(dd): + + for pl in dd.index: + bte=dd['build_date'][pl] + if bte!=None: + dd['build_date'][pl]=re.sub("\D", "", str(bte)) + + most_build=Counter(dd['build_date']).most_common(1) + dd['build_date']=dd['build_date'].fillna(most_build[0][0]) + + return dd[dd['build_date']==most_build[0][0]] + + def lower_dimention(dd): + + url_set=list( set(dd['url']) ) + + url_price=pandas.Series() + + index_list=[] + for i in url_set: + dd1=dd[dd['url']==i] + if dd1.shape[0]<=2: + index_list.extend(list(dd1.index)) + else: + most_price=k_means(dd1['total_price'],3)*1.0 + + url_price[i]=round(most_price,2) + + for j in index_list: + url_price[dd['url'][j]]=dd['total_price'][j] + + return url_price + + + def k_means_ratio(pp,clus): + from sklearn.cluster import KMeans + pv=list(pp) + if len(set(pv))>3: + gf=numpy.array([pv]).T + estimator = KMeans(n_clusters=clus)#构造聚类器 + estimator.fit(gf)#聚类 + label_pred = estimator.labels_ #获取聚类标签 + + aa=collections.Counter(label_pred) + + #print('aa=',aa) + v=pandas.Series(aa) + v=v.sort_values(ascending=False) + + ratio=v.iloc[0]/v.iloc[1] + + return round(ratio,2) + else: + return 1 + + + def popo(dmi,columns): + + most=Counter(dmi[columns]).most_common(1) + #print(dmi) + dmi2=dmi[dmi[columns]==most[0][0]] + #print('dmi2=',dmi2) + + dmiy=lower_dimention(dmi2) + #print(dmiy) + ratio2=(dmiy.max()-dmiy.min())/dmiy.mean() + ww1=round(k_means( dmiy ,3 ),2) + ww2=round(k_means(dmi2['area'].dropna(),3),2) + ww3=round(k_means(dmi2['room'].dropna(),3),2) + ww4=list(dmiy.index) + + #ww5=round(k_means_ratio(dmi['total_price'].dropna(),3),2) + ww5=round(k_means_ratio(dmiy,3),2) + #print('dmi2=',dmi2) + + if (ratio2<0.15 and len(dmiy)>1) or (ratio2>1.2 and len(dmiy)>1) : + + return [round(ww1,2),round(ww2,2),round(ww3,2),round(100,2),ww4,ww5] + + #print('area_filter',dd) + """ + else: + return [round(ww1,2),round(ww2,2),round(ww3,2),round(len(dmiy),2),ww4,ww5] + """ + + + + + def room_define(dd,j): + + pdd=numpy.array([[None,None,None,None,None,None], + [None,None,None,None,None,None], + [None,None,None,None,None,None]]) + + + d4=dd[dd['room']==j] + + #print('room=',j,'的样本个数为',d4.shape[0]) + + pdd[0]=popo(d4,'room') + + most_area=Counter(d4['area']).most_common(1) + + + + if most_area[0][1]>=2: + + d4area=d4[d4['area']==most_area[0][0]] + + pdd[1]=popo(d4area,'area') + + most_floor=Counter(d4area['floor']).most_common(1) + + if most_floor[0][1]>=2: + d4floor=d4area[d4area['floor']==most_floor[0][0]] + #print('d4floor=',d4floor) + pdd[2]=popo(d4floor,'floor') + + return ( pdd ) + + + def dout(room_pair,dd): + + bbb=[] + #print('room_pair=',room_pair) + #print('dd=',dd) + for j in room_pair.index: + + kgg=pandas.DataFrame(room_define(dd,j), + columns=['total_price','area','room','credit','url_list','choose_index']) + kgg['avg_price']=kgg['total_price']/kgg['area'] + + #print('kgg=',kgg) + + bbb.append(kgg) + return bbb + + + + + + + + def dsf_average_price(ct,rl,lk,timd,timm): + + client = pymongo.MongoClient('192.168.0.136',27017) #'192.168.0.65',27777 + db = client.fangjia + seawater = db.average_statistics + + timd1=timd//100 + timm1=timm//100 + + query = {"city":ct,"region":rl, + "name":{"$in":lk}, + "s_date":{"$gt":timd1,"$lt":timm1}} + #lt= seawater.count(query) + #print(lt) + pos = list(seawater.find(query).limit(3000)) + + data=pandas.DataFrame(pos) + del pos + return data + + + dsf_name_list={'soufang_new_price','soufang_esf_price', + 'anjuke_new_price','anjuke_esf_price', + 'sina_new_price','sina_esf_price', + 'ganji_esf_price','cric_new_price', + 'cric_esf_price', + 'lianjia_esf_price','zhugezhaofang_esf_price' + + 'fanggugu_esf_price','yungujia_esf_price' + } + + + + + + + company_list=['lianjia','zhuge','5i5j','iwjw','fangdd','centanet', + 'cityhouse','esf','baixing','ganji','anjuke','58'] + + company_name=['链家','诸葛找房','我爱我家','爱屋吉屋','房多多','中原', + '城市房产','二手房','百姓','赶集','安居客','58同城'] + company_values=[10, 10, 5, 5, 3, 2, + 1 , 1, 1, 1, 1, 1] + + #companypp=[] + import numpy as np + + a=np.array([company_name,company_values]) + + company_pandas=pandas.DataFrame(a.T,columns=['name','values'],index=company_list) + name_pair=pandas.Series(company_list,index=company_name) + + + def add_floor(ppd): + if 'floor' not in ppd.columns: + ppd['floor']= 6 + return ppd + + #ddd=url_lable(company_list,ddd) + + + + def opfm(ddo,fm): + + floor_column=ddo[fm].dropna() + if len(floor_column)>0: + + #print('floor_column=',floor_column) + most_floor=Counter(floor_column).most_common(1) + #print('floor=',ddo['floor']) + ddo[fm]=ddo[fm].fillna(most_floor[0][0]) + return ddo + else: + ddo[fm]=ddo[fm].fillna(2) + return ddo + + + + + + + def export_pp(ddd): + + ddo=ddd + #ddo=ddo.drop(0) + #print('ddo=',ddo) + ddo=add_floor(ddo) + #print('ddo=',ddo) + + if ddo.shape[0]>1: + + + ddo=opfm(ddo,'floor') + ddo=opfm(ddo,'room') + + """ + floor_column=ddo['floor'].dropna() + print('floor_column=',floor_column) + most_floor=Counter(floor_column).most_common(1) + print('floor=',ddo['floor']) + ddo['floor']=ddo['floor'].fillna(most_floor[0][0]) + + room_column=ddo['room'].dropna() + most_room=Counter(room_column).most_common(1) + ddo['room']=ddo['room'].fillna(most_room[0][0]) + """ + + ddo['area']=(ddo['area']//2)*2+1 + + #dd1=url_lable(company_list,ddo) + + #dd=drop_rubbish(dd1) + + #print(Counter(dd['url'])) + dd=ddo + mr=Counter(dd['room']) + room_pair=pandas.Series(mr) + room_pair1=room_pair.sort_values(ascending=False) + room_pair=room_pair1.iloc[:3] + opt=dout(room_pair,dd) + return opt + else: + return pandas.DataFrame() + + opt=export_pp(ddd) + + #print('opt=',opt) + + def k_means22(pp,clus): + from sklearn.cluster import KMeans + pv=list(pp) + if len(set(pv))>1: + gf=numpy.array([pv]).T + estimator = KMeans(n_clusters=clus)#构造聚类器 + estimator.fit(gf)#聚类 + label_pred = estimator.labels_ #获取聚类标签 + + aa=collections.Counter(label_pred) + + #print('aa=',aa) + v=pandas.Series(aa) + gg=list(v) + index_max=gg.index(max(gg)) + + centroids = estimator.cluster_centers_ #获取聚类中心 + #inertia = estimator.inertia_ # 获取聚类准则的总和 + center=centroids[index_max][0] + return round(center,2) + else: + return pp.mean() + + + def out_put_price(opt): + + re = pandas.concat(opt) + + #print(re) + + oo=re[re['credit']==100] + oo1=re.dropna() + + #print(oo) + if oo.shape[0]>1: + return round(k_means22( oo['avg_price'] ,2 ),2) + elif oo1.shape[0]>0: + + re1=re.sort_values(by='credit',ascending=False) + re2=re1.sort_values(by='choose_index',ascending=False) + #print(re2) + #print('#'*20) + return (re2['avg_price'].iloc[0]) + else: + #print(ddd['avg'].dropna()) + kkm=k_means(ddd['avg'].dropna(),2) + return kkm + + return int(out_put_price(opt)) + +#ddd=pandas.read_excel('ddd11.xls') + +#df=zhuge_out(ddd) +""" +ct='上海' +rl='崇明' +lk=['风伟四村'] + + +timd=20170301 +timm=20171205 + + + +def some_district_information(ct,rl,lk,timd,timm): + client = pymongo.MongoClient('192.168.0.105',27777) #'192.168.0.65',27777 + db = client.fangjia + seawater = db.seawater + seawater.find_one() + query = {"city":ct,"cat":"sell","region":rl, + "district_name":{"$in":lk}, + "p_date":{"$gt":timd,"$lt":timm}} + lt= seawater.count(query) + #print(lt) + pos = list(seawater.find(query).limit(lt)) + + data=pandas.DataFrame(pos) + del pos + return data + +ddd=some_district_information(ct,rl,lk,timd,timm) + + + +df=zhuge_out(ddd) +""" + + + + + + + + + + + + + + + + + + + + + + diff --git a/PriceSystem/zhuge/zhuge_consist.py b/PriceSystem/zhuge/zhuge_consist.py new file mode 100644 index 0000000..cc3665b --- /dev/null +++ b/PriceSystem/zhuge/zhuge_consist.py @@ -0,0 +1,486 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue Nov 28 13:15:49 2017 + +@author: Administrator +""" +import pymongo +import pandas +from collections import Counter +import numpy +import collections +import re + +def url_lable(dmi1): + + company_list=['lianjia','zhuge','5i5j','iwjw','fangdd','centanet', + 'cityhouse','esf','baixing','ganji','anjuke','58'] + + for p in company_list: + #print('p=',p) + for k in dmi1.index: + #print('k=',k) + #print(dmi1['url'][k]) + ff=dmi1['url'][k] + if type(ff)==str: + + if p in dmi1['url'][k]: + dmi1.loc[k,'url']=p + return dmi1 + + +def drop_rubbish(dd1): + + for i in dd1.index: + pp=dd1['url'][i] + + # print(pp) + + if type(pp)==str: + + if len(pp)>12 : + dd1=dd1.drop(i) + return dd1 + + +#dd=drop_rubbish(dd1) + + + +def zhuge_out(ddd): + + #print('我在诸葛文件内',ddd) + + ''' + def url_lable(company_list,dmi1): + + for p in company_list: + #print('p=',p) + for k in dmi1.index: + #print('k=',k) + #print(dmi1['url'][k]) + ff=dmi1['url'][k] + if type(ff)==str: + + if p in dmi1['url'][k]: + dmi1['url'][k]=p + return dmi1 + + def drop_rubbish(dd1): + + for i in dd1.index: + pp=dd1['url'][i] + + # print(pp) + + if type(pp)==str: + + if len(pp)>12 : + dd1=dd1.drop(i) + return dd1 + ''' + + + + + + def k_means(pp,clus): + from sklearn.cluster import KMeans + pv=list(pp) + if len(set(pv))>clus: + gf=numpy.array([pv]).T + estimator = KMeans(n_clusters=clus)#构造聚类器 + estimator.fit(gf)#聚类 + label_pred = estimator.labels_ #获取聚类标签 + + aa=collections.Counter(label_pred) + + #print('aa=',aa) + v=pandas.Series(aa) + gg=list(v) + index_max=gg.index(max(gg)) + + centroids = estimator.cluster_centers_ #获取聚类中心 + #inertia = estimator.inertia_ # 获取聚类准则的总和 + center=centroids[index_max][0] + return int(center) + else: + return int(pp.mean()) + + def spilit_number_from_string(dd): + + for pl in dd.index: + bte=dd['build_date'][pl] + if bte!=None: + dd['build_date'][pl]=re.sub("\D", "", str(bte)) + + most_build=Counter(dd['build_date']).most_common(1) + dd['build_date']=dd['build_date'].fillna(most_build[0][0]) + + return dd[dd['build_date']==most_build[0][0]] + + def lower_dimention(dd): + + url_set=list( set(dd['url']) ) + + url_price=pandas.Series() + + index_list=[] + for i in url_set: + dd1=dd[dd['url']==i] + if dd1.shape[0]<=2: + index_list.extend(list(dd1.index)) + else: + most_price=k_means(dd1['total_price'],3)*1.0 + + url_price[i]=round(most_price,2) + + for j in index_list: + url_price[dd['url'][j]]=dd['total_price'][j] + + return url_price + + + def k_means_ratio(pp,clus): + from sklearn.cluster import KMeans + pv=list(pp) + if len(set(pv))>3: + gf=numpy.array([pv]).T + estimator = KMeans(n_clusters=clus)#构造聚类器 + estimator.fit(gf)#聚类 + label_pred = estimator.labels_ #获取聚类标签 + + aa=collections.Counter(label_pred) + + #print('aa=',aa) + v=pandas.Series(aa) + v=v.sort_values(ascending=False) + + ratio=v.iloc[0]/v.iloc[1] + + return round(ratio,2) + else: + return 1 + + + def popo(dmi,columns): + + most=Counter(dmi[columns]).most_common(1) + #print(dmi) + dmi2=dmi[dmi[columns]==most[0][0]] + #print('dmi2=',dmi2) + + dmiy=lower_dimention(dmi2) + #print(dmiy) + ratio2=(dmiy.max()-dmiy.min())/dmiy.mean() + ww1=round(k_means( dmiy ,3 ),2) + ww2=round(k_means(dmi2['area'].dropna(),3),2) + ww3=round(k_means(dmi2['room'].dropna(),3),2) + ww4=list(dmiy.index) + + #ww5=round(k_means_ratio(dmi['total_price'].dropna(),3),2) + ww5=round(k_means_ratio(dmiy,3),2) + #print('dmi2=',dmi2) + + if (ratio2<0.15 and len(dmiy)>1) or (ratio2>1.2 and len(dmiy)>1) : + + return [round(ww1,2),round(ww2,2),round(ww3,2),round(100,2),ww4,ww5] + + #print('area_filter',dd) + """ + else: + return [round(ww1,2),round(ww2,2),round(ww3,2),round(len(dmiy),2),ww4,ww5] + """ + + + + + def room_define(dd,j): + + pdd=numpy.array([[None,None,None,None,None,None], + [None,None,None,None,None,None], + [None,None,None,None,None,None]]) + + + d4=dd[dd['room']==j] + + #print('room=',j,'的样本个数为',d4.shape[0]) + + pdd[0]=popo(d4,'room') + + most_area=Counter(d4['area']).most_common(1) + + + + if most_area[0][1]>=2: + + d4area=d4[d4['area']==most_area[0][0]] + + pdd[1]=popo(d4area,'area') + + most_floor=Counter(d4area['floor']).most_common(1) + + if most_floor[0][1]>=2: + d4floor=d4area[d4area['floor']==most_floor[0][0]] + #print('d4floor=',d4floor) + pdd[2]=popo(d4floor,'floor') + + return ( pdd ) + + + def dout(room_pair,dd): + + bbb=[] + #print('room_pair=',room_pair) + #print('dd=',dd) + for j in room_pair.index: + + kgg=pandas.DataFrame(room_define(dd,j), + columns=['total_price','area','room','credit','url_list','choose_index']) + kgg['avg_price']=kgg['total_price']/kgg['area'] + + #print('kgg=',kgg) + + bbb.append(kgg) + return bbb + + + + + + + + def dsf_average_price(ct,rl,lk,timd,timm): + + client = pymongo.MongoClient('192.168.0.136',27017) #'192.168.0.65',27777 + db = client.fangjia + seawater = db.average_statistics + + timd1=timd//100 + timm1=timm//100 + + query = {"city":ct,"region":rl, + "name":{"$in":lk}, + "s_date":{"$gt":timd1,"$lt":timm1}} + #lt= seawater.count(query) + #print(lt) + pos = list(seawater.find(query).limit(3000)) + + data=pandas.DataFrame(pos) + del pos + return data + + + dsf_name_list={'soufang_new_price','soufang_esf_price', + 'anjuke_new_price','anjuke_esf_price', + 'sina_new_price','sina_esf_price', + 'ganji_esf_price','cric_new_price', + 'cric_esf_price', + 'lianjia_esf_price','zhugezhaofang_esf_price' + + 'fanggugu_esf_price','yungujia_esf_price' + } + + + + + + + company_list=['lianjia','zhuge','5i5j','iwjw','fangdd','centanet', + 'cityhouse','esf','baixing','ganji','anjuke','58'] + + company_name=['链家','诸葛找房','我爱我家','爱屋吉屋','房多多','中原', + '城市房产','二手房','百姓','赶集','安居客','58同城'] + company_values=[10, 10, 5, 5, 3, 2, + 1 , 1, 1, 1, 1, 1] + + #companypp=[] + import numpy as np + + a=np.array([company_name,company_values]) + + company_pandas=pandas.DataFrame(a.T,columns=['name','values'],index=company_list) + name_pair=pandas.Series(company_list,index=company_name) + + + def add_floor(ppd): + if 'floor' not in ppd.columns: + ppd['floor']= 6 + return ppd + + #ddd=url_lable(company_list,ddd) + + + + def opfm(ddo,fm): + + floor_column=ddo[fm].dropna() + if len(floor_column)>0: + + #print('floor_column=',floor_column) + most_floor=Counter(floor_column).most_common(1) + #print('floor=',ddo['floor']) + ddo[fm]=ddo[fm].fillna(most_floor[0][0]) + return ddo + else: + ddo[fm]=ddo[fm].fillna(2) + return ddo + + + + + + + def export_pp(ddd): + + ddo=ddd + #ddo=ddo.drop(0) + #print('ddo=',ddo) + ddo=add_floor(ddo) + #print('ddo=',ddo) + + if ddo.shape[0]>1: + + + ddo=opfm(ddo,'floor') + ddo=opfm(ddo,'room') + + """ + floor_column=ddo['floor'].dropna() + print('floor_column=',floor_column) + most_floor=Counter(floor_column).most_common(1) + print('floor=',ddo['floor']) + ddo['floor']=ddo['floor'].fillna(most_floor[0][0]) + + room_column=ddo['room'].dropna() + most_room=Counter(room_column).most_common(1) + ddo['room']=ddo['room'].fillna(most_room[0][0]) + """ + + ddo['area']=(ddo['area']//2)*2+1 + + #dd1=url_lable(company_list,ddo) + + #dd=drop_rubbish(dd1) + + #print(Counter(dd['url'])) + dd=ddo + mr=Counter(dd['room']) + room_pair=pandas.Series(mr) + room_pair1=room_pair.sort_values(ascending=False) + room_pair=room_pair1.iloc[:3] + opt=dout(room_pair,dd) + return opt + else: + return pandas.DataFrame() + + opt=export_pp(ddd) + + #print('opt=',opt) + + def k_means22(pp,clus): + from sklearn.cluster import KMeans + pv=list(pp) + if len(set(pv))>1: + gf=numpy.array([pv]).T + estimator = KMeans(n_clusters=clus)#构造聚类器 + estimator.fit(gf)#聚类 + label_pred = estimator.labels_ #获取聚类标签 + + aa=collections.Counter(label_pred) + + #print('aa=',aa) + v=pandas.Series(aa) + gg=list(v) + index_max=gg.index(max(gg)) + + centroids = estimator.cluster_centers_ #获取聚类中心 + #inertia = estimator.inertia_ # 获取聚类准则的总和 + center=centroids[index_max][0] + return round(center,2) + else: + return pp.mean() + + + def out_put_price(opt): + + re = pandas.concat(opt) + + #print(re) + + oo=re[re['credit']==100] + oo1=re.dropna() + + #print(oo) + if oo.shape[0]>1: + return round(k_means22( oo['avg_price'] ,2 ),2) + elif oo1.shape[0]>0: + + re1=re.sort_values(by='credit',ascending=False) + re2=re1.sort_values(by='choose_index',ascending=False) + #print(re2) + #print('#'*20) + return (re2['avg_price'].iloc[0]) + else: + #print(ddd['avg'].dropna()) + kkm=k_means(ddd['avg'].dropna(),2) + return kkm + + return int(out_put_price(opt)) + + +""" +ct='上海' +rl='崇明' +lk=['风伟四村'] + + +timd=20170301 +timm=20171205 + + + +def some_district_information(ct,rl,lk,timd,timm): + client = pymongo.MongoClient('192.168.0.105',27777) #'192.168.0.65',27777 + db = client.fangjia + seawater = db.seawater + seawater.find_one() + query = {"city":ct,"cat":"sell","region":rl, + "district_name":{"$in":lk}, + "p_date":{"$gt":timd,"$lt":timm}} + lt= seawater.count(query) + #print(lt) + pos = list(seawater.find(query).limit(lt)) + + data=pandas.DataFrame(pos) + del pos + return data + +ddd=some_district_information(ct,rl,lk,timd,timm) + + + +df=zhuge_out(ddd) +""" + + + + + + + + + + + + + + + + + + + + + + diff --git a/PriceSystem/zhuge0118A.py b/PriceSystem/zhuge0118A.py new file mode 100644 index 0000000..f0a5870 --- /dev/null +++ b/PriceSystem/zhuge0118A.py @@ -0,0 +1,1423 @@ + # -*- coding: utf-8 -*- +""" +Created on Tue Jan 2 13:24:07 2018 + +@author: Administrator +""" + +# -*- coding: utf-8 -*- +""" +Created on Fri Sep 22 18:18:15 2017 + +@author: Administrator +""" + +import os +print( os.getcwd()) +import pymongo +import numpy as np +import pandas as pd +import random +import collections +import pymongo +import sklearn.cluster +import pandas +import numpy +import sys, time +import pp +import zhuge +import fangjia +import sklearn.svm +import datetime + +#import yesterday + +def front_step_time(day): + now = datetime.datetime.now() + front=now-datetime.timedelta(days=day) + d1= front.strftime('%Y%m%d') + return int(d1) + + + +#提取房源的数据时间上限,如从20170301之后的数据 +some_district_information_time=front_step_time(360) + +#提周边小区均价的数据时间上限,如从20170801之后的数据 +zhoubian_xiaoqu_time=front_step_time(200) + + +#第三方均价的数据时间上限,如从201703之后的数据 +dsfjujia_time=front_step_time(430)//100 + +############################################################ +############################################################ +#存到数据库的日期 + + +""" +def weekend00(): + + today = datetime.datetime.now() + now_time=datetime.datetime(today.year, today.month, today.day, 0, 0, 0) + week_time = now_time.weekday() + + + if week_time>=4: + future_time = now_time-datetime.timedelta(days=week_time+1-7) + d1= future_time.strftime('%Y%m%d') + weekend1= datetime.datetime.strptime(d1,'%Y%m%d') + weekend=weekend1+datetime.timedelta(hours=-8) + return weekend + else: + future_time = now_time-datetime.timedelta(days=week_time+1) + d1= future_time.strftime('%Y%m%d') + weekend1= datetime.datetime.strptime(d1,'%Y%m%d') + weekend=weekend1+datetime.timedelta(hours=-8) + return weekend + +""" + + +#date="2018-02-04" +############################################################ +############################################################ + +credit=pandas.Series([0.9,0.8,0.7,0.6,0.5],index=['0.9','0.8','0.7','0.6','0.5']) + +value_type=pandas.Series([1,2,3,4,5,6,7], + index=['指导价','自身样本','第三方','交易数据','周边楼盘','推导','优质样本']) + +#1:指导价 2:自身样本 3:第三方 4:交易数据 5:周边楼盘, 6:推导 + + +def commonprice(ct,rl,p): + + + + def add_block(ppd): + if 'block' not in ppd.columns: + ppd['block']='没有' + return ppd + + def add_room(ppd): + if 'room' not in ppd.columns: + ppd['room']= 2 + return ppd + + + def some_district_information(ct,rl,lk): + + + query = {"city":ct,"cat":"sell","region":rl, + "district_name":{"$in":lk}, + "p_date":{"$gt":some_district_information_time}} + + return fangjia.findHouse(query) + """ + return fangjia.find_house_from_elastic_search(ct, rl, lk, + "lianjia.com", some_district_information_time) + """ + + def disanganf_price(ct,rl,lk,tt): + #db1 = client1.fangjia + #dsf = db1.average_statistics + query = {"city":ct,"region":rl, + "name":lk,"s_date":{"$gt":tt}} + #pos = list(dsf136.find(query)) + + #data1=pandas.DataFrame(pos) + data1=fangjia.find_disanganf_price(query) + + if data1.shape[0]>1: + a_set={'fangjia_price'} + b_set=set(data1.columns) + if a_set & b_set==a_set: + + data=data1[data1['credit']>0.7] + if data.shape[0]>=1: + choose_class=['city','region','name','fangjia_price','credit', + 's_date'] + dc=data[choose_class] + return dc + else: + return numpy.array([]) + else: + + return numpy.array([]) + else: + return numpy.array([]) + + + + + def take_location(ct,rl,p): + + dirlt=p[0] + + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + seaweed136=fangjia.get_collection("fangjia","seaweed", '192.168.0.136',27017) + query1 = {"status":0,"cat":"district","city":ct, + "region":rl, "name":dirlt} + + for err_count in range(0, 10): + + try: + ltt= seaweed136.count(query1) + if ltt>0: + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1,"cat":1,"name":1} + lct= seaweed136.find(query1, fields1) + lf=pandas.DataFrame(list(lct)) + + #lf=fangjia.find_seaweed(query1, fields1) + le=lf + le.index=le['name'] + cluns=lf.columns + if ('lng2' in cluns )and ('lat2' in cluns): + lr=le[['lng2','lat2']] + del lf,le,cluns + return lr + else: + del lf,le,cluns + return numpy.array([]) + + else: + return numpy.array([]) + except: + + ltt= seaweed136.count(query1) + if ltt>0: + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1,"cat":1,"name":1} + lct= seaweed136.find(query1, fields1) + lf=pandas.DataFrame(list(lct)) + + #lf=fangjia.find_seaweed(query1, fields1) + le=lf + le.index=le['name'] + cluns=lf.columns + if ('lng2' in cluns )and ('lat2' in cluns): + lr=le[['lng2','lat2']] + del lf,le,cluns + return lr + else: + del lf,le,cluns + return numpy.array([]) + + else: + return numpy.array([]) + + def lianjia(ur): + #print(ur) + if 'lianjia' in ur: + return 0 + else: + return 1 + + + def zhuge(ur): + if 'zhuge' in ur: + return 0 + else: + return 1 + + def wawj(ur): + if '5i5j' in ur: + return 0 + else: + return 1 + + def iwjw(ur): + if 'iwjw' in ur: + return 0 + else: + return 1 + + def fangdd(ur): + if 'fangdd' in ur: + return 0 + else: + return 1 + + def wkzf(ur): + if 'wkzf' in ur: + return 0 + else: + return 1 + + + + def label(dd,f): + aa=list() + + for i in dd.index: + d2f=dd['url'][i] + if type(d2f)==str: + if f(dd['url'][i])==0: + aa.append(i) + return set(aa) + + + + def new_data(ds): + bb=set(ds['name']) + bb1=list(bb) + x1=pandas.Series() + for i in bb1: + a1=ds[ds['name']==i] + a2=a1.sort_values(by='s_date',ascending=False) + x1[i]=a2.iloc[0]['fangjia_price'] + return x1 + + + def zhoubian_xiaoqu(ct,rl,dirst): + + #提取周边小聚的均价,本方法不用房源,直接用上一批数据的价格来计算无房源小区 + ################### + ################### + tii=zhoubian_xiaoqu_time + ################### + ################### + + tt=tii//100-1 + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #dsf = db1.average_statistics + query = {"city":ct,"region":rl, + "name":{"$in":dirst},"s_date":{"$gt":tt}} + #pos = list( dsf136.find(query)) + + #data=pandas.DataFrame(pos) + data=fangjia.find_disanganf_price(query) + #print(data) + if data.shape[0]>=1: + #data=add_block(data) + data=add_room(data) + + a_set={'city','region','name','fangjia_price','credit', + 's_date'} + b_set=set(data.columns) + + if a_set & b_set==a_set: + + choose_class=['city','region','name','fangjia_price','credit', + 's_date'] + dc=data[choose_class] + return dc + else: + return numpy.array([]) + return numpy.array([]) + else: + + return numpy.array([]) + + + def hh(ppp,df): + ppp['price']=None + aa=list(df.index) + #print('aa=',aa) + for i in ppp.index: + kk=ppp['district_name'][i] + #print('kk=',kk) + if kk in aa: + #ppp['price'][i]=df[kk] + ppp.loc[i,'price'] = df[kk] + #print('df[kk]',df[kk]) + return ppp + + + + def neighbor(lng,lat,scale,ct,rl,p): + + ################### + ################### + #timm=20171122 + ################### + ################### + + estype=p[2] + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + kkk=scale/1000 + longtitude=lng + latitude=lat + + query1 = {"status":0,"cat":"district", + "lat2":{"$gt":latitude-kkk*10**10/0.86,"$lt":latitude+kkk*10**10/0.86}, + "lng2":{"$gt":longtitude-kkk*10**10,"$lt":longtitude+kkk*10**10}} + #fields1 = {"lat2":1,"lng2":1, "city":1,"region":1,"cat":1,"name":1} + + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, + "cat":1,"name":1,'estate_type2':1} + + #lct= list( seaweed136.find(query1, fields1)) + + #lf=pandas.DataFrame(lct) + lf=fangjia.find_seaweed(query1, fields1) + + le=le=lf[lf['estate_type2']==estype] + + le.index=le['name'] + + lr=le[['lng2','lat2']] + + + dirst1=list(lr.index) + + random.shuffle(dirst1) + + dirst=dirst1[:80] + #print('dirst=',dirst) + + dirst_plus=dirst+[p[0]] + + query_feature = {"city":ct,"region":rl, + "district_name":{"$in":dirst_plus}} + + ppp=fangjia.dis_feature(query_feature) + #print('ox'*10,ppp) + + #print('小区个数= ',len(dirst)) + df1=zhoubian_xiaoqu(ct,rl,dirst) + #print('df1=',df1,'*'*20) + if df1.shape[0]>1: + df=new_data(df1) + #print('df=',df) + vv=hh(ppp,df) + #print('#'*36) + #print('vv=',vv) + return vv + else: + return numpy.array([]) + + + def svr(vv,lk): + #提取要计算小区的特征 + vk=vv[vv['district_name']==lk] + #删除要计算小区的空特征 + #print('vk=',vk) + if vk.shape[0]>0: + + vk1=vk.iloc[0].dropna() + + #将要计算小区的特征提取出来 + + vk2=list( set(vk1.index)-{'_id','district_name','region','city'}) + + #print('vk2=',vk2) + if len(vk2)>1: + vvo=vv[vv['district_name']!=lk] + vk3=vk2+['price'] + vk31=list(set(vk3)) + vvo1=vvo[vk31].dropna() + + x1 =vvo1.drop('price',axis=1) + #print('x1=',x1) + y = vvo1['price'] + #print('y',y) + + if len(x1)>2: + + clf = sklearn.svm.SVR() + clf.fit(x1, y) + pp1= vk[vk31].drop('price',axis=1) + result = clf.predict(pp1 ) + #print (result) + return int(result[0]) + else: + km=vv['price'].dropna() + return int(k_means(km)) + + + else: + km=vv['price'].dropna() + return int(k_means(km)) + + else: + km=vv['price'].dropna() + return int(k_means(km)) + + + + def no_source_price(ct,rl,location,p): + lk=p[0] + hg=location.shape[0] + + if hg==0: + return '没有经纬度数据',0 + elif hg>1: + return '纬度数据有误,有多个经纬度',0 + else: + #t1=time.time() + dd=neighbor(location['lng2'][0],location['lat2'][0],5,ct,rl,p) + #print(time.time()-t1,'t1'*10) + #print('dd=',dd,'dd'*20) + if dd.shape[0]>=20: + #print('$'*50) + #dd1=neighbor(location['lng2'][0],location['lat2'][0],5) + #print('dd',dd) + #svr_time=time.time() + other=svr(dd,lk) + #print(time.time()-svr_time,'svr'*10) + print('other= ' ,other,'9'*20) + return other,dd.shape[0] + else: + #t2=time.time() + dd1=neighbor(location['lng2'][0],location['lat2'][0],10,ct,rl,p) + #print(time.time()-t2,'t2'*10) + #print('dd1=',dd1) + if dd1.shape[0]>=20: + #print('$'*50) + + + #svr_time=time.time() + other1=svr(dd1,lk) + #print(time.time()-svr_time,'svr'*10) + #print('other1=',other1) + return other1,dd1.shape[0] + else: + #t3=time.time() + dd2=neighbor(location['lng2'][0],location['lat2'][0],50,ct,rl,p) + #print(time.time()-t3,'t3'*10) + print('dd2.shape',dd2.shape[0],'@'*60) + if dd2.shape[0]>=20: + print('$'*50) + svr_time=time.time() + other1=svr(dd2,lk) + print(time.time()-svr_time,'e'*10) + #print('other1=',other1) + return other1,dd2.shape[0] + else: + dd3=neighbor(location['lng2'][0],location['lat2'][0],100,ct,rl,p) + #print('dd2=',dd2,'@'*60) + if dd3.shape[0]>=20: + #svr_time=time.time() + other1=svr(dd3,lk) + #print(time.time()-svr_time,'svr'*10) + #print('other1=',other1) + return other1,dd3.shape[0] + else: + + return '没有周围小区数据' ,0 + + + def region_district_list(ct,rl): + #import pandas + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + + query1 = {"status":0,"cat":"district","city":ct,"region":rl} + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, + "cat":1,"name":1,'estate_type2':1,'alias':1} + + #lct= list(seaweed136.find(query1, fields1)) + #lf=pandas.DataFrame(lct) + lf=fangjia.find_seaweed(query1, fields1) + return lf + + + + + def k_means(pp1): + from sklearn.cluster import KMeans + pv=list(pp1) + if len(set(pv))>3: + gf=numpy.array([pv]).T + estimator = KMeans(n_clusters=3)#构造聚类器 + estimator.fit(gf)#聚类 + label_pred = estimator.labels_ #获取聚类标签 + aa=collections.Counter(label_pred) + v=pandas.Series(aa) + gg=list(v) + index_max=gg.index(max(gg)) + + centroids = estimator.cluster_centers_ #获取聚类中心 + #inertia = estimator.inertia_ # 获取聚类准则的总和 + center=centroids[index_max][0] + return (int(center)) + else: + return int(pp1.mean()) + + + def region_district_list_name(ct,rl,lkk): + #import pandas + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + + query1 = {"status":0,"cat":"district","city":ct,"region":rl,"name":lkk} + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, + "cat":1,"name":1,'estate_type2':1,'alias':1} + + #lct= list( seaweed136.find(query1, fields1)) + + + #lf=pandas.DataFrame(lct) + lf=fangjia.find_seaweed(query1, fields1 ) + return lf + + + def strip_villa(ct,rl,p): + lw=p[0] + #estype=p[2] + alias=p[1] + + #print('alias=',alias) + + def filt(x): + + return type(x)==str and len(x)>1 + + bb=list(filter(filt,alias)) + #print('bb=',bb) + bb.extend([lw]) + #lk1=list(set(lk)) + + #print('本小区名=',lw) + + #print('type(lw)=',type(bb)) + + #print('小区名列表=',bb) + + cc1= some_district_information(ct,rl,bb) + + cc1= add_room(cc1) + + print('cc1本小区数据条数=',cc1.shape[0]) + + if cc1.shape[0]>2: + + del lw,alias,bb + return cc1 + else: + + if '别墅' in lw: + + lkk=lw.strip('别墅') + cc=region_district_list_name(ct,rl,lkk) + #print('cc=',cc) + if cc.shape[0]>1: + + cc1=list(cc['alias'].dropna()) + cc2=list(filter(filt,alias)) + bb.extend([lkk]) + #clk1=list(set(clk)) + cc2=some_district_information(ct,rl,bb) + if cc2.shape[0]>2: + del lw,alias,bb,cc1,cc + return cc2 + else: + del lw,alias,bb,cc1,cc + return cc1 + else: + return cc1 + else: + return cc1 + + + def consider_villa(ct,rl,p): + + #print('villa p=',p) + + estype=p[2] + df=strip_villa(ct,rl,p) + if df.shape[0]>1: + + if estype=='普通住宅': + all_index=set(df.index) + vv=fetch_villa_index(df) + ww=all_index-vv + + d1=df.loc[ww,:] + if d1.shape[0]>3: + return d1 + else: + return df + elif estype=='别墅': + vv=fetch_villa_index(df) + d1=df.loc[vv,:] + if d1.shape[0]>3: + return d1 + else: + return df + else: + return df + else: + return df + + + def fetch_villa_index(dd): + + #dd=dd.dropna() + d1=dd[dd['area']>150] + + #print(dd) + + + villa_label=["独栋", "联排", "叠加", "双拼", + "花园面积", "泳池", "别墅", "带花园"] + bb=list() + for i in d1.index: + we=d1['name'][i] + for j in villa_label: + if j in we: + bb.append(i) + d21=dd[dd['room']>=6] + d2=d21[d21['area']>=360] + + d3=dd[dd['area']>500] + + villa=set(bb)|set(d2.index)|set(d3.index) + + return villa + + + + + + + def dsfjujia(ct,rl,lk): + + query = {"city":ct,"region":rl, + "name":lk,"s_date":{"$gt":dsfjujia_time}} + + data=fangjia.find_disanganf_price(query) + + alist=['anjuke_esf_price','ganji_esf_price', + 'lianjia_esf_price','soufang_esf_price', + 'tuboshi_esf_price','zhugezhaofang_esf_price', + 'yungujia_esf_price','fanggugu_esf_price' + ] + a1list=set(alist) + blist=set(data.columns) + + clist=a1list & blist + + ddq=list(clist) + + data1=data[ddq] + + ser=pandas.Series() + + for i in ddq: + pdd=data1[i].dropna() + #ser[i]=k_means(pdd) + ser[i]=list(pdd)[-1] + #print('采用抓取的第三方均价') + return ser + + + + + def sfjj(ll_zz,aa_zz,dddf): + + dxx=pandas.Series() + for i in ll_zz: + if i in aa_zz: + dxx[i]=dddf[i] + return int(dxx.mean()) + """ + tt=timm//100 + + lk=p[0] + dsfp1=disanganf_price(ct,rl,lk,tt) + + data= consider_villa(ct,rl,p,timm) + """ + + ####################################################### + """ + junjia=dsfjujia(ct,rl,lk) + ll_zz={'zhugezhaofang_esf_price','lianjia_esf_price'} + aa_zz=set(junjia.index) + jiaji_lz= list(ll_zz & aa_zz) + """ + + def k_means_clus(pp1,clus): + + from sklearn.cluster import KMeans + pv=list(pp1) + if len(set(pv))>clus: + gf=numpy.array([pv]).T + estimator = KMeans(n_clusters=clus)#构造聚类器 + estimator.fit(gf)#聚类 + label_pred = estimator.labels_ #获取聚类标签 + aa=collections.Counter(label_pred) + v=pandas.Series(aa) + gg=list(v) + index_max=gg.index(max(gg)) + + centroids = estimator.cluster_centers_ #获取聚类中心 + #inertia = estimator.inertia_ # 获取聚类准则的总和 + center=centroids[index_max][0] + return (int(center)) + else: + return int(pp1.mean()) + + def group(junjia_lzf,junjia_anst,ctr): + + bb=[] + for i in junjia_lzf.index: + #print(junjia_lzf) + someprice=junjia_lzf[i] + def compare15(b): + if 100*abs(someprice-b)/b<16: + return 1 + else: + return 0 + countmap=list(map(compare15,junjia_anst)) + #print('countmap=',countmap) + countmap1=pandas.Series(countmap) + countmap2=countmap1[countmap1>0] + #print('countmap2=',countmap2) + + numb=len(countmap2) + #print('len(countmap2)=',len(countmap2)) + if numb>ctr: + bb.append([someprice,numb]) + return bb + + def middle_operate(junjia_lzf,junjia_anst): + + #print('%%%',ppd3['total_price']) + + #print('junjia_lzf=',junjia_lzf) + #print('junjia_anst=',junjia_anst) + + + def mid_out_1(junjia_lzf): + + if len(junjia_lzf)>1: + #print('aa1',aa) + bb=group(junjia_lzf,junjia_lzf,1) + #print('bb=',bb) + if len(bb)>0: + bb1=pandas.DataFrame(bb,columns=['price','count']) + bb2=bb1.sort_values(by='count',ascending=False) + #print(bb2) + return bb2.iloc[0]['price'] + else: + return None + else: + return None + + + def mid_out_2(junjia_lzf,junjia_anst): + + if len(junjia_lzf)>0 and len(junjia_anst)>0: + + bb=group(junjia_lzf,junjia_anst,0) + #print('bb1=',bb) + if len(bb)>0: + bb1=pandas.DataFrame(bb,columns=['price','count']) + bb2=bb1.sort_values(by='count',ascending=False) + #print(bb2) + return bb2.iloc[0]['price'] + else: + return None + else: + return None + a11=mid_out_1(junjia_lzf) + a22=mid_out_2(junjia_lzf,junjia_anst) + a33=mid_out_1(junjia_anst) + aa1_3=pandas.Series([a11,a22,a33]).dropna() + #print(a11,a22,a33) + if len(aa1_3)>0: + return aa1_3.iloc[0] + else: + return None + + + def devide_group(al,bl,junjia): + + alls=set(junjia.index) + def set_div(ds,alls): + if len(set(ds) & alls)>0: + return junjia[ds].dropna() + else: + return pandas.Series() + lzf=set_div(al,alls) + anst=set_div(bl,alls) + + return lzf , anst + + + def dglg_out1(junjia,neighb): + + """ + alist=['anjuke_esf_price','ganji_esf_price', + 'lianjia_esf_price','soufang_esf_price', + 'tuboshi_esf_price','zhugezhaofang_esf_price', + 'yungujia_esf_price','fanggugu_esf_price' + ] + """ + + + al=[ + 'lianjia_esf_price', + 'zhugezhaofang_esf_price', + 'fanggugu_esf_price' + ] + bl=['anjuke_esf_price', + 'soufang_esf_price', + 'tuboshi_esf_price', + ] + + #ppd3dp=ppd3['total_price'].dropna() + if type(neighb)!=str and neighb!=None : + + junjia=junjia[junjia>0.55*neighb] + ''' + junjia_lzf=junjia[al].dropna() + junjia_anst=junjia[bl].dropna() + ''' + alldi=devide_group(al,bl,junjia) + + junjia_lzf=alldi[0] + junjia_anst=alldi[1] + dg11=middle_operate(junjia_lzf,junjia_anst) + return dg11 + + + else: + alldi=devide_group(al,bl,junjia) + + junjia_lzf=alldi[0] + junjia_anst=alldi[1] + + dg22=middle_operate(junjia_lzf,junjia_anst) + return dg22 + + + lk=p[0] + location=take_location(ct,rl,p) + + #ttp=time.time() + neighb=no_source_price(ct,rl,location,p) + + #print(time.time()-ttp,'m'*20) + + junjia=dsfjujia(ct,rl,lk) + #print('junjia=',junjia) + + ''' + dglg=dsplg(junjia,neighb,2) + dggg=dsplg(junjia,neighb,1) + ''' + dglg=dglg_out1(junjia,neighb[0]) + #dggg=dglg_out1(junjia,neighb[0]) + + #print('dglg=',dglg) + #print('dggg=',dggg) + + #tt=time.time() + data= consider_villa(ct,rl,p) + #print(time.time()-tt,'<>'*26) + ####################################################### + + if data.shape[0]>3: + + paa=add_block(data) + ppd=paa[['total_price','area','region','district_name','url','block','p_date']] + #print('ppd##############') + #print('ppd=',ppd) + ppd=ppd.fillna(method='ffill') + + ppd1=ppd.dropna() + + #t3=time.time() + + ppd1['total_price']=ppd1['total_price']/ppd1['area'] + #print('ppd1time',time.time()-t3,'o'*36) + + #ppd1.loc['total_price']=ppd1['total_price']/ppd1['area'] + + #ppd2=ppd1 + + + + #if type(neighb) is not str: + + if ppd1.shape[0]>0: + + def guolv_data(ppd2,neighb): + if type(neighb) is not str: + ph=ppd2[ppd2['total_price']>0.5*neighb] + return ph + else: + return ppd2 + + #ppd3=ppd2[ppd2['total_price']>0.6*neighb] + ###################################### + ppd3=guolv_data(ppd1,neighb[0]) + + ###################################### + ldf=label(ppd3,lianjia) + #print('ldf=',ldf) + + ldf1=label(ppd3,zhuge) + + ldf2=label(ppd3,wawj) + + ldf3=label(ppd3,iwjw) + ldf4=label(ppd3,fangdd) + + ldf5=label(ppd3,wkzf) + l_and_z=ldf|ldf1 + #print(l_and_z) + #print('l_and_z=',l_and_z) + allf=l_and_z|ldf3|ldf4|ldf5|ldf2 + #print('l_and_z_and_i=',l_and_z_and_i) + #mm=0 + length=len(l_and_z) + length1=len(allf) + + + if dglg != None: + + #print('junjia.shape',len(junjia)) + sample_size=len(junjia) + return dglg,float('%.2f' % credit['0.8']),value_type['第三方'],sample_size + + elif length>3: + #print('ldf=',l_and_z_and_i) + pppk=ppd3.loc[l_and_z,:] + + ppp=pppk[['total_price','p_date']] + #print(ppp) + ppp1=ppp.sort_values(['p_date'], ascending=False) + + gdf=ppp1['total_price'][:36].dropna() + #print('链家和诸葛找房合并后的数据集=',gdf) + sample_size=len(gdf) + gm=k_means_clus(gdf,2) + print('链猪= ',gm) + return int(gm),float('%.2f' % credit['0.8']),value_type['自身样本'] ,sample_size #优质样本 + + + elif data.shape[0]>0: + + print('诸葛找房算法') + from zhuge import zhuge_consist + data['avg']=data['total_price']/data['area'] + + def guolv_data11(data,neighb): + if type(neighb) is not str: + + print('周边价格=',neighb) + ddd11=data[data['avg']>0.5*neighb] + return ddd11 + else: + return data + + ddd11=guolv_data11(data,neighb[0]) + + if ddd11.shape[0]>2: + + + s=zhuge_consist.url_lable(ddd11) + s1=zhuge_consist.drop_rubbish(s) + #print('s1=',s1) + s2=set(s1) + #print('s2=',s2) + if len(s2)>3: + + from zhuge import zhuge_consist + + s=zhuge_consist.zhuge_out(ddd11) + + def compelex(ddd11,s,neighb,junjia): + cc1=ddd11['avg'].dropna() + + + kmzhuge=k_means(cc1) + + junjia['zhuge_logic']=s + + junjia['kmzg']=kmzhuge + + if type(neighb)!=str and (neighb)!=None: + + junjia['neighb']=neighb + #print('junjia',junjia) + + junjia_front=junjia[['zhuge_logic','kmzg']] + cc=group(junjia_front,junjia,1) + #junjia_km=k_means(junjia) + #print('cc=',cc) + ############ + if len(cc)>0: + cc1=pandas.DataFrame(cc,columns=['price','count']) + cc2=cc1.sort_values(by='count',ascending=False) + #print(bb2) + sample_size=ddd11.shape[0] + return cc2.iloc[0]['price']*1.0,float('%.2f' % credit['0.7']),value_type['第三方'],sample_size + else: + sample_size=ddd11.shape[0] + return junjia['kmzg']*1.0,float('%.2f' % credit['0.6']),value_type['自身样本'],sample_size + + + junjia_km=compelex(ddd11,s,neighb[0],junjia) + + return junjia_km + + + + else: + print('*'*20,'采用周边算法') + if ddd11.shape[0]>3: + cc1=ddd11['avg'].dropna() + kmzhuge=k_means_clus(cc1,2) + sample_size=ddd11.shape[0] + return kmzhuge,float('%.2f' % credit['0.6']),value_type['自身样本'],sample_size + else: + return neighb[0],float('%.2f' % credit['0.5']),value_type['周边楼盘'],neighb[1] + + elif dglg != None: + print('优质第三方均价2 = ',dglg) + sample_size=len(junjia) + return dglg,float('%.2f' % credit['0.8']),value_type['第三方'] ,sample_size #'优质样本' + + else: + return neighb[0],float('%.2f' % credit['0.5']),value_type['周边楼盘'],neighb[1] + + elif dglg != None: + print('优质第三方均价3 = ',dglg) + sample_size=len(junjia) + return dglg,float('%.2f' % credit['0.8']),value_type['第三方'] #'优质样本' + + else: + + return neighb[0],float('%.2f' % credit['0.5']),value_type['周边楼盘'],neighb[1] + + + elif dglg != None: + print('优质第三方均价4 = ',dglg) + return dglg,float('%.2f' % credit['0.8']),value_type['第三方'] ,len(junjia) #'优质样本' + + else: + + #print('*'*100) + #print(neighb) + return neighb[0],float('%.2f' % credit['0.5']),value_type['周边楼盘'],neighb[1] + + + + +client65 = pymongo.MongoClient('192.168.0.105',27777) +db65 = client65.fangjia +seawater65 = db65.seawater + +client136 = pymongo.MongoClient('192.168.0.136',27017) +db136 = client136.fangjia +dsf136 = db136.average_statistics +seaweed136 = db136.seaweed + + +dsf = db136.district_stat + +#client136 = pymongo.MongoClient('192.168.0.136',27017) +#db136 = client136.fangjia + + + + + +def region_district_list(ct,rl): + #import pandas + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + + query1 = {"status":0,"cat":"district","city":ct,"region":rl} + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, + "cat":1,"name":1,'estate_type2':1,'alias':1} + + #lct= list(seaweed136.find(query1, fields1)) + #lf=pandas.DataFrame(lct) + lf=fangjia.find_seaweed(query1,fields1) + return lf + +def city_list(): + #client = pymongo.MongoClient('192.168.0.136',27017) #'192.168.0.65',27777 + #db = client.fangjia + city = db136.server_area + #seawater.find_one() + + + pos= city.find({"display":{"$ne":"全国"}}, + {"display":1,"city_grade":1,"_id":1}).sort([("city_grade",1),("_id",1)]) + + + + data=pandas.DataFrame(list(pos)) + #print(data) + return data['display'] + + +#print(some_district_information()) + + + +def region_list(city): + #client = pymongo.MongoClient('192.168.0.136',27017) + #db = client.fangjia + region = db136.region_block + #seawater.find_one() + query = {"city":city,"category":"region"} + fields1 = {"name":1} + pos = list(region.find(query,fields1)) + + + data=pandas.DataFrame(pos) + return data['name'] + + + + + + + + +def add_alias(ppd): + if 'alias' not in ppd.columns: + ppd['alias']='无' + return ppd + +def ghhh(kkg): + kkg=kkg.fillna('无') + a=list() + for i in kkg.index: + b=list(kkg.loc[[i],:].values.flatten()) + a.append(b) + return a + + + + +client1 = pymongo.MongoClient('192.168.10.182',27017) +db1 = client1.fangjia +stat = db1.distrit_stat + +pt=client1.fangjia_log.calc_command + + +################################################################# + +def region_district_list_lk(ct,rl,lk): + #import pandas + #client1 = pymongo.MongoClient('192.168.0.136',27017) + #db1 = client1.fangjia + #seaweed1 = db1.seaweed + + query1 = {"status":0,"cat":"district","city":ct,"region":rl,"name":lk} + fields1 = {"lat2":1,"lng2":1, "city":1,"region":1, + "cat":1,"name":1,'estate_type2':1,'alias':1} + + #lct= list(seaweed136.find(query1, fields1)) + #lf=pandas.DataFrame(lct) + lf=fangjia.find_seaweed(query1,fields1) + return lf + + + + + +def last_sunday(weekend): + #import datetime + """ + today = datetime.datetime.now() + + today=today+datetime.timedelta(days=-4) + + now_time=datetime.datetime(today.year, today.month, today.day, 0, 0, 0) + week_time = now_time.weekday() + """ + future_time = weekend-datetime.timedelta(days=7-1) + d1= future_time.strftime('%Y%m%d') + weekend1= datetime.datetime.strptime(d1,'%Y%m%d') + weekend=weekend1+datetime.timedelta(hours=-8) + return weekend + + + + + + + +#last_sunday=last_sunday(weekend) + +def make_up_price(ct,rl,lk,weekend): + + + #last_sunday=last_sunday(weekend) + #save_lg=client136.fangjia.district_stat + if lk!=None: + + #cc=last_sunday + + + ly=last_sunday(weekend) + #stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) + stat=dsf + query = {"city":ct,"region":rl,"district_name":lk,'weekend':ly + } + + + pos1=list(stat.find(query).limit(30)) + + pos=pandas.DataFrame(pos1) + + #pos=timestamp_to_int(pos) + #pos=pos[pos['week1']>cc-5] + #pos=pos.sort_values(by='week1',ascending=False) + + #print('5'*10) + #print(pos) + + if pos.shape[0]>0: + #pp1""" + pp1=pos['avg_price'].iloc[0] + if type(pp1)!=None and type(pp1)!=str: + + + + return int(pp1) + else: + return None + else: + return None + + else: + + #stat= fangjia.get_collection("fangjia_stat","test_lg5", '192.168.0.136',27017) + stat=dsf + query = {"city":ct,"region":rl,"district_name":lk,'weekend':last_sunday + } + + + pos1=list(stat.find(query).limit(30)) + + pos=pandas.DataFrame(pos1) + + #pos=timestamp_to_int(pos) + #pos=pos[pos['week1']>cc-5] + #pos=pos.sort_values(by='week1',ascending=False) + + #print('5'*10) + #print(pos) + + if pos.shape[0]>0: + #pp1 + pp1=pos['avg_price'].iloc[0] + if type(pp1)!=None and type(pp1)!=str: + + + + return int(pp1) + else: + return None + else: + return None + + + + +#weekend=weekend00() + +@fangjia.timer +def out_put_price(ct,rl,lk,weekend): + lf=region_district_list_lk(ct,rl,lk) + lf=add_alias(lf) + + p=ghhh(lf[['name','alias','estate_type2']])[0] + + mm=commonprice(ct,rl,p) + + c_date = datetime.datetime.utcnow() + m_date=datetime.datetime.utcnow() + + if type(mm[0])!=str: + + print(ct,rl,lk,mm,'$'*20) + #print(lf['estate_type2'].iloc[0],'*'*10) + estate_type=lf['estate_type2'].iloc[0] + + + + #last_sunday=last_sunday() + #t1=time.time() + last_sunday_price=make_up_price(ct,rl,lk,weekend) + #print(time.time()-t1,'*'*20) + + #print('last_sunday_price',last_sunday_price) + #print('mm[0]',mm[0]) + + def make_up(): + if type(last_sunday_price) != str and last_sunday_price != None and mm[0]!=None and type(mm[0])!=str: + + make_up=10000*(mm[0]-last_sunday_price)/last_sunday_price + return make_up + else: + return 0 + + make_up=int(make_up()) + + + try: + + stat.replace_one( + + {"city":ct,"region":rl,"district_name":p[0],"weekend":weekend}, + + {"city":ct,"region":rl,"district_name":p[0],'credit':mm[1], + "calc_value":int(mm[0]), "weekend":weekend,"c_date":c_date,'avg_price':int(mm[0]), + "m_date":m_date,'value_type':int(mm[2]),'weekend_markup':make_up, + 'changed_markup':make_up,'markup':make_up,'sample_size':mm[3], + 'estate_type':estate_type},True + ) + + + + pt.replace_one( + + {"city":ct,"region":rl,"district_name":p[0],"weekend":weekend}, + + + {"city":ct,"region":rl,"district_name":p[0],"c_date":c_date,'status':int(0), + 'remark':'week','category':'sell','weekend':weekend,'priority':int(0), + 'problem':None, + },True + ) + + + except: + + stat.replace_one( + + {"city":ct,"region":rl,"district_name":p[0],"weekend":weekend}, + + {"city":ct,"region":rl,"district_name":p[0],'credit':mm[1], + "calc_value":int(mm[0]), "weekend":weekend,"c_date":c_date,'avg_price':int(mm[0]), + "m_date":m_date,'value_type':int(mm[2]),'weekend_markup':make_up, + 'changed_markup':make_up,'markup':make_up,'sample_size':mm[3], + 'estate_type':estate_type},True + ) + + else: + pt.replace_one( + + {"city":ct,"region":rl,"district_name":p[0],"weekend":weekend}, + + + {"city":ct,"region":rl,"district_name":p[0],"c_date":c_date,'status':int(1), + 'remark':'week','category':'sell','weekend':weekend,'priority':int(0), + 'problem':mm[0], + },True + ) + + + \ No newline at end of file