from pymongo import MongoClient from bson.codec_options import CodecOptions import datetime from common import configs import requests import common import pytz from elasticsearch import Elasticsearch from elasticsearch import helpers import pandas as pd from retrying import retry from geojson_utils import destination_point from sklearn.cluster import KMeans from sklearn import metrics import numpy as np es = Elasticsearch([configs.SEAWATER_ES_HOST], http_auth=configs.SEAWATER_ES_HTTP_AUTH, scheme="http", port=9200, retry_on_timeout=True, timeout=60000) def find_house(query, index, limit=None): houses = list() for doc in helpers.scan(es, query=query, scroll="10m", index=index, doc_type="_doc"): doc = doc["_source"] if isinstance(doc["p_date"], str): doc["p_date"] = int(doc["p_date"]) houses.append(doc) if limit is None and limit == len(houses): break return houses @retry(stop_max_attempt_number=10, wait_fixed=5000) def load_house_data(city=None, region=None, name=None, start_date=None, end_date=None, cat=None, rent_type=None, alias=None, limit=None, room=None): source = ["city", "region", "district_name", "direction", "fitment", "total_price", "room", "p_date", "area", "avg_price", "root"] must = list() if cat is not None: must.append({"term": {"cat": cat}}) if rent_type is not None: must.append({"term": {"rent_type": rent_type}}) if city is not None: must.append({"term": {"city": city}}) if region is not None: must.append({"term": {"region": region}}) names = list() if name is not None: names.append(name) if alias is not None and len(alias) > 0: for alia in alias: if alia is not None and alia not in names: names.append(alia) if len(names) > 0: must.append({"terms": {"district_name": names}}) if start_date is not None or end_date is not None: publish_date_range = {} if start_date is not None: publish_date_range["gt"] = start_date if end_date is not None: publish_date_range["lt"] = end_date must.append({"range": {"p_date": publish_date_range}}) if room: must.append({"term": {"room": room}}) else: must.append({"range": {"room": {"gte": 1, "lt": 4}}}) must.append({"range": {"area": {"gte": 20, "lte": 260}}}) must.append({"range": {"avg_price": {"gte": 1}}}) query = { "_source": source, "query": { "bool": { "must": must } }, "sort": [ { "publish_date": "DESC" } ] } return pd.DataFrame(find_house(query, "latest_seawater", limit)) def get_last_weekend(): sunday = datetime.datetime.utcnow() sunday = sunday.replace(hour=0, minute=0, second=0, microsecond=0) one_day = datetime.timedelta(days=1) while sunday.weekday() != 6: sunday -= one_day pacific = pytz.timezone('Asia/Shanghai') return pacific.localize(sunday) def get_mongodb_collection(uri, db, collection): return MongoClient(uri).get_database(db).get_collection(collection) \ .with_options(codec_options=CodecOptions(tz_aware=True, tzinfo=pytz.timezone('Asia/Shanghai'))) COMMAND = get_mongodb_collection(configs.LOG_MONGO_URI, "fangjia_log", "calc_command") SERVER_AREA = get_mongodb_collection(configs.MONGO_URI, "fangjia", "server_area") DISTRICT_STAT = get_mongodb_collection(configs.MONGO_URI, "fangjia_stat", "district_stat") REGION_STAT = get_mongodb_collection(configs.MONGO_URI, "fangjia", "region_stat") REGION_BLOCK = get_mongodb_collection(configs.MONGO_URI, "fangjia", "region_block") SEAWEED = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia", "seaweed") def send_refitted_trend(city, region, name, estate_type, weekend): cmd = { "city": city, "region": region, "name": name, "estate_type": estate_type, "calc_date": weekend, "calc_type": 0, "category": "sell", "priority": 3, "status": 102, "remark": "week", "redraw_trend_type": 1, "ip": "192.168.0.24", "c_date": datetime.utcnow() } COMMAND.save(cmd) def get_least_district_stat(city, region, name, estate_type, start_date=None, end_date=None, limit=1): if estate_type is not None and "别墅" in estate_type: estate_type = "别墅" else: estate_type = "小区" query = { "city": city, "region": region, "district_name": name, "estate_type": estate_type } weekend = {} if start_date is not None: weekend["$gte"] = start_date if end_date is not None: weekend["$lte"] = end_date if len(weekend) != 0: query["weekend"] = weekend stats = list(DISTRICT_STAT.find(query).sort([("weekend", -1)]).limit(limit)) return stats def get_least_region_stat(city, region=None, start_date=None, end_date=None, limit=1): query = { "city": city, "region": region, "block": None } weekend = {} if start_date is not None: weekend["$gte"] = start_date if end_date is not None: weekend["$lte"] = end_date if len(weekend) != 0: query["weekend"] = weekend stats = list(REGION_STAT.find(query).sort([("weekend", -1)]).limit(limit)) return stats def get_all_city(): return [doc["display"] for doc in SERVER_AREA.find({"display": {"$ne": "全国"}})] def get_regions(city): return [region["name"] for region in list(REGION_BLOCK.find({"city": city, "category": "region"}))] def get_seaweed(city, region=None, name=None): query = {"cat": "district", "status": 0, "city": city} if region is not None: query["region"] = region if name is not None: query["name"] = name keys = {"city": 1, "region": 1, "name": 1, "alias": 1, "estate_type2": 1, "lng2": 1, "lat2": 1, "finish_building_date": 1} return list(SEAWEED.find(query, keys)) def get_nearby_seaweed(city, lng, lat, radius): query = {"cat": "district", "status": 0, "city": city} box = get_geo_box(lng=lng/10000000000, lat=lat/10000000000, radius=radius) query["lat2"] = {"$gte": box["min_lat"] * 10000000000, "$lte": box["max_lat"] * 10000000000} query["lng2"] = {"$gte": box["min_lng"] * 10000000000, "$lte": box["max_lng"] * 10000000000} keys = {"city": 1, "region": 1, "name": 1, "alias": 1, "estate_type2": 1, "lng2": 1, "lat2": 1, "finish_building_date": 1} return list(SEAWEED.find(query, keys)) def match(city, address, region=""): params = { "city": city, "address": address, "region": region, "token": "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E\ 05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" } return requests.get("http://open.fangjia.com/address/match", params).json()["result"] def parse_date(time, fm='%Y-%m-%d'): pacific = pytz.timezone('Asia/Shanghai') return pacific.localize(datetime.datetime.strptime(time, fm)) def localize(time): pacific = pytz.timezone('Asia/Shanghai') return pacific.localize(time) def format_date(time, fm='%Y%m%d'): return time.strftime(fm) def get_geo_box(lng, lat, radius): central_point = {"type": "Point", "coordinates": [lng, lat]} right_point = destination_point(central_point, 45, radius/1000) left_point = destination_point(central_point, 225, radius/1000) return {"max_lng": right_point["coordinates"][0], "min_lng": left_point["coordinates"][0], "min_lat": left_point["coordinates"][1], "max_lat": right_point["coordinates"][1]} def k_means(series): values = np.array(list(series)) length = len(set(series)) if length < 2: return values.mean() values.sort() length = len(values) if length > 10: values = values[int(length * 0.2):int(length * 0.8)] length = len(set(series)) if length <= 2: return values.mean() cluster = None max_score = -1 values = np.array(list(series)) values = values.reshape(len(values), 1) for n_clusters in range(2, min(len(set(series)), 6)): _cluster = KMeans(n_clusters=n_clusters, max_iter=10000).fit(values) score = metrics.silhouette_score(values, _cluster.predict(values), metric='euclidean') if score > max_score: cluster = _cluster max_score = score cluster_centers = list(cluster.cluster_centers_) labels_ = cluster.labels_ labels_count = dict() for label in labels_: if label in labels_count: labels_count[label] = labels_count[label] + 1 else: labels_count[label] = 1 max_cluster_label = max(labels_count.items(), key=lambda x: x[1])[0] return cluster_centers[max_cluster_label][0]