Files
2019-05-07 11:33:15 +08:00

262 lines
8.9 KiB
Python

from pymongo import MongoClient
from bson.codec_options import CodecOptions
import datetime
from common import configs
import requests
import common
import pytz
from elasticsearch import Elasticsearch
from elasticsearch import helpers
import pandas as pd
from retrying import retry
from geojson_utils import destination_point
from sklearn.cluster import KMeans
from sklearn import metrics
import numpy as np
es = Elasticsearch([configs.SEAWATER_ES_HOST], http_auth=configs.SEAWATER_ES_HTTP_AUTH, scheme="http",
port=9200, retry_on_timeout=True, timeout=60000)
def find_house(query, index, limit=None):
houses = list()
for doc in helpers.scan(es, query=query, scroll="10m", index=index, doc_type="_doc"):
doc = doc["_source"]
if isinstance(doc["p_date"], str):
doc["p_date"] = int(doc["p_date"])
houses.append(doc)
if limit is None and limit == len(houses):
break
return houses
@retry(stop_max_attempt_number=10, wait_fixed=5000)
def load_house_data(city=None, region=None, name=None, start_date=None, end_date=None, cat=None,
rent_type=None, alias=None, limit=None, room=None):
source = ["city", "region", "district_name", "direction", "fitment", "total_price", "room", "p_date", "area",
"avg_price", "root"]
must = list()
if cat is not None:
must.append({"term": {"cat": cat}})
if rent_type is not None:
must.append({"term": {"rent_type": rent_type}})
if city is not None:
must.append({"term": {"city": city}})
if region is not None:
must.append({"term": {"region": region}})
names = list()
if name is not None:
names.append(name)
if alias is not None and len(alias) > 0:
for alia in alias:
if alia is not None and alia not in names:
names.append(alia)
if len(names) > 0:
must.append({"terms": {"district_name": names}})
if start_date is not None or end_date is not None:
publish_date_range = {}
if start_date is not None:
publish_date_range["gt"] = start_date
if end_date is not None:
publish_date_range["lt"] = end_date
must.append({"range": {"p_date": publish_date_range}})
if room:
must.append({"term": {"room": room}})
else:
must.append({"range": {"room": {"gte": 1, "lt": 4}}})
must.append({"range": {"area": {"gte": 20, "lte": 260}}})
must.append({"range": {"avg_price": {"gte": 1}}})
query = {
"_source": source,
"query": {
"bool": {
"must": must
}
},
"sort": [
{
"publish_date": "DESC"
}
]
}
return pd.DataFrame(find_house(query, "latest_seawater", limit))
def get_last_weekend():
sunday = datetime.datetime.utcnow()
sunday = sunday.replace(hour=0, minute=0, second=0, microsecond=0)
one_day = datetime.timedelta(days=1)
while sunday.weekday() != 6:
sunday -= one_day
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(sunday)
def get_mongodb_collection(uri, db, collection):
return MongoClient(uri).get_database(db).get_collection(collection) \
.with_options(codec_options=CodecOptions(tz_aware=True, tzinfo=pytz.timezone('Asia/Shanghai')))
COMMAND = get_mongodb_collection(configs.LOG_MONGO_URI, "fangjia_log", "calc_command")
SERVER_AREA = get_mongodb_collection(configs.MONGO_URI, "fangjia", "server_area")
DISTRICT_STAT = get_mongodb_collection(configs.MONGO_URI, "fangjia_stat", "district_stat")
REGION_STAT = get_mongodb_collection(configs.MONGO_URI, "fangjia", "region_stat")
REGION_BLOCK = get_mongodb_collection(configs.MONGO_URI, "fangjia", "region_block")
SEAWEED = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia", "seaweed")
def send_refitted_trend(city, region, name, estate_type, weekend):
cmd = {
"city": city,
"region": region,
"name": name,
"estate_type": estate_type,
"calc_date": weekend,
"calc_type": 0,
"category": "sell",
"priority": 3,
"status": 102,
"remark": "week",
"redraw_trend_type": 1,
"ip": "192.168.0.24",
"c_date": datetime.utcnow()
}
COMMAND.save(cmd)
def get_least_district_stat(city, region, name, estate_type, start_date=None, end_date=None, limit=1):
if estate_type is not None and "别墅" in estate_type:
estate_type = "别墅"
else:
estate_type = "小区"
query = {
"city": city,
"region": region,
"district_name": name,
"estate_type": estate_type
}
weekend = {}
if start_date is not None:
weekend["$gte"] = start_date
if end_date is not None:
weekend["$lte"] = end_date
if len(weekend) != 0:
query["weekend"] = weekend
stats = list(DISTRICT_STAT.find(query).sort([("weekend", -1)]).limit(limit))
return stats
def get_least_region_stat(city, region=None, start_date=None, end_date=None, limit=1):
query = {
"city": city,
"region": region,
"block": None
}
weekend = {}
if start_date is not None:
weekend["$gte"] = start_date
if end_date is not None:
weekend["$lte"] = end_date
if len(weekend) != 0:
query["weekend"] = weekend
stats = list(REGION_STAT.find(query).sort([("weekend", -1)]).limit(limit))
return stats
def get_all_city():
return [doc["display"] for doc in SERVER_AREA.find({"display": {"$ne": "全国"}})]
def get_regions(city):
return [region["name"] for region in list(REGION_BLOCK.find({"city": city, "category": "region"}))]
def get_seaweed(city, region=None, name=None):
query = {"cat": "district", "status": 0, "city": city}
if region is not None:
query["region"] = region
if name is not None:
query["name"] = name
keys = {"city": 1, "region": 1, "name": 1, "alias": 1, "estate_type2": 1, "lng2": 1, "lat2": 1,
"finish_building_date": 1}
return list(SEAWEED.find(query, keys))
def get_nearby_seaweed(city, lng, lat, radius):
query = {"cat": "district", "status": 0, "city": city}
box = get_geo_box(lng=lng/10000000000, lat=lat/10000000000, radius=radius)
query["lat2"] = {"$gte": box["min_lat"] * 10000000000, "$lte": box["max_lat"] * 10000000000}
query["lng2"] = {"$gte": box["min_lng"] * 10000000000, "$lte": box["max_lng"] * 10000000000}
keys = {"city": 1, "region": 1, "name": 1, "alias": 1, "estate_type2": 1, "lng2": 1, "lat2": 1,
"finish_building_date": 1}
return list(SEAWEED.find(query, keys))
def match(city, address, region=""):
params = {
"city": city,
"address": address,
"region": region,
"token": "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E\
05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
}
return requests.get("http://open.fangjia.com/address/match", params).json()["result"]
def parse_date(time, fm='%Y-%m-%d'):
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(datetime.datetime.strptime(time, fm))
def localize(time):
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(time)
def format_date(time, fm='%Y%m%d'):
return time.strftime(fm)
def get_geo_box(lng, lat, radius):
central_point = {"type": "Point", "coordinates": [lng, lat]}
right_point = destination_point(central_point, 45, radius/1000)
left_point = destination_point(central_point, 225, radius/1000)
return {"max_lng": right_point["coordinates"][0],
"min_lng": left_point["coordinates"][0],
"min_lat": left_point["coordinates"][1],
"max_lat": right_point["coordinates"][1]}
def k_means(series):
values = np.array(list(series))
length = len(set(series))
if length < 2:
return values.mean()
values.sort()
length = len(values)
if length > 10:
values = values[int(length * 0.2):int(length * 0.8)]
length = len(set(series))
if length <= 2:
return values.mean()
cluster = None
max_score = -1
values = np.array(list(series))
values = values.reshape(len(values), 1)
for n_clusters in range(2, min(len(set(series)), 6)):
_cluster = KMeans(n_clusters=n_clusters, max_iter=10000).fit(values)
score = metrics.silhouette_score(values, _cluster.predict(values), metric='euclidean')
if score > max_score:
cluster = _cluster
max_score = score
cluster_centers = list(cluster.cluster_centers_)
labels_ = cluster.labels_
labels_count = dict()
for label in labels_:
if label in labels_count:
labels_count[label] = labels_count[label] + 1
else:
labels_count[label] = 1
max_cluster_label = max(labels_count.items(), key=lambda x: x[1])[0]
return cluster_centers[max_cluster_label][0]