262 lines
8.9 KiB
Python
262 lines
8.9 KiB
Python
from pymongo import MongoClient
|
|
from bson.codec_options import CodecOptions
|
|
import datetime
|
|
from common import configs
|
|
import requests
|
|
import common
|
|
import pytz
|
|
from elasticsearch import Elasticsearch
|
|
from elasticsearch import helpers
|
|
import pandas as pd
|
|
from retrying import retry
|
|
from geojson_utils import destination_point
|
|
from sklearn.cluster import KMeans
|
|
from sklearn import metrics
|
|
import numpy as np
|
|
|
|
es = Elasticsearch([configs.SEAWATER_ES_HOST], http_auth=configs.SEAWATER_ES_HTTP_AUTH, scheme="http",
|
|
port=9200, retry_on_timeout=True, timeout=60000)
|
|
|
|
|
|
def find_house(query, index, limit=None):
|
|
houses = list()
|
|
for doc in helpers.scan(es, query=query, scroll="10m", index=index, doc_type="_doc"):
|
|
doc = doc["_source"]
|
|
if isinstance(doc["p_date"], str):
|
|
doc["p_date"] = int(doc["p_date"])
|
|
houses.append(doc)
|
|
if limit is None and limit == len(houses):
|
|
break
|
|
return houses
|
|
|
|
|
|
@retry(stop_max_attempt_number=10, wait_fixed=5000)
|
|
def load_house_data(city=None, region=None, name=None, start_date=None, end_date=None, cat=None,
|
|
rent_type=None, alias=None, limit=None, room=None):
|
|
source = ["city", "region", "district_name", "direction", "fitment", "total_price", "room", "p_date", "area",
|
|
"avg_price", "root"]
|
|
must = list()
|
|
if cat is not None:
|
|
must.append({"term": {"cat": cat}})
|
|
if rent_type is not None:
|
|
must.append({"term": {"rent_type": rent_type}})
|
|
if city is not None:
|
|
must.append({"term": {"city": city}})
|
|
if region is not None:
|
|
must.append({"term": {"region": region}})
|
|
names = list()
|
|
if name is not None:
|
|
names.append(name)
|
|
if alias is not None and len(alias) > 0:
|
|
for alia in alias:
|
|
if alia is not None and alia not in names:
|
|
names.append(alia)
|
|
if len(names) > 0:
|
|
must.append({"terms": {"district_name": names}})
|
|
if start_date is not None or end_date is not None:
|
|
publish_date_range = {}
|
|
if start_date is not None:
|
|
publish_date_range["gt"] = start_date
|
|
if end_date is not None:
|
|
publish_date_range["lt"] = end_date
|
|
must.append({"range": {"p_date": publish_date_range}})
|
|
if room:
|
|
must.append({"term": {"room": room}})
|
|
else:
|
|
must.append({"range": {"room": {"gte": 1, "lt": 4}}})
|
|
must.append({"range": {"area": {"gte": 20, "lte": 260}}})
|
|
must.append({"range": {"avg_price": {"gte": 1}}})
|
|
query = {
|
|
"_source": source,
|
|
"query": {
|
|
"bool": {
|
|
"must": must
|
|
}
|
|
},
|
|
"sort": [
|
|
{
|
|
"publish_date": "DESC"
|
|
}
|
|
]
|
|
}
|
|
return pd.DataFrame(find_house(query, "latest_seawater", limit))
|
|
|
|
|
|
def get_last_weekend():
|
|
sunday = datetime.datetime.utcnow()
|
|
sunday = sunday.replace(hour=0, minute=0, second=0, microsecond=0)
|
|
one_day = datetime.timedelta(days=1)
|
|
while sunday.weekday() != 6:
|
|
sunday -= one_day
|
|
pacific = pytz.timezone('Asia/Shanghai')
|
|
return pacific.localize(sunday)
|
|
|
|
|
|
def get_mongodb_collection(uri, db, collection):
|
|
return MongoClient(uri).get_database(db).get_collection(collection) \
|
|
.with_options(codec_options=CodecOptions(tz_aware=True, tzinfo=pytz.timezone('Asia/Shanghai')))
|
|
|
|
|
|
COMMAND = get_mongodb_collection(configs.LOG_MONGO_URI, "fangjia_log", "calc_command")
|
|
SERVER_AREA = get_mongodb_collection(configs.MONGO_URI, "fangjia", "server_area")
|
|
DISTRICT_STAT = get_mongodb_collection(configs.MONGO_URI, "fangjia_stat", "district_stat")
|
|
REGION_STAT = get_mongodb_collection(configs.MONGO_URI, "fangjia", "region_stat")
|
|
REGION_BLOCK = get_mongodb_collection(configs.MONGO_URI, "fangjia", "region_block")
|
|
SEAWEED = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia", "seaweed")
|
|
|
|
|
|
def send_refitted_trend(city, region, name, estate_type, weekend):
|
|
cmd = {
|
|
"city": city,
|
|
"region": region,
|
|
"name": name,
|
|
"estate_type": estate_type,
|
|
"calc_date": weekend,
|
|
"calc_type": 0,
|
|
"category": "sell",
|
|
"priority": 3,
|
|
"status": 102,
|
|
"remark": "week",
|
|
"redraw_trend_type": 1,
|
|
"ip": "192.168.0.24",
|
|
"c_date": datetime.utcnow()
|
|
}
|
|
COMMAND.save(cmd)
|
|
|
|
|
|
def get_least_district_stat(city, region, name, estate_type, start_date=None, end_date=None, limit=1):
|
|
if estate_type is not None and "别墅" in estate_type:
|
|
estate_type = "别墅"
|
|
else:
|
|
estate_type = "小区"
|
|
query = {
|
|
"city": city,
|
|
"region": region,
|
|
"district_name": name,
|
|
"estate_type": estate_type
|
|
}
|
|
weekend = {}
|
|
if start_date is not None:
|
|
weekend["$gte"] = start_date
|
|
if end_date is not None:
|
|
weekend["$lte"] = end_date
|
|
if len(weekend) != 0:
|
|
query["weekend"] = weekend
|
|
stats = list(DISTRICT_STAT.find(query).sort([("weekend", -1)]).limit(limit))
|
|
return stats
|
|
|
|
|
|
def get_least_region_stat(city, region=None, start_date=None, end_date=None, limit=1):
|
|
query = {
|
|
"city": city,
|
|
"region": region,
|
|
"block": None
|
|
}
|
|
weekend = {}
|
|
if start_date is not None:
|
|
weekend["$gte"] = start_date
|
|
if end_date is not None:
|
|
weekend["$lte"] = end_date
|
|
if len(weekend) != 0:
|
|
query["weekend"] = weekend
|
|
stats = list(REGION_STAT.find(query).sort([("weekend", -1)]).limit(limit))
|
|
return stats
|
|
|
|
|
|
def get_all_city():
|
|
return [doc["display"] for doc in SERVER_AREA.find({"display": {"$ne": "全国"}})]
|
|
|
|
|
|
def get_regions(city):
|
|
return [region["name"] for region in list(REGION_BLOCK.find({"city": city, "category": "region"}))]
|
|
|
|
|
|
def get_seaweed(city, region=None, name=None):
|
|
query = {"cat": "district", "status": 0, "city": city}
|
|
if region is not None:
|
|
query["region"] = region
|
|
if name is not None:
|
|
query["name"] = name
|
|
keys = {"city": 1, "region": 1, "name": 1, "alias": 1, "estate_type2": 1, "lng2": 1, "lat2": 1,
|
|
"finish_building_date": 1}
|
|
return list(SEAWEED.find(query, keys))
|
|
|
|
|
|
def get_nearby_seaweed(city, lng, lat, radius):
|
|
query = {"cat": "district", "status": 0, "city": city}
|
|
box = get_geo_box(lng=lng/10000000000, lat=lat/10000000000, radius=radius)
|
|
query["lat2"] = {"$gte": box["min_lat"] * 10000000000, "$lte": box["max_lat"] * 10000000000}
|
|
query["lng2"] = {"$gte": box["min_lng"] * 10000000000, "$lte": box["max_lng"] * 10000000000}
|
|
keys = {"city": 1, "region": 1, "name": 1, "alias": 1, "estate_type2": 1, "lng2": 1, "lat2": 1,
|
|
"finish_building_date": 1}
|
|
return list(SEAWEED.find(query, keys))
|
|
|
|
|
|
def match(city, address, region=""):
|
|
params = {
|
|
"city": city,
|
|
"address": address,
|
|
"region": region,
|
|
"token": "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E\
|
|
05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
|
}
|
|
return requests.get("http://open.fangjia.com/address/match", params).json()["result"]
|
|
|
|
|
|
def parse_date(time, fm='%Y-%m-%d'):
|
|
pacific = pytz.timezone('Asia/Shanghai')
|
|
return pacific.localize(datetime.datetime.strptime(time, fm))
|
|
|
|
|
|
def localize(time):
|
|
pacific = pytz.timezone('Asia/Shanghai')
|
|
return pacific.localize(time)
|
|
|
|
|
|
def format_date(time, fm='%Y%m%d'):
|
|
return time.strftime(fm)
|
|
|
|
|
|
def get_geo_box(lng, lat, radius):
|
|
central_point = {"type": "Point", "coordinates": [lng, lat]}
|
|
right_point = destination_point(central_point, 45, radius/1000)
|
|
left_point = destination_point(central_point, 225, radius/1000)
|
|
return {"max_lng": right_point["coordinates"][0],
|
|
"min_lng": left_point["coordinates"][0],
|
|
"min_lat": left_point["coordinates"][1],
|
|
"max_lat": right_point["coordinates"][1]}
|
|
|
|
|
|
def k_means(series):
|
|
values = np.array(list(series))
|
|
length = len(set(series))
|
|
if length < 2:
|
|
return values.mean()
|
|
values.sort()
|
|
length = len(values)
|
|
if length > 10:
|
|
values = values[int(length * 0.2):int(length * 0.8)]
|
|
length = len(set(series))
|
|
if length <= 2:
|
|
return values.mean()
|
|
cluster = None
|
|
max_score = -1
|
|
values = np.array(list(series))
|
|
values = values.reshape(len(values), 1)
|
|
for n_clusters in range(2, min(len(set(series)), 6)):
|
|
_cluster = KMeans(n_clusters=n_clusters, max_iter=10000).fit(values)
|
|
score = metrics.silhouette_score(values, _cluster.predict(values), metric='euclidean')
|
|
if score > max_score:
|
|
cluster = _cluster
|
|
max_score = score
|
|
cluster_centers = list(cluster.cluster_centers_)
|
|
labels_ = cluster.labels_
|
|
labels_count = dict()
|
|
for label in labels_:
|
|
if label in labels_count:
|
|
labels_count[label] = labels_count[label] + 1
|
|
else:
|
|
labels_count[label] = 1
|
|
max_cluster_label = max(labels_count.items(), key=lambda x: x[1])[0]
|
|
return cluster_centers[max_cluster_label][0]
|