Files
Sklearn/rent/calc_by_svg.py
2019-05-07 11:33:15 +08:00

177 lines
6.3 KiB
Python

from sklearn import svm
import re
import pandas as pd
import warnings
import numpy as np
from dateutil.relativedelta import relativedelta
from sklearn.model_selection import GridSearchCV, train_test_split
from sklearn.pipeline import Pipeline
from sklearn import preprocessing
import datetime
import os
import sys
curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)
import common
from common import configs as cfg
warnings.simplefilter("ignore")
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
district_room_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_room_rent")
district_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
district_sell_stat = common.get_mongodb_collection(cfg.MONGO_URI, "fangjia_stat", "district_sell_stat")
def has_geo_info(seaweed):
return "lat2" in seaweed and "lng2" in seaweed and seaweed["lng2"] > 0
def get_build_year(seaweed):
if "finish_building_date" not in seaweed or seaweed["finish_building_date"] is None:
return None
pattern = re.compile(r'\d{4}')
match = pattern.match(str(seaweed["finish_building_date"]))
if not match:
return None
return int(match.group())
def is_common_community(seaweed):
return seaweed["estate_type2"] == "普通住宅"
def get_apply_seaweed(city, year_month):
stats = list(district_sell_stat.find({"city": city, "date": year_month}))
stat_dict = dict()
for stat in stats:
key = stat["city"] + "_" + stat["region"] + "_" + stat["district_name"]
stat_dict[key] = stat["avg_price"]
seaweed_list = list()
for seaweed in common.get_seaweed(city):
if not has_geo_info(seaweed):
continue
if not is_common_community(seaweed):
continue
build_year = get_build_year(seaweed)
if not build_year:
continue
key = seaweed["city"] + "_" + seaweed["region"] + "_" + seaweed["name"]
if key not in stat_dict:
continue
seaweed["build_year"] = build_year
seaweed["lat"] = seaweed["lat2"] / 10000000000
seaweed["lng"] = seaweed["lng2"] / 10000000000
seaweed["avg_price"] = stat_dict[key]
seaweed_list.append(seaweed)
return seaweed_list
def get_rent_table_by_calc_type(calc_type):
if calc_type == "ht":
return district_room_rent
elif calc_type == "sq":
return district_rent
else:
return None
def load_train_data(city, year_month, calc_type):
"""
:param city:
:param year_month:
:param calc_type:计算类型 ht:户型租金 sq: 平米租金
:return:
"""
table_name = get_rent_table_by_calc_type(calc_type)
data = list()
for seaweed in get_apply_seaweed(city, year_month):
query = {"city": seaweed["city"], "region": seaweed["region"], "name": seaweed["name"], "date": year_month,
"calc_model": {"$ne": "svr"}}
stats = list(table_name.find(query))
if len(stats) < 3:
continue
for stat in stats:
item = seaweed.copy()
item["room"] = stat["room"]
item["rent_price"] = stat["rent_price"]
data.append(item)
print(item)
return pd.DataFrame(data)
def has_rent_price(seaweed, room, year_month, calc_type):
table_name = get_rent_table_by_calc_type(calc_type)
query = {"city": seaweed["city"], "region": seaweed["region"], "name": seaweed["name"],
"date": year_month, "room": room}
return table_name.count(query) > 0
def train(city, year_month, calc_type):
"""
:param city:
:param year_month:
:param calc_type:计算类型 ht:户型租金 sq: 平米租金
:return:
"""
train_data = load_train_data(city, year_month, calc_type)
x = train_data[["lat", "lng", "build_year", "room", "avg_price"]]
y = train_data[["rent_price"]]
x = x.astype(np.float64)
y = y.astype(np.float64)
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.3)
c_range = np.logspace(-5, 15, 11, base=2)
gamma_range = np.logspace(-9, 3, 13, base=2)
param_grid = [{'svr__kernel': ['rbf', 'linear'], 'svr__C': c_range, 'svr__gamma': gamma_range}]
svr = svm.SVR()
pipeline = Pipeline([('standard', preprocessing.StandardScaler()),
('minMax', preprocessing.MinMaxScaler()),
('svr', svr)])
grid = GridSearchCV(pipeline, param_grid, cv=3, n_jobs=10)
clf = grid.fit(x_train, y_train.values.ravel())
score = grid.score(x_test, y_test)
print('精度为%s' % score)
predict_seaweed = list()
for seaweed in get_apply_seaweed(city, year_month):
for room in range(1, 4):
if not has_rent_price(seaweed, room, year_month, calc_type):
item = seaweed.copy()
item["room"] = room
predict_seaweed.append(item)
predict_df = pd.DataFrame(predict_seaweed)
result = clf.predict(predict_df[["lat", "lng", "build_year", "room", "avg_price"]])
result = pd.DataFrame([[int(val)] for val in result])
result.columns = ["rent"]
predict_df = predict_df.join(result)
predict_df = predict_df[["city", "region", "name", "room", "rent"]]
table_name = get_rent_table_by_calc_type(calc_type)
for index, row in predict_df.iterrows():
if row["rent"] <= 0:
continue
query = {
"date": year_month,
"city": row["city"],
"region": row["region"],
"name": row["name"],
"room": int(row["room"])
}
data = dict()
data["m_date"] = datetime.datetime.utcnow()
data["rent_price"] = row["rent"]
data["calc_model"] = "svr"
table_name.update_one(query, {"$set": data}, True)
print(row)
if __name__ == "__main__":
start_date = common.parse_date("2019-05-01")
now_date = common.localize(datetime.datetime.utcnow())
while start_date < now_date:
year_month = int(common.format_date(start_date - relativedelta(months=1), '%Y%m'))
for city in ["北京"]:
train("北京", year_month, "ht")
train("北京", year_month, "sq")
start_date += relativedelta(months=1)