177 lines
6.3 KiB
Python
177 lines
6.3 KiB
Python
from sklearn import svm
|
|
import re
|
|
import pandas as pd
|
|
import warnings
|
|
import numpy as np
|
|
from dateutil.relativedelta import relativedelta
|
|
from sklearn.model_selection import GridSearchCV, train_test_split
|
|
from sklearn.pipeline import Pipeline
|
|
from sklearn import preprocessing
|
|
import datetime
|
|
import os
|
|
import sys
|
|
curPath = os.path.abspath(os.path.dirname(__file__))
|
|
rootPath = os.path.split(curPath)[0]
|
|
sys.path.append(rootPath)
|
|
import common
|
|
from common import configs as cfg
|
|
warnings.simplefilter("ignore")
|
|
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
|
|
district_room_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_room_rent")
|
|
district_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
|
|
district_sell_stat = common.get_mongodb_collection(cfg.MONGO_URI, "fangjia_stat", "district_sell_stat")
|
|
|
|
|
|
def has_geo_info(seaweed):
|
|
return "lat2" in seaweed and "lng2" in seaweed and seaweed["lng2"] > 0
|
|
|
|
|
|
def get_build_year(seaweed):
|
|
if "finish_building_date" not in seaweed or seaweed["finish_building_date"] is None:
|
|
return None
|
|
pattern = re.compile(r'\d{4}')
|
|
match = pattern.match(str(seaweed["finish_building_date"]))
|
|
if not match:
|
|
return None
|
|
return int(match.group())
|
|
|
|
|
|
def is_common_community(seaweed):
|
|
return seaweed["estate_type2"] == "普通住宅"
|
|
|
|
|
|
def get_apply_seaweed(city, year_month):
|
|
stats = list(district_sell_stat.find({"city": city, "date": year_month}))
|
|
stat_dict = dict()
|
|
for stat in stats:
|
|
key = stat["city"] + "_" + stat["region"] + "_" + stat["district_name"]
|
|
stat_dict[key] = stat["avg_price"]
|
|
seaweed_list = list()
|
|
for seaweed in common.get_seaweed(city):
|
|
if not has_geo_info(seaweed):
|
|
continue
|
|
if not is_common_community(seaweed):
|
|
continue
|
|
build_year = get_build_year(seaweed)
|
|
if not build_year:
|
|
continue
|
|
key = seaweed["city"] + "_" + seaweed["region"] + "_" + seaweed["name"]
|
|
if key not in stat_dict:
|
|
continue
|
|
seaweed["build_year"] = build_year
|
|
seaweed["lat"] = seaweed["lat2"] / 10000000000
|
|
seaweed["lng"] = seaweed["lng2"] / 10000000000
|
|
seaweed["avg_price"] = stat_dict[key]
|
|
seaweed_list.append(seaweed)
|
|
return seaweed_list
|
|
|
|
|
|
def get_rent_table_by_calc_type(calc_type):
|
|
if calc_type == "ht":
|
|
return district_room_rent
|
|
elif calc_type == "sq":
|
|
return district_rent
|
|
else:
|
|
return None
|
|
|
|
|
|
def load_train_data(city, year_month, calc_type):
|
|
"""
|
|
|
|
:param city:
|
|
:param year_month:
|
|
:param calc_type:计算类型 ht:户型租金 sq: 平米租金
|
|
:return:
|
|
"""
|
|
table_name = get_rent_table_by_calc_type(calc_type)
|
|
data = list()
|
|
for seaweed in get_apply_seaweed(city, year_month):
|
|
query = {"city": seaweed["city"], "region": seaweed["region"], "name": seaweed["name"], "date": year_month,
|
|
"calc_model": {"$ne": "svr"}}
|
|
stats = list(table_name.find(query))
|
|
if len(stats) < 3:
|
|
continue
|
|
for stat in stats:
|
|
item = seaweed.copy()
|
|
item["room"] = stat["room"]
|
|
item["rent_price"] = stat["rent_price"]
|
|
data.append(item)
|
|
print(item)
|
|
return pd.DataFrame(data)
|
|
|
|
|
|
def has_rent_price(seaweed, room, year_month, calc_type):
|
|
table_name = get_rent_table_by_calc_type(calc_type)
|
|
query = {"city": seaweed["city"], "region": seaweed["region"], "name": seaweed["name"],
|
|
"date": year_month, "room": room}
|
|
return table_name.count(query) > 0
|
|
|
|
|
|
def train(city, year_month, calc_type):
|
|
"""
|
|
|
|
:param city:
|
|
:param year_month:
|
|
:param calc_type:计算类型 ht:户型租金 sq: 平米租金
|
|
:return:
|
|
"""
|
|
train_data = load_train_data(city, year_month, calc_type)
|
|
x = train_data[["lat", "lng", "build_year", "room", "avg_price"]]
|
|
y = train_data[["rent_price"]]
|
|
x = x.astype(np.float64)
|
|
y = y.astype(np.float64)
|
|
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.3)
|
|
c_range = np.logspace(-5, 15, 11, base=2)
|
|
gamma_range = np.logspace(-9, 3, 13, base=2)
|
|
param_grid = [{'svr__kernel': ['rbf', 'linear'], 'svr__C': c_range, 'svr__gamma': gamma_range}]
|
|
svr = svm.SVR()
|
|
pipeline = Pipeline([('standard', preprocessing.StandardScaler()),
|
|
('minMax', preprocessing.MinMaxScaler()),
|
|
('svr', svr)])
|
|
grid = GridSearchCV(pipeline, param_grid, cv=3, n_jobs=10)
|
|
clf = grid.fit(x_train, y_train.values.ravel())
|
|
score = grid.score(x_test, y_test)
|
|
print('精度为%s' % score)
|
|
predict_seaweed = list()
|
|
for seaweed in get_apply_seaweed(city, year_month):
|
|
for room in range(1, 4):
|
|
if not has_rent_price(seaweed, room, year_month, calc_type):
|
|
item = seaweed.copy()
|
|
item["room"] = room
|
|
predict_seaweed.append(item)
|
|
predict_df = pd.DataFrame(predict_seaweed)
|
|
result = clf.predict(predict_df[["lat", "lng", "build_year", "room", "avg_price"]])
|
|
result = pd.DataFrame([[int(val)] for val in result])
|
|
result.columns = ["rent"]
|
|
predict_df = predict_df.join(result)
|
|
predict_df = predict_df[["city", "region", "name", "room", "rent"]]
|
|
table_name = get_rent_table_by_calc_type(calc_type)
|
|
for index, row in predict_df.iterrows():
|
|
if row["rent"] <= 0:
|
|
continue
|
|
query = {
|
|
"date": year_month,
|
|
"city": row["city"],
|
|
"region": row["region"],
|
|
"name": row["name"],
|
|
"room": int(row["room"])
|
|
}
|
|
data = dict()
|
|
data["m_date"] = datetime.datetime.utcnow()
|
|
data["rent_price"] = row["rent"]
|
|
data["calc_model"] = "svr"
|
|
table_name.update_one(query, {"$set": data}, True)
|
|
print(row)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
start_date = common.parse_date("2019-05-01")
|
|
now_date = common.localize(datetime.datetime.utcnow())
|
|
while start_date < now_date:
|
|
year_month = int(common.format_date(start_date - relativedelta(months=1), '%Y%m'))
|
|
for city in ["北京"]:
|
|
train("北京", year_month, "ht")
|
|
train("北京", year_month, "sq")
|
|
start_date += relativedelta(months=1)
|
|
|