git commit -m "Initial commit"
This commit is contained in:
@@ -0,0 +1,71 @@
|
||||
import pandas as pd
|
||||
from common import configs as cfg
|
||||
|
||||
import common
|
||||
|
||||
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
|
||||
HOUSE_RENT = common.get_mongodb_collection("mongodb://192.168.0.105:27018/", "vip_site", "esf_houseRent")
|
||||
DISTRICT_STAT_RENT = common.get_mongodb_collection(cfg.MONGO_URI, "fangjia", "district_stat_rent")
|
||||
DISTRICT_RENT = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
|
||||
|
||||
|
||||
def find_room_rent(city, region, name, room):
|
||||
stat = DISTRICT_STAT_RENT.find_one({"city": city, "region": region, "district_name": name, "house_type": room,
|
||||
"end_month_date": common.parse_date("2019-03-31")})
|
||||
if stat:
|
||||
return stat["avg_price"]
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def find_sq_rent(city, region, name, room):
|
||||
stat = DISTRICT_RENT.find_one({"city": city, "region": region, "name": name, "room": room, "date": 201902})
|
||||
if stat:
|
||||
return stat["rent_price"]
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def find_rent_house(city, region, alias, room):
|
||||
fields_filter = {"address": 0, "_id": 0, "block": 0, "c_date": 0, "title": 0, "m_date": 0, "rent_type": 0}
|
||||
houses = pd.DataFrame(list(HOUSE_RENT.find({"city": city, "region": region, "rent_type": "整租",
|
||||
"district_name": {"$in": alias}, "room": room, "publish_date": {"$gt": common.parse_date("2019-01-01")}}, fields_filter)))
|
||||
if len(houses) == 0:
|
||||
return houses
|
||||
houses["avg_price"] = houses["total_price"] / houses["area"]
|
||||
houses["avg_price"] = houses["avg_price"].map(int)
|
||||
houses.drop_duplicates(keep='first', inplace=True)
|
||||
houses.sort_values(by="area", inplace=True)
|
||||
return houses
|
||||
|
||||
|
||||
regions = ["朝阳", "海淀", "丰台", "西城", "通州", "石景山", "东城"]
|
||||
|
||||
|
||||
def process():
|
||||
all_house = pd.DataFrame()
|
||||
for region in regions:
|
||||
for seaweed in common.get_seaweed("北京", region):
|
||||
names = [seaweed["name"]]
|
||||
if "alias" in seaweed and seaweed["alias"] is not None:
|
||||
names += seaweed["alias"]
|
||||
for room in range(1, 3):
|
||||
houses = find_rent_house(seaweed["city"], seaweed["region"], names, room)
|
||||
if len(houses) > 1:
|
||||
all_house = all_house.append(houses)
|
||||
all_house["publish_date"] = all_house["publish_date"].apply(lambda x: common.format_date(x, '%Y%m%d'))
|
||||
for col in ["direction", "fitment", "floor_info", "hall", "height", "toilet", "avg_price", "publish_date"]:
|
||||
del all_house[col]
|
||||
excel_data = []
|
||||
for index, row in all_house.iterrows():
|
||||
row = dict(row)
|
||||
row["户型租金"] = find_room_rent(row["city"], row["region"], row["district_name"], row["room"])
|
||||
row["平米租金"] = find_sq_rent(row["city"], row["region"], row["district_name"], row["room"])
|
||||
print(row)
|
||||
excel_data.append(row)
|
||||
excel_data = pd.DataFrame(excel_data)
|
||||
excel_data = excel_data.dropna(how="any")
|
||||
excel_data.to_excel("D:\\建行户型房源数据.xlsx", index=False)
|
||||
|
||||
|
||||
process()
|
||||
@@ -0,0 +1,38 @@
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn import preprocessing
|
||||
from sklearn import svm
|
||||
from sklearn.model_selection import GridSearchCV, train_test_split
|
||||
from sklearn.pipeline import Pipeline
|
||||
import common
|
||||
from sklearn.preprocessing import PolynomialFeatures
|
||||
|
||||
|
||||
base_year = common.parse_date("1900-01-01")
|
||||
|
||||
|
||||
def years(now):
|
||||
year = common.parse_date(str(now), '%Y')
|
||||
return int((year - base_year).days/365)
|
||||
|
||||
|
||||
train_data = pd.read_csv(open('../data/小区租金样本.csv'))
|
||||
train_data["build_year"] = train_data["build_year"].apply(lambda param: years(param))
|
||||
x = train_data[["lat", "lng", "build_year", "room", "avg_price"]]
|
||||
y = train_data[["rent_price"]]
|
||||
x = x.astype(np.float64)
|
||||
y = y.astype(np.float64)
|
||||
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.2)
|
||||
c_range = np.logspace(-5, 15, 11, base=2)
|
||||
gamma_range = np.logspace(-9, 3, 13, base=2)
|
||||
param_grid = [{'svr__kernel': ['rbf', 'linear'], 'svr__C': c_range, 'svr__gamma': gamma_range}]
|
||||
svr = svm.SVR()
|
||||
pipeline = Pipeline([('poly', PolynomialFeatures(3)),
|
||||
('standard', preprocessing.StandardScaler()),
|
||||
('minMax', preprocessing.MinMaxScaler()),
|
||||
('svr', svr)])
|
||||
grid = GridSearchCV(pipeline, param_grid, cv=3, n_jobs=10)
|
||||
clf = grid.fit(x_train, y_train.values.ravel())
|
||||
test_score = grid.score(x_test, y_test)
|
||||
print('测试得分是%s' % test_score)
|
||||
print('训练得分是%s' % grid.score(x_train, y_train))
|
||||
@@ -0,0 +1,176 @@
|
||||
from sklearn import svm
|
||||
import re
|
||||
import pandas as pd
|
||||
import warnings
|
||||
import numpy as np
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from sklearn.model_selection import GridSearchCV, train_test_split
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn import preprocessing
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
curPath = os.path.abspath(os.path.dirname(__file__))
|
||||
rootPath = os.path.split(curPath)[0]
|
||||
sys.path.append(rootPath)
|
||||
import common
|
||||
from common import configs as cfg
|
||||
warnings.simplefilter("ignore")
|
||||
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
|
||||
district_room_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_room_rent")
|
||||
district_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
|
||||
district_sell_stat = common.get_mongodb_collection(cfg.MONGO_URI, "fangjia_stat", "district_sell_stat")
|
||||
|
||||
|
||||
def has_geo_info(seaweed):
|
||||
return "lat2" in seaweed and "lng2" in seaweed and seaweed["lng2"] > 0
|
||||
|
||||
|
||||
def get_build_year(seaweed):
|
||||
if "finish_building_date" not in seaweed or seaweed["finish_building_date"] is None:
|
||||
return None
|
||||
pattern = re.compile(r'\d{4}')
|
||||
match = pattern.match(str(seaweed["finish_building_date"]))
|
||||
if not match:
|
||||
return None
|
||||
return int(match.group())
|
||||
|
||||
|
||||
def is_common_community(seaweed):
|
||||
return seaweed["estate_type2"] == "普通住宅"
|
||||
|
||||
|
||||
def get_apply_seaweed(city, year_month):
|
||||
stats = list(district_sell_stat.find({"city": city, "date": year_month}))
|
||||
stat_dict = dict()
|
||||
for stat in stats:
|
||||
key = stat["city"] + "_" + stat["region"] + "_" + stat["district_name"]
|
||||
stat_dict[key] = stat["avg_price"]
|
||||
seaweed_list = list()
|
||||
for seaweed in common.get_seaweed(city):
|
||||
if not has_geo_info(seaweed):
|
||||
continue
|
||||
if not is_common_community(seaweed):
|
||||
continue
|
||||
build_year = get_build_year(seaweed)
|
||||
if not build_year:
|
||||
continue
|
||||
key = seaweed["city"] + "_" + seaweed["region"] + "_" + seaweed["name"]
|
||||
if key not in stat_dict:
|
||||
continue
|
||||
seaweed["build_year"] = build_year
|
||||
seaweed["lat"] = seaweed["lat2"] / 10000000000
|
||||
seaweed["lng"] = seaweed["lng2"] / 10000000000
|
||||
seaweed["avg_price"] = stat_dict[key]
|
||||
seaweed_list.append(seaweed)
|
||||
return seaweed_list
|
||||
|
||||
|
||||
def get_rent_table_by_calc_type(calc_type):
|
||||
if calc_type == "ht":
|
||||
return district_room_rent
|
||||
elif calc_type == "sq":
|
||||
return district_rent
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def load_train_data(city, year_month, calc_type):
|
||||
"""
|
||||
|
||||
:param city:
|
||||
:param year_month:
|
||||
:param calc_type:计算类型 ht:户型租金 sq: 平米租金
|
||||
:return:
|
||||
"""
|
||||
table_name = get_rent_table_by_calc_type(calc_type)
|
||||
data = list()
|
||||
for seaweed in get_apply_seaweed(city, year_month):
|
||||
query = {"city": seaweed["city"], "region": seaweed["region"], "name": seaweed["name"], "date": year_month,
|
||||
"calc_model": {"$ne": "svr"}}
|
||||
stats = list(table_name.find(query))
|
||||
if len(stats) < 3:
|
||||
continue
|
||||
for stat in stats:
|
||||
item = seaweed.copy()
|
||||
item["room"] = stat["room"]
|
||||
item["rent_price"] = stat["rent_price"]
|
||||
data.append(item)
|
||||
print(item)
|
||||
return pd.DataFrame(data)
|
||||
|
||||
|
||||
def has_rent_price(seaweed, room, year_month, calc_type):
|
||||
table_name = get_rent_table_by_calc_type(calc_type)
|
||||
query = {"city": seaweed["city"], "region": seaweed["region"], "name": seaweed["name"],
|
||||
"date": year_month, "room": room}
|
||||
return table_name.count(query) > 0
|
||||
|
||||
|
||||
def train(city, year_month, calc_type):
|
||||
"""
|
||||
|
||||
:param city:
|
||||
:param year_month:
|
||||
:param calc_type:计算类型 ht:户型租金 sq: 平米租金
|
||||
:return:
|
||||
"""
|
||||
train_data = load_train_data(city, year_month, calc_type)
|
||||
x = train_data[["lat", "lng", "build_year", "room", "avg_price"]]
|
||||
y = train_data[["rent_price"]]
|
||||
x = x.astype(np.float64)
|
||||
y = y.astype(np.float64)
|
||||
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.3)
|
||||
c_range = np.logspace(-5, 15, 11, base=2)
|
||||
gamma_range = np.logspace(-9, 3, 13, base=2)
|
||||
param_grid = [{'svr__kernel': ['rbf', 'linear'], 'svr__C': c_range, 'svr__gamma': gamma_range}]
|
||||
svr = svm.SVR()
|
||||
pipeline = Pipeline([('standard', preprocessing.StandardScaler()),
|
||||
('minMax', preprocessing.MinMaxScaler()),
|
||||
('svr', svr)])
|
||||
grid = GridSearchCV(pipeline, param_grid, cv=3, n_jobs=10)
|
||||
clf = grid.fit(x_train, y_train.values.ravel())
|
||||
score = grid.score(x_test, y_test)
|
||||
print('精度为%s' % score)
|
||||
predict_seaweed = list()
|
||||
for seaweed in get_apply_seaweed(city, year_month):
|
||||
for room in range(1, 4):
|
||||
if not has_rent_price(seaweed, room, year_month, calc_type):
|
||||
item = seaweed.copy()
|
||||
item["room"] = room
|
||||
predict_seaweed.append(item)
|
||||
predict_df = pd.DataFrame(predict_seaweed)
|
||||
result = clf.predict(predict_df[["lat", "lng", "build_year", "room", "avg_price"]])
|
||||
result = pd.DataFrame([[int(val)] for val in result])
|
||||
result.columns = ["rent"]
|
||||
predict_df = predict_df.join(result)
|
||||
predict_df = predict_df[["city", "region", "name", "room", "rent"]]
|
||||
table_name = get_rent_table_by_calc_type(calc_type)
|
||||
for index, row in predict_df.iterrows():
|
||||
if row["rent"] <= 0:
|
||||
continue
|
||||
query = {
|
||||
"date": year_month,
|
||||
"city": row["city"],
|
||||
"region": row["region"],
|
||||
"name": row["name"],
|
||||
"room": int(row["room"])
|
||||
}
|
||||
data = dict()
|
||||
data["m_date"] = datetime.datetime.utcnow()
|
||||
data["rent_price"] = row["rent"]
|
||||
data["calc_model"] = "svr"
|
||||
table_name.update_one(query, {"$set": data}, True)
|
||||
print(row)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
start_date = common.parse_date("2019-05-01")
|
||||
now_date = common.localize(datetime.datetime.utcnow())
|
||||
while start_date < now_date:
|
||||
year_month = int(common.format_date(start_date - relativedelta(months=1), '%Y%m'))
|
||||
for city in ["北京"]:
|
||||
train("北京", year_month, "ht")
|
||||
train("北京", year_month, "sq")
|
||||
start_date += relativedelta(months=1)
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
import calendar
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
from math import isnan
|
||||
import pandas as pd
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from joblib import load
|
||||
curPath = os.path.abspath(os.path.dirname(__file__))
|
||||
rootPath = os.path.split(curPath)[0]
|
||||
sys.path.append(rootPath)
|
||||
import common
|
||||
from common import configs as cfg
|
||||
warnings.simplefilter("ignore")
|
||||
|
||||
|
||||
def k_means(series):
|
||||
return common.k_means(series)
|
||||
|
||||
|
||||
clf = load("C:\\Users\\wenyan\\PycharmProjects\\Sklearn\\data\\house_type_classify_model.joblib")
|
||||
|
||||
|
||||
def house_type_filter(houses):
|
||||
"""
|
||||
通过模型过滤掉合租房源
|
||||
:param houses:
|
||||
:return:
|
||||
"""
|
||||
if len(houses) == 0:
|
||||
return houses
|
||||
X = houses[["room", "area"]]
|
||||
houses["rent_type_predict"] = clf.predict(X)
|
||||
houses["prob1"] = clf.predict_proba(X)[:, 1]
|
||||
houses = houses[(houses.rent_type_predict == 1) & (houses.prob1 > 0.8)]
|
||||
del houses["rent_type_predict"]
|
||||
del houses["prob1"]
|
||||
return houses
|
||||
|
||||
|
||||
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
|
||||
region_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "region_rent")
|
||||
district_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
|
||||
|
||||
|
||||
# 删除无用列
|
||||
def remove_useless_columns(df):
|
||||
for col in ["fitment", "direction", "root"]:
|
||||
if col in df.columns:
|
||||
del df[col]
|
||||
|
||||
|
||||
def weighting(df):
|
||||
if 'root' not in df.columns.values.tolist():
|
||||
return df
|
||||
root_weight = {
|
||||
"zhuge.com": 3,
|
||||
"ke.com": 8,
|
||||
"lianjia.com": 8
|
||||
}
|
||||
weighting_data = list()
|
||||
for index, row in df.iterrows():
|
||||
row = dict(row)
|
||||
weighting_data.append(row)
|
||||
if row['root'] in root_weight:
|
||||
for i in range(1, root_weight[row['root']]):
|
||||
weighting_data.append(row)
|
||||
print("org:{} dest:{}".format(len(df), len(weighting_data)))
|
||||
return pd.DataFrame(weighting_data)
|
||||
|
||||
|
||||
def get_start_date_and_end_date(year_month):
|
||||
date = common.parse_date(str(year_month), '%Y%m')
|
||||
year = date.year
|
||||
month = date.month
|
||||
first_day = datetime.datetime(year=year, month=month, day=1)
|
||||
month_range = calendar.monthrange(year, month)
|
||||
end_day = datetime.datetime(year=year, month=month, day=month_range[1])
|
||||
return first_day, end_day
|
||||
|
||||
|
||||
def calc_city_rent(_city, year_month):
|
||||
start_date, end_date = get_start_date_and_end_date(year_month)
|
||||
stats = common.get_least_region_stat(_city, None, start_date=start_date, end_date=end_date)
|
||||
start_date = int(common.format_date(start_date, "%Y%m%d"))
|
||||
end_date = int(common.format_date(end_date, "%Y%m%d"))
|
||||
houses = common.load_house_data(city=_city, region=None, name=None, cat="rent", rent_type="整租",
|
||||
start_date=start_date, end_date=end_date)
|
||||
houses = house_type_filter(houses)
|
||||
houses = weighting(houses)
|
||||
remove_useless_columns(houses)
|
||||
houses.dropna(axis=0, how='any', inplace=True)
|
||||
houses = houses[houses.avg_price > 0]
|
||||
houses.sort_values(by='p_date', ascending=False, inplace=True)
|
||||
agg = houses.groupby(by="room")[['avg_price']].agg(['mean', 'median', 'count', k_means, 'min', 'max'])
|
||||
agg = agg.applymap(int)
|
||||
agg.columns = ["mean", "median", "count", "k_means", "min", "max"]
|
||||
for index, row in agg.iterrows():
|
||||
query = {
|
||||
"date": year_month,
|
||||
"cat": "city",
|
||||
"city": _city,
|
||||
"room": int(index)
|
||||
}
|
||||
data = dict(row)
|
||||
for k, v in data.items():
|
||||
data[k] = int(v)
|
||||
data["m_date"] = datetime.datetime.utcnow()
|
||||
if data["count"] > cfg.CITY_RENT_MIN_HOUSE_COUNT:
|
||||
data["rent_price"] = data["k_means"]
|
||||
if len(stats) > 0:
|
||||
data["avg_price"] = stats[0]["avg_price"]
|
||||
data["price_rent_ratio"] = int(data["avg_price"] / data["k_means"])
|
||||
region_rent.update_one(query, {"$set": data}, True)
|
||||
|
||||
|
||||
def calc_region_rent(_city, _region, year_month):
|
||||
start_date, end_date = get_start_date_and_end_date(year_month)
|
||||
stats = common.get_least_region_stat(city=_city, region=_region, start_date=start_date, end_date=end_date, limit=1)
|
||||
start_date = int(common.format_date(start_date, "%Y%m%d"))
|
||||
end_date = int(common.format_date(end_date, "%Y%m%d"))
|
||||
houses = common.load_house_data(city=_city, region=_region, name=None, cat="rent", rent_type="整租",
|
||||
start_date=start_date, end_date=end_date)
|
||||
houses = house_type_filter(houses)
|
||||
houses = weighting(houses)
|
||||
remove_useless_columns(houses)
|
||||
houses.dropna(axis=0, how='any', inplace=True)
|
||||
if len(houses) == 0:
|
||||
return
|
||||
houses.sort_values(by='p_date', ascending=False, inplace=True)
|
||||
agg = houses.groupby(by="room")[['avg_price']].agg(['mean', 'median', 'count', k_means])
|
||||
agg = agg.applymap(int)
|
||||
agg.columns = ["mean", "median", "count", "k_means"]
|
||||
city_rent = dict()
|
||||
for item in list(region_rent.find({"cat": "city", "city": _city, "date": year_month})):
|
||||
city_rent[item["room"]] = item["price_rent_ratio"]
|
||||
for index, row in agg.iterrows():
|
||||
query = {
|
||||
"date": year_month,
|
||||
"cat": "region",
|
||||
"region": _region,
|
||||
"city": _city,
|
||||
"room": int(index)
|
||||
}
|
||||
data = dict(row)
|
||||
for k, v in data.items():
|
||||
data[k] = int(v)
|
||||
data["m_date"] = datetime.datetime.utcnow()
|
||||
if data["count"] > cfg.REGION_RENT_MIN_HOUSE_COUNT:
|
||||
if len(stats) == 0:
|
||||
data["rent_price"] = data["k_means"]
|
||||
else:
|
||||
region_price = stats[0]["avg_price"]
|
||||
data["avg_price"] = region_price
|
||||
data["rent_price"] = data["k_means"]
|
||||
data["price_rent_ratio"] = int(region_price / data["k_means"])
|
||||
region_rent.update_one(query, {"$set": data}, True)
|
||||
|
||||
|
||||
def filter_by_price_rent(houses, region_price):
|
||||
if region_price is None or region_price == 0:
|
||||
return houses
|
||||
filtered_houses = list()
|
||||
for index, row in houses.iterrows():
|
||||
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3:
|
||||
continue
|
||||
if isnan(row["avg_price"]):
|
||||
row["avg_price"] = int(row["total_price"] / row["area"])
|
||||
house_price_rent_ratio = int(region_price / row["avg_price"])
|
||||
if 100 < house_price_rent_ratio < 1500:
|
||||
filtered_houses.append(row)
|
||||
return pd.DataFrame(filtered_houses)
|
||||
|
||||
|
||||
def filter_by_price_to_rent(houses, seaweed, year_month):
|
||||
stats = common.get_least_district_stat(seaweed["city"], seaweed["region"], seaweed["name"], seaweed["estate_type2"]
|
||||
, limit=1)
|
||||
if len(stats) == 0:
|
||||
return houses
|
||||
seaweed_avg_price = stats[0]["avg_price"]
|
||||
house_type_price_rent = dict()
|
||||
for stat in list(region_rent.find({"city": seaweed["city"], "region": seaweed["region"], "date": year_month})):
|
||||
if stat["cat"] == "city":
|
||||
house_type_price_rent["city_{}".format(stat["room"])] = stat["price_rent_ratio"]
|
||||
else:
|
||||
house_type_price_rent["region_{}".format(stat["room"])] = stat["price_rent_ratio"]
|
||||
|
||||
filtered_houses = list()
|
||||
for index, row in houses.iterrows():
|
||||
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3:
|
||||
continue
|
||||
if isnan(row["avg_price"]):
|
||||
row["avg_price"] = int(row["total_price"] / row["area"])
|
||||
house_price_rent_ratio = int(seaweed_avg_price / row["avg_price"])
|
||||
if "region_{}".format(row["room"]) in house_type_price_rent:
|
||||
key = "region_{}".format(row["room"])
|
||||
if house_type_price_rent[key] * 0.5 < house_price_rent_ratio < house_type_price_rent[key] * 2:
|
||||
filtered_houses.append(row)
|
||||
else:
|
||||
print("过滤:" + str(row))
|
||||
elif "city_{}".format(row["room"]) in house_type_price_rent:
|
||||
key = "city_{}".format(row["room"])
|
||||
if house_type_price_rent[key] * 0.25 < house_price_rent_ratio < house_type_price_rent[key] * 4:
|
||||
filtered_houses.append(row)
|
||||
else:
|
||||
print("过滤:" + str(row))
|
||||
else:
|
||||
filtered_houses.append(row)
|
||||
return pd.DataFrame(filtered_houses)
|
||||
|
||||
|
||||
def main():
|
||||
start_date = common.parse_date("2019-05-01")
|
||||
now_date = common.localize(datetime.datetime.utcnow())
|
||||
while start_date < now_date:
|
||||
year_month = int(common.format_date(start_date - relativedelta(months=1), '%Y%m'))
|
||||
for city in ["北京"]:
|
||||
print("{}:{}\t开始计算".format(year_month, city))
|
||||
calc_city_rent(city, year_month)
|
||||
for region in common.get_regions(city):
|
||||
print("{}:{} {}\t开始计算:".format(year_month, city, region))
|
||||
calc_region_rent(city, region, year_month)
|
||||
start_date += relativedelta(months=1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,261 @@
|
||||
import calendar
|
||||
import datetime
|
||||
import multiprocessing
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
from math import isnan
|
||||
from joblib import load
|
||||
import pandas as pd
|
||||
from dateutil.relativedelta import relativedelta
|
||||
|
||||
curPath = os.path.abspath(os.path.dirname(__file__))
|
||||
rootPath = os.path.split(curPath)[0]
|
||||
sys.path.append(rootPath)
|
||||
import common
|
||||
warnings.simplefilter("ignore")
|
||||
|
||||
|
||||
def k_means(series):
|
||||
return common.k_means(series)
|
||||
|
||||
|
||||
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
|
||||
region_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "region_rent")
|
||||
district_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
|
||||
district_room_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_room_rent")
|
||||
|
||||
|
||||
# 删除无用列
|
||||
def remove_useless_columns(df):
|
||||
for col in ["fitment", "direction", "root"]:
|
||||
if col in df.columns:
|
||||
del df[col]
|
||||
|
||||
|
||||
def weighting(df):
|
||||
root_weight = {
|
||||
"zhuge.com": 2,
|
||||
"5i5j.com": 3,
|
||||
"ke.com": 8,
|
||||
"lianjia.com": 8
|
||||
}
|
||||
weighting_data = list()
|
||||
for index, row in df.iterrows():
|
||||
row = dict(row)
|
||||
weighting_data.append(row)
|
||||
if 'root' in row and row['root'] in root_weight:
|
||||
for i in range(1, root_weight[row['root']]):
|
||||
weighting_data.append(row)
|
||||
return pd.DataFrame(weighting_data)
|
||||
|
||||
|
||||
def get_region_markup(_city, _region, _start, _end, _room):
|
||||
_room = 2
|
||||
start_stat = region_rent.find_one(
|
||||
{"cat": "region", "city": _city, "region": _region, "date": _start, "room": _room})
|
||||
end_stat = region_rent.find_one({"cat": "region", "city": _city, "region": _region, "date": _end, "room": _room})
|
||||
if not start_stat or not end_stat:
|
||||
return 0
|
||||
return (end_stat["rent_price"] - start_stat["rent_price"]) / start_stat["rent_price"]
|
||||
|
||||
|
||||
def correct_house(_houses, _year_month):
|
||||
calc_year_month = common.parse_date(str(_year_month), '%Y%m')
|
||||
for index, house in _houses.iterrows():
|
||||
p_date = common.parse_date(str(house["p_date"]), '%Y%m%d')
|
||||
days = (calc_year_month - p_date).days
|
||||
if days > 180:
|
||||
markup = get_region_markup(house["city"], house["region"], int(house["p_date"] / 100), _year_month, house["room"])
|
||||
_houses.at[index, "avg_price"] = house["avg_price"] * (1 + markup)
|
||||
_houses.at[index, "total_price"] = house["total_price"] * (1 + markup)
|
||||
return _houses
|
||||
|
||||
|
||||
def get_start_date_and_end_date(year_month):
|
||||
date = common.parse_date(str(year_month), '%Y%m')
|
||||
year = date.year
|
||||
month = date.month
|
||||
first_day = datetime.datetime(year=year, month=month, day=1)
|
||||
month_range = calendar.monthrange(year, month)
|
||||
end_day = datetime.datetime(year=year, month=month, day=month_range[1])
|
||||
return first_day, end_day
|
||||
|
||||
|
||||
def filter_by_price_rent(houses, region_price):
|
||||
if region_rent is None:
|
||||
return houses
|
||||
filtered_houses = list()
|
||||
for index, row in houses.iterrows():
|
||||
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3:
|
||||
continue
|
||||
if isnan(row["avg_price"]):
|
||||
row["avg_price"] = int(row["total_price"] / row["area"])
|
||||
house_price_rent_ratio = int(region_price / row["avg_price"])
|
||||
if 100 < house_price_rent_ratio < 1000:
|
||||
filtered_houses.append(row)
|
||||
return pd.DataFrame(filtered_houses)
|
||||
|
||||
|
||||
def filter_by_price_to_rent(houses, seaweed, year_month):
|
||||
stats = common.get_least_district_stat(seaweed["city"], seaweed["region"], seaweed["name"], seaweed["estate_type2"]
|
||||
, limit=1)
|
||||
if len(stats) == 0:
|
||||
return houses
|
||||
seaweed_avg_price = stats[0]["avg_price"]
|
||||
house_type_price_rent = dict()
|
||||
for stat in list(region_rent.find({"city": seaweed["city"], "region": seaweed["region"], "date": year_month})):
|
||||
if stat["cat"] == "city":
|
||||
house_type_price_rent["city_{}".format(stat["room"])] = stat["price_rent_ratio"]
|
||||
else:
|
||||
house_type_price_rent["region_{}".format(stat["room"])] = stat["price_rent_ratio"]
|
||||
if len(house_type_price_rent) > 0:
|
||||
house_type_price_rent["region"] = int(pd.Series(list(house_type_price_rent.values())).mean())
|
||||
filtered_houses = list()
|
||||
houses[["room", "total_price"]].to_csv("../data/房源.csv", index=False)
|
||||
for index, row in houses.iterrows():
|
||||
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3 or row["area"] == 0:
|
||||
continue
|
||||
if "avg_price" not in row or isnan(row["avg_price"]):
|
||||
row["avg_price"] = int(row["total_price"] / row["area"])
|
||||
if row["avg_price"] < 10:
|
||||
continue
|
||||
house_price_rent_ratio = int(seaweed_avg_price / row["avg_price"])
|
||||
if "region_{}".format(row["room"]) in house_type_price_rent:
|
||||
key = "region_{}".format(row["room"])
|
||||
if house_type_price_rent[key] * 0.5 < house_price_rent_ratio < house_type_price_rent[key] * 2:
|
||||
filtered_houses.append(row)
|
||||
elif "city_{}".format(row["room"]) in house_type_price_rent:
|
||||
key = "city_{}".format(row["room"])
|
||||
if house_type_price_rent[key] * 0.25 < house_price_rent_ratio < house_type_price_rent[key] * 4:
|
||||
filtered_houses.append(row)
|
||||
elif "region" in house_type_price_rent:
|
||||
key = "region"
|
||||
if house_type_price_rent[key] * 0.4 < house_price_rent_ratio < house_type_price_rent[key] * 2:
|
||||
filtered_houses.append(row)
|
||||
else:
|
||||
filtered_houses.append(row)
|
||||
return pd.DataFrame(filtered_houses)
|
||||
|
||||
|
||||
clf = load("C:\\Users\\wenyan\\PycharmProjects\\Sklearn\\data\\house_type_classify_model.joblib")
|
||||
|
||||
|
||||
def house_type_filter(houses):
|
||||
"""
|
||||
通过模型过滤掉合租房源
|
||||
:param houses:
|
||||
:return:
|
||||
"""
|
||||
X = houses[["room", "area"]]
|
||||
houses["rent_type_predict"] = clf.predict(X)
|
||||
houses["prob1"] = clf.predict_proba(X)[:, 1]
|
||||
houses = houses[(houses.rent_type_predict == 1) & (0.8 < houses.prob1) & (houses.prob1 < 0.99999)]
|
||||
del houses["rent_type_predict"]
|
||||
del houses["prob1"]
|
||||
return houses
|
||||
|
||||
|
||||
def drop_duplicates(houses):
|
||||
houses["p_month"] = houses["p_date"].apply(lambda x: int(x/100))
|
||||
houses = houses.drop_duplicates(subset=["area", "avg_price", "p_month", "room", "root", "total_price"])
|
||||
del houses["p_month"]
|
||||
return houses
|
||||
|
||||
|
||||
def calc_seaweed_rent(seaweed, year_month):
|
||||
print("开始计算小区租金{}: {} {} {}".format(year_month, seaweed["city"], seaweed["region"], seaweed["name"]))
|
||||
start_date, end_date = get_start_date_and_end_date(year_month)
|
||||
stats = common.get_least_district_stat(seaweed["city"], seaweed["region"], seaweed["name"], seaweed["estate_type2"]
|
||||
, start_date=start_date, end_date=end_date, limit=1)
|
||||
start_date = end_date - relativedelta(years=1)
|
||||
first_date = end_date - relativedelta(months=1)
|
||||
start_date = int(common.format_date(start_date, "%Y%m%d"))
|
||||
first_date = int(common.format_date(first_date, "%Y%m%d"))
|
||||
end_date = int(common.format_date(end_date, "%Y%m%d"))
|
||||
alias = None
|
||||
if alias in seaweed:
|
||||
alias = seaweed["alias"]
|
||||
houses = pd.DataFrame()
|
||||
for house_type in range(1, 4):
|
||||
_houses = common.load_house_data(city=seaweed["city"], region=seaweed["region"], name=seaweed["name"],
|
||||
cat="rent", room=house_type, rent_type="整租", end_date=end_date,
|
||||
start_date=first_date, alias=alias, limit=100)
|
||||
if len(_houses) < 5:
|
||||
_houses = common.load_house_data(city=seaweed["city"], region=seaweed["region"], name=seaweed["name"],
|
||||
cat="rent", room=house_type,
|
||||
rent_type="整租", end_date=end_date, start_date=start_date, alias=alias,
|
||||
limit=100)
|
||||
houses = houses.append(_houses, ignore_index=True)
|
||||
|
||||
if len(houses) == 0:
|
||||
return
|
||||
houses = drop_duplicates(houses)
|
||||
houses = house_type_filter(houses)
|
||||
houses = filter_by_price_to_rent(houses, seaweed, year_month)
|
||||
houses = correct_house(houses, year_month)
|
||||
houses = weighting(houses)
|
||||
remove_useless_columns(houses)
|
||||
houses.dropna(axis=0, how='any', inplace=True)
|
||||
if len(houses) < 3:
|
||||
return
|
||||
if "room" not in houses.columns:
|
||||
return
|
||||
houses = houses[houses.avg_price > 0]
|
||||
houses.sort_values(by='p_date', ascending=False, inplace=True)
|
||||
agg = houses.groupby(by="room")[['avg_price']].agg(['mean', 'median', 'count', k_means])
|
||||
total_price_agg = houses.groupby(by="room")[['total_price']].agg(['mean', 'median', 'count', k_means])
|
||||
agg.columns = ["mean", "median", "count", "k_means"]
|
||||
total_price_agg.columns = ["mean", "median", "count", "k_means"]
|
||||
for index, row in agg.iterrows():
|
||||
query = {
|
||||
"date": year_month,
|
||||
"city": seaweed["city"],
|
||||
"region": seaweed["region"],
|
||||
"name": seaweed["name"],
|
||||
"room": int(index)
|
||||
}
|
||||
data = dict(row)
|
||||
for k, v in data.items():
|
||||
data[k] = int(v)
|
||||
data["m_date"] = datetime.datetime.utcnow()
|
||||
data["rent_price"] = data["k_means"]
|
||||
if len(stats) != 0:
|
||||
data["avg_price"] = stats[0]["avg_price"]
|
||||
data["price_rent_ratio"] = int(data["avg_price"] / data["rent_price"])
|
||||
district_rent.update_one(query, {"$set": data}, True)
|
||||
|
||||
for index, row in total_price_agg.iterrows():
|
||||
query = {
|
||||
"date": year_month,
|
||||
"city": seaweed["city"],
|
||||
"region": seaweed["region"],
|
||||
"name": seaweed["name"],
|
||||
"room": int(index)
|
||||
}
|
||||
data = dict(row)
|
||||
for k, v in data.items():
|
||||
data[k] = int(v)
|
||||
data["m_date"] = datetime.datetime.utcnow()
|
||||
data["rent_price"] = data["k_means"]
|
||||
district_room_rent.update_one(query, {"$set": data}, True)
|
||||
|
||||
|
||||
def main():
|
||||
pool = multiprocessing.Pool(processes=30)
|
||||
start_date = common.parse_date("2019-05-01")
|
||||
now_date = common.localize(datetime.datetime.utcnow())
|
||||
while start_date < now_date:
|
||||
year_month = int(common.format_date(start_date - relativedelta(months=1), '%Y%m'))
|
||||
for city in ["北京"]:
|
||||
for community in common.get_seaweed(city):
|
||||
pool.apply_async(calc_seaweed_rent, (community, year_month,))
|
||||
start_date += relativedelta(months=1)
|
||||
pool.close()
|
||||
pool.join()
|
||||
print("done")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
calc_seaweed_rent(common.get_seaweed("北京", "西城", "富国里")[0], 201904)
|
||||
# main()
|
||||
@@ -0,0 +1,33 @@
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from common import configs
|
||||
import common
|
||||
plt.rcParams['savefig.dpi'] = 400
|
||||
plt.rcParams['figure.dpi'] = 400
|
||||
plt.rcParams['figure.figsize'] = (12, 4)
|
||||
plt.rcParams['font.sans-serif'] = ['SimHei']
|
||||
region_rent = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia_rent", "region_rent")
|
||||
price_rent = dict()
|
||||
for region in common.get_regions("北京"):
|
||||
df = pd.DataFrame(list(region_rent.find({"cat": "region", "region": region}, {"price_rent_ratio": 1})))
|
||||
if len(df) > 0:
|
||||
price_rent[region] = int(df["price_rent_ratio"].mean())
|
||||
fig, ax = plt.subplots()
|
||||
x = []
|
||||
y = []
|
||||
z = []
|
||||
avg = np.array(list(price_rent.values())).mean()
|
||||
for k, v in price_rent.items():
|
||||
x.append(k)
|
||||
y.append(v)
|
||||
z.append(avg)
|
||||
ax.bar(x, y, width=0.5)
|
||||
ax.plot(x, y, label='区域售租比')
|
||||
ax.plot(x, z, label='平均线:' + str(avg))
|
||||
ax.set(xlabel='区域', ylabel='租售比', title='区域租售比分布图')
|
||||
ax.grid()
|
||||
plt.legend()
|
||||
plt.savefig("D:\\建设银行\\租售比2000以内\\各区域租售比.png")
|
||||
for item in price_rent.items():
|
||||
print(str(item[0]) + "\t" + str(item[1]))
|
||||
@@ -0,0 +1,12 @@
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn import linear_model
|
||||
df = pd.read_csv(open("C:\\Users\\wenyan\\PycharmProjects\\Sklearn\\data\\房源.csv"))
|
||||
fig, ax = plt.subplots()
|
||||
ax.scatter(df["room"], df["total_price"])
|
||||
|
||||
lr = linear_model.LinearRegression(normalize=True)
|
||||
X = df["room"].values.reshape(len(df["room"]), 1)
|
||||
Y = df["total_price"].values.ravel()
|
||||
lr.fit(X, Y)
|
||||
ax.plot([1, 2, 3], lr.predict([[1], [2], [3]]), 'r')
|
||||
@@ -0,0 +1,17 @@
|
||||
import matplotlib.pyplot as plt
|
||||
import pandas as pd
|
||||
import common
|
||||
|
||||
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
|
||||
HOUSE_RENT = common.get_mongodb_collection("mongodb://192.168.0.105:27018/", "vip_site", "esf_houseRent")
|
||||
DISTRICT_RENT = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_room_rent")
|
||||
stats = list(DISTRICT_RENT.find({"city": "北京", "region": "朝阳", "name": "甜水园"}))
|
||||
df = pd.DataFrame(stats)
|
||||
df.dropna()
|
||||
df = df[["city", "region", "name", "date", "room", "rent_price"]]
|
||||
df["date"] = df["date"].apply(lambda x: common.parse_date(str(x), '%Y%m'))
|
||||
room1_df = df[df["room"] == 2]
|
||||
room1_df = room1_df.sort_values(by="date")
|
||||
fig, ax = plt.subplots()
|
||||
ax.plot(room1_df["date"], room1_df["rent_price"], 'h-.')
|
||||
plt.show()
|
||||
@@ -0,0 +1,36 @@
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from common import configs
|
||||
import common
|
||||
|
||||
# region_rent = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia_rent", "region_rent")
|
||||
# price_rent = dict()
|
||||
# for region in common.get_regions("北京"):
|
||||
# df = pd.DataFrame(list(region_rent.find({"cat": "region", "region": region}, {"price_rent_ratio": 1})))
|
||||
# if len(df) > 0:
|
||||
# price_rent[region] = int(df["price_rent_ratio"].mean())
|
||||
# plt.rcParams['font.sans-serif'] = ['SimHei']
|
||||
# district_rent = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia_rent", "district_rent")
|
||||
# plt.rcParams['savefig.dpi'] = 800
|
||||
# plt.rcParams['figure.dpi'] = 800
|
||||
# plt.rcParams['figure.figsize'] = (6, 4)
|
||||
# for region in common.get_regions("北京"):
|
||||
# df = pd.DataFrame(list(district_rent.find({"region": region}, {"price_rent_ratio": 1})))
|
||||
# ratio = np.array(df["price_rent_ratio"])
|
||||
# result = pd.value_counts(ratio)
|
||||
# result = result.sort_index()
|
||||
# fig, ax = plt.subplots()
|
||||
# ax.plot(result.index, result.values)
|
||||
# if region in price_rent:
|
||||
# y = np.arange(int(max(result.values) * 1.2))
|
||||
# x = np.zeros(len(y)) + price_rent[region]
|
||||
# ax.plot(x, y, label="区域售租比")
|
||||
# ax.plot(x * 2, y, '--', label="2倍区域售租比")
|
||||
# ax.plot(x * 0.4, y, '-', label="0.4倍区域售租比")
|
||||
# ax.set(xlabel='售租比', ylabel='小区量', title=region + '区小区售租比分布图')
|
||||
# ax.grid()
|
||||
# plt.legend()
|
||||
# plt.savefig("D:\\建设银行\\租售比\\" + region + "租售比分布分布图.png")
|
||||
numbers = np.array([12000, 12000, 14900, 20000])
|
||||
print(common.k_means(numbers))
|
||||
Reference in New Issue
Block a user