git commit -m "Initial commit"

This commit is contained in:
2019-05-07 11:33:15 +08:00
commit 51b2b6baa7
30 changed files with 13620 additions and 0 deletions
View File
+71
View File
@@ -0,0 +1,71 @@
import pandas as pd
from common import configs as cfg
import common
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
HOUSE_RENT = common.get_mongodb_collection("mongodb://192.168.0.105:27018/", "vip_site", "esf_houseRent")
DISTRICT_STAT_RENT = common.get_mongodb_collection(cfg.MONGO_URI, "fangjia", "district_stat_rent")
DISTRICT_RENT = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
def find_room_rent(city, region, name, room):
stat = DISTRICT_STAT_RENT.find_one({"city": city, "region": region, "district_name": name, "house_type": room,
"end_month_date": common.parse_date("2019-03-31")})
if stat:
return stat["avg_price"]
else:
return None
def find_sq_rent(city, region, name, room):
stat = DISTRICT_RENT.find_one({"city": city, "region": region, "name": name, "room": room, "date": 201902})
if stat:
return stat["rent_price"]
else:
return None
def find_rent_house(city, region, alias, room):
fields_filter = {"address": 0, "_id": 0, "block": 0, "c_date": 0, "title": 0, "m_date": 0, "rent_type": 0}
houses = pd.DataFrame(list(HOUSE_RENT.find({"city": city, "region": region, "rent_type": "整租",
"district_name": {"$in": alias}, "room": room, "publish_date": {"$gt": common.parse_date("2019-01-01")}}, fields_filter)))
if len(houses) == 0:
return houses
houses["avg_price"] = houses["total_price"] / houses["area"]
houses["avg_price"] = houses["avg_price"].map(int)
houses.drop_duplicates(keep='first', inplace=True)
houses.sort_values(by="area", inplace=True)
return houses
regions = ["朝阳", "海淀", "丰台", "西城", "通州", "石景山", "东城"]
def process():
all_house = pd.DataFrame()
for region in regions:
for seaweed in common.get_seaweed("北京", region):
names = [seaweed["name"]]
if "alias" in seaweed and seaweed["alias"] is not None:
names += seaweed["alias"]
for room in range(1, 3):
houses = find_rent_house(seaweed["city"], seaweed["region"], names, room)
if len(houses) > 1:
all_house = all_house.append(houses)
all_house["publish_date"] = all_house["publish_date"].apply(lambda x: common.format_date(x, '%Y%m%d'))
for col in ["direction", "fitment", "floor_info", "hall", "height", "toilet", "avg_price", "publish_date"]:
del all_house[col]
excel_data = []
for index, row in all_house.iterrows():
row = dict(row)
row["户型租金"] = find_room_rent(row["city"], row["region"], row["district_name"], row["room"])
row["平米租金"] = find_sq_rent(row["city"], row["region"], row["district_name"], row["room"])
print(row)
excel_data.append(row)
excel_data = pd.DataFrame(excel_data)
excel_data = excel_data.dropna(how="any")
excel_data.to_excel("D:\\建行户型房源数据.xlsx", index=False)
process()
+38
View File
@@ -0,0 +1,38 @@
import numpy as np
import pandas as pd
from sklearn import preprocessing
from sklearn import svm
from sklearn.model_selection import GridSearchCV, train_test_split
from sklearn.pipeline import Pipeline
import common
from sklearn.preprocessing import PolynomialFeatures
base_year = common.parse_date("1900-01-01")
def years(now):
year = common.parse_date(str(now), '%Y')
return int((year - base_year).days/365)
train_data = pd.read_csv(open('../data/小区租金样本.csv'))
train_data["build_year"] = train_data["build_year"].apply(lambda param: years(param))
x = train_data[["lat", "lng", "build_year", "room", "avg_price"]]
y = train_data[["rent_price"]]
x = x.astype(np.float64)
y = y.astype(np.float64)
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.2)
c_range = np.logspace(-5, 15, 11, base=2)
gamma_range = np.logspace(-9, 3, 13, base=2)
param_grid = [{'svr__kernel': ['rbf', 'linear'], 'svr__C': c_range, 'svr__gamma': gamma_range}]
svr = svm.SVR()
pipeline = Pipeline([('poly', PolynomialFeatures(3)),
('standard', preprocessing.StandardScaler()),
('minMax', preprocessing.MinMaxScaler()),
('svr', svr)])
grid = GridSearchCV(pipeline, param_grid, cv=3, n_jobs=10)
clf = grid.fit(x_train, y_train.values.ravel())
test_score = grid.score(x_test, y_test)
print('测试得分是%s' % test_score)
print('训练得分是%s' % grid.score(x_train, y_train))
+176
View File
@@ -0,0 +1,176 @@
from sklearn import svm
import re
import pandas as pd
import warnings
import numpy as np
from dateutil.relativedelta import relativedelta
from sklearn.model_selection import GridSearchCV, train_test_split
from sklearn.pipeline import Pipeline
from sklearn import preprocessing
import datetime
import os
import sys
curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)
import common
from common import configs as cfg
warnings.simplefilter("ignore")
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
district_room_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_room_rent")
district_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
district_sell_stat = common.get_mongodb_collection(cfg.MONGO_URI, "fangjia_stat", "district_sell_stat")
def has_geo_info(seaweed):
return "lat2" in seaweed and "lng2" in seaweed and seaweed["lng2"] > 0
def get_build_year(seaweed):
if "finish_building_date" not in seaweed or seaweed["finish_building_date"] is None:
return None
pattern = re.compile(r'\d{4}')
match = pattern.match(str(seaweed["finish_building_date"]))
if not match:
return None
return int(match.group())
def is_common_community(seaweed):
return seaweed["estate_type2"] == "普通住宅"
def get_apply_seaweed(city, year_month):
stats = list(district_sell_stat.find({"city": city, "date": year_month}))
stat_dict = dict()
for stat in stats:
key = stat["city"] + "_" + stat["region"] + "_" + stat["district_name"]
stat_dict[key] = stat["avg_price"]
seaweed_list = list()
for seaweed in common.get_seaweed(city):
if not has_geo_info(seaweed):
continue
if not is_common_community(seaweed):
continue
build_year = get_build_year(seaweed)
if not build_year:
continue
key = seaweed["city"] + "_" + seaweed["region"] + "_" + seaweed["name"]
if key not in stat_dict:
continue
seaweed["build_year"] = build_year
seaweed["lat"] = seaweed["lat2"] / 10000000000
seaweed["lng"] = seaweed["lng2"] / 10000000000
seaweed["avg_price"] = stat_dict[key]
seaweed_list.append(seaweed)
return seaweed_list
def get_rent_table_by_calc_type(calc_type):
if calc_type == "ht":
return district_room_rent
elif calc_type == "sq":
return district_rent
else:
return None
def load_train_data(city, year_month, calc_type):
"""
:param city:
:param year_month:
:param calc_type:计算类型 ht:户型租金 sq: 平米租金
:return:
"""
table_name = get_rent_table_by_calc_type(calc_type)
data = list()
for seaweed in get_apply_seaweed(city, year_month):
query = {"city": seaweed["city"], "region": seaweed["region"], "name": seaweed["name"], "date": year_month,
"calc_model": {"$ne": "svr"}}
stats = list(table_name.find(query))
if len(stats) < 3:
continue
for stat in stats:
item = seaweed.copy()
item["room"] = stat["room"]
item["rent_price"] = stat["rent_price"]
data.append(item)
print(item)
return pd.DataFrame(data)
def has_rent_price(seaweed, room, year_month, calc_type):
table_name = get_rent_table_by_calc_type(calc_type)
query = {"city": seaweed["city"], "region": seaweed["region"], "name": seaweed["name"],
"date": year_month, "room": room}
return table_name.count(query) > 0
def train(city, year_month, calc_type):
"""
:param city:
:param year_month:
:param calc_type:计算类型 ht:户型租金 sq: 平米租金
:return:
"""
train_data = load_train_data(city, year_month, calc_type)
x = train_data[["lat", "lng", "build_year", "room", "avg_price"]]
y = train_data[["rent_price"]]
x = x.astype(np.float64)
y = y.astype(np.float64)
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.3)
c_range = np.logspace(-5, 15, 11, base=2)
gamma_range = np.logspace(-9, 3, 13, base=2)
param_grid = [{'svr__kernel': ['rbf', 'linear'], 'svr__C': c_range, 'svr__gamma': gamma_range}]
svr = svm.SVR()
pipeline = Pipeline([('standard', preprocessing.StandardScaler()),
('minMax', preprocessing.MinMaxScaler()),
('svr', svr)])
grid = GridSearchCV(pipeline, param_grid, cv=3, n_jobs=10)
clf = grid.fit(x_train, y_train.values.ravel())
score = grid.score(x_test, y_test)
print('精度为%s' % score)
predict_seaweed = list()
for seaweed in get_apply_seaweed(city, year_month):
for room in range(1, 4):
if not has_rent_price(seaweed, room, year_month, calc_type):
item = seaweed.copy()
item["room"] = room
predict_seaweed.append(item)
predict_df = pd.DataFrame(predict_seaweed)
result = clf.predict(predict_df[["lat", "lng", "build_year", "room", "avg_price"]])
result = pd.DataFrame([[int(val)] for val in result])
result.columns = ["rent"]
predict_df = predict_df.join(result)
predict_df = predict_df[["city", "region", "name", "room", "rent"]]
table_name = get_rent_table_by_calc_type(calc_type)
for index, row in predict_df.iterrows():
if row["rent"] <= 0:
continue
query = {
"date": year_month,
"city": row["city"],
"region": row["region"],
"name": row["name"],
"room": int(row["room"])
}
data = dict()
data["m_date"] = datetime.datetime.utcnow()
data["rent_price"] = row["rent"]
data["calc_model"] = "svr"
table_name.update_one(query, {"$set": data}, True)
print(row)
if __name__ == "__main__":
start_date = common.parse_date("2019-05-01")
now_date = common.localize(datetime.datetime.utcnow())
while start_date < now_date:
year_month = int(common.format_date(start_date - relativedelta(months=1), '%Y%m'))
for city in ["北京"]:
train("北京", year_month, "ht")
train("北京", year_month, "sq")
start_date += relativedelta(months=1)
+228
View File
@@ -0,0 +1,228 @@
import calendar
import datetime
import os
import sys
import warnings
from math import isnan
import pandas as pd
from dateutil.relativedelta import relativedelta
from joblib import load
curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)
import common
from common import configs as cfg
warnings.simplefilter("ignore")
def k_means(series):
return common.k_means(series)
clf = load("C:\\Users\\wenyan\\PycharmProjects\\Sklearn\\data\\house_type_classify_model.joblib")
def house_type_filter(houses):
"""
通过模型过滤掉合租房源
:param houses:
:return:
"""
if len(houses) == 0:
return houses
X = houses[["room", "area"]]
houses["rent_type_predict"] = clf.predict(X)
houses["prob1"] = clf.predict_proba(X)[:, 1]
houses = houses[(houses.rent_type_predict == 1) & (houses.prob1 > 0.8)]
del houses["rent_type_predict"]
del houses["prob1"]
return houses
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
region_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "region_rent")
district_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
# 删除无用列
def remove_useless_columns(df):
for col in ["fitment", "direction", "root"]:
if col in df.columns:
del df[col]
def weighting(df):
if 'root' not in df.columns.values.tolist():
return df
root_weight = {
"zhuge.com": 3,
"ke.com": 8,
"lianjia.com": 8
}
weighting_data = list()
for index, row in df.iterrows():
row = dict(row)
weighting_data.append(row)
if row['root'] in root_weight:
for i in range(1, root_weight[row['root']]):
weighting_data.append(row)
print("org:{} dest:{}".format(len(df), len(weighting_data)))
return pd.DataFrame(weighting_data)
def get_start_date_and_end_date(year_month):
date = common.parse_date(str(year_month), '%Y%m')
year = date.year
month = date.month
first_day = datetime.datetime(year=year, month=month, day=1)
month_range = calendar.monthrange(year, month)
end_day = datetime.datetime(year=year, month=month, day=month_range[1])
return first_day, end_day
def calc_city_rent(_city, year_month):
start_date, end_date = get_start_date_and_end_date(year_month)
stats = common.get_least_region_stat(_city, None, start_date=start_date, end_date=end_date)
start_date = int(common.format_date(start_date, "%Y%m%d"))
end_date = int(common.format_date(end_date, "%Y%m%d"))
houses = common.load_house_data(city=_city, region=None, name=None, cat="rent", rent_type="整租",
start_date=start_date, end_date=end_date)
houses = house_type_filter(houses)
houses = weighting(houses)
remove_useless_columns(houses)
houses.dropna(axis=0, how='any', inplace=True)
houses = houses[houses.avg_price > 0]
houses.sort_values(by='p_date', ascending=False, inplace=True)
agg = houses.groupby(by="room")[['avg_price']].agg(['mean', 'median', 'count', k_means, 'min', 'max'])
agg = agg.applymap(int)
agg.columns = ["mean", "median", "count", "k_means", "min", "max"]
for index, row in agg.iterrows():
query = {
"date": year_month,
"cat": "city",
"city": _city,
"room": int(index)
}
data = dict(row)
for k, v in data.items():
data[k] = int(v)
data["m_date"] = datetime.datetime.utcnow()
if data["count"] > cfg.CITY_RENT_MIN_HOUSE_COUNT:
data["rent_price"] = data["k_means"]
if len(stats) > 0:
data["avg_price"] = stats[0]["avg_price"]
data["price_rent_ratio"] = int(data["avg_price"] / data["k_means"])
region_rent.update_one(query, {"$set": data}, True)
def calc_region_rent(_city, _region, year_month):
start_date, end_date = get_start_date_and_end_date(year_month)
stats = common.get_least_region_stat(city=_city, region=_region, start_date=start_date, end_date=end_date, limit=1)
start_date = int(common.format_date(start_date, "%Y%m%d"))
end_date = int(common.format_date(end_date, "%Y%m%d"))
houses = common.load_house_data(city=_city, region=_region, name=None, cat="rent", rent_type="整租",
start_date=start_date, end_date=end_date)
houses = house_type_filter(houses)
houses = weighting(houses)
remove_useless_columns(houses)
houses.dropna(axis=0, how='any', inplace=True)
if len(houses) == 0:
return
houses.sort_values(by='p_date', ascending=False, inplace=True)
agg = houses.groupby(by="room")[['avg_price']].agg(['mean', 'median', 'count', k_means])
agg = agg.applymap(int)
agg.columns = ["mean", "median", "count", "k_means"]
city_rent = dict()
for item in list(region_rent.find({"cat": "city", "city": _city, "date": year_month})):
city_rent[item["room"]] = item["price_rent_ratio"]
for index, row in agg.iterrows():
query = {
"date": year_month,
"cat": "region",
"region": _region,
"city": _city,
"room": int(index)
}
data = dict(row)
for k, v in data.items():
data[k] = int(v)
data["m_date"] = datetime.datetime.utcnow()
if data["count"] > cfg.REGION_RENT_MIN_HOUSE_COUNT:
if len(stats) == 0:
data["rent_price"] = data["k_means"]
else:
region_price = stats[0]["avg_price"]
data["avg_price"] = region_price
data["rent_price"] = data["k_means"]
data["price_rent_ratio"] = int(region_price / data["k_means"])
region_rent.update_one(query, {"$set": data}, True)
def filter_by_price_rent(houses, region_price):
if region_price is None or region_price == 0:
return houses
filtered_houses = list()
for index, row in houses.iterrows():
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3:
continue
if isnan(row["avg_price"]):
row["avg_price"] = int(row["total_price"] / row["area"])
house_price_rent_ratio = int(region_price / row["avg_price"])
if 100 < house_price_rent_ratio < 1500:
filtered_houses.append(row)
return pd.DataFrame(filtered_houses)
def filter_by_price_to_rent(houses, seaweed, year_month):
stats = common.get_least_district_stat(seaweed["city"], seaweed["region"], seaweed["name"], seaweed["estate_type2"]
, limit=1)
if len(stats) == 0:
return houses
seaweed_avg_price = stats[0]["avg_price"]
house_type_price_rent = dict()
for stat in list(region_rent.find({"city": seaweed["city"], "region": seaweed["region"], "date": year_month})):
if stat["cat"] == "city":
house_type_price_rent["city_{}".format(stat["room"])] = stat["price_rent_ratio"]
else:
house_type_price_rent["region_{}".format(stat["room"])] = stat["price_rent_ratio"]
filtered_houses = list()
for index, row in houses.iterrows():
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3:
continue
if isnan(row["avg_price"]):
row["avg_price"] = int(row["total_price"] / row["area"])
house_price_rent_ratio = int(seaweed_avg_price / row["avg_price"])
if "region_{}".format(row["room"]) in house_type_price_rent:
key = "region_{}".format(row["room"])
if house_type_price_rent[key] * 0.5 < house_price_rent_ratio < house_type_price_rent[key] * 2:
filtered_houses.append(row)
else:
print("过滤:" + str(row))
elif "city_{}".format(row["room"]) in house_type_price_rent:
key = "city_{}".format(row["room"])
if house_type_price_rent[key] * 0.25 < house_price_rent_ratio < house_type_price_rent[key] * 4:
filtered_houses.append(row)
else:
print("过滤:" + str(row))
else:
filtered_houses.append(row)
return pd.DataFrame(filtered_houses)
def main():
start_date = common.parse_date("2019-05-01")
now_date = common.localize(datetime.datetime.utcnow())
while start_date < now_date:
year_month = int(common.format_date(start_date - relativedelta(months=1), '%Y%m'))
for city in ["北京"]:
print("{}:{}\t开始计算".format(year_month, city))
calc_city_rent(city, year_month)
for region in common.get_regions(city):
print("{}:{} {}\t开始计算:".format(year_month, city, region))
calc_region_rent(city, region, year_month)
start_date += relativedelta(months=1)
if __name__ == "__main__":
main()
+261
View File
@@ -0,0 +1,261 @@
import calendar
import datetime
import multiprocessing
import os
import sys
import warnings
from math import isnan
from joblib import load
import pandas as pd
from dateutil.relativedelta import relativedelta
curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)
import common
warnings.simplefilter("ignore")
def k_means(series):
return common.k_means(series)
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
region_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "region_rent")
district_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
district_room_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_room_rent")
# 删除无用列
def remove_useless_columns(df):
for col in ["fitment", "direction", "root"]:
if col in df.columns:
del df[col]
def weighting(df):
root_weight = {
"zhuge.com": 2,
"5i5j.com": 3,
"ke.com": 8,
"lianjia.com": 8
}
weighting_data = list()
for index, row in df.iterrows():
row = dict(row)
weighting_data.append(row)
if 'root' in row and row['root'] in root_weight:
for i in range(1, root_weight[row['root']]):
weighting_data.append(row)
return pd.DataFrame(weighting_data)
def get_region_markup(_city, _region, _start, _end, _room):
_room = 2
start_stat = region_rent.find_one(
{"cat": "region", "city": _city, "region": _region, "date": _start, "room": _room})
end_stat = region_rent.find_one({"cat": "region", "city": _city, "region": _region, "date": _end, "room": _room})
if not start_stat or not end_stat:
return 0
return (end_stat["rent_price"] - start_stat["rent_price"]) / start_stat["rent_price"]
def correct_house(_houses, _year_month):
calc_year_month = common.parse_date(str(_year_month), '%Y%m')
for index, house in _houses.iterrows():
p_date = common.parse_date(str(house["p_date"]), '%Y%m%d')
days = (calc_year_month - p_date).days
if days > 180:
markup = get_region_markup(house["city"], house["region"], int(house["p_date"] / 100), _year_month, house["room"])
_houses.at[index, "avg_price"] = house["avg_price"] * (1 + markup)
_houses.at[index, "total_price"] = house["total_price"] * (1 + markup)
return _houses
def get_start_date_and_end_date(year_month):
date = common.parse_date(str(year_month), '%Y%m')
year = date.year
month = date.month
first_day = datetime.datetime(year=year, month=month, day=1)
month_range = calendar.monthrange(year, month)
end_day = datetime.datetime(year=year, month=month, day=month_range[1])
return first_day, end_day
def filter_by_price_rent(houses, region_price):
if region_rent is None:
return houses
filtered_houses = list()
for index, row in houses.iterrows():
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3:
continue
if isnan(row["avg_price"]):
row["avg_price"] = int(row["total_price"] / row["area"])
house_price_rent_ratio = int(region_price / row["avg_price"])
if 100 < house_price_rent_ratio < 1000:
filtered_houses.append(row)
return pd.DataFrame(filtered_houses)
def filter_by_price_to_rent(houses, seaweed, year_month):
stats = common.get_least_district_stat(seaweed["city"], seaweed["region"], seaweed["name"], seaweed["estate_type2"]
, limit=1)
if len(stats) == 0:
return houses
seaweed_avg_price = stats[0]["avg_price"]
house_type_price_rent = dict()
for stat in list(region_rent.find({"city": seaweed["city"], "region": seaweed["region"], "date": year_month})):
if stat["cat"] == "city":
house_type_price_rent["city_{}".format(stat["room"])] = stat["price_rent_ratio"]
else:
house_type_price_rent["region_{}".format(stat["room"])] = stat["price_rent_ratio"]
if len(house_type_price_rent) > 0:
house_type_price_rent["region"] = int(pd.Series(list(house_type_price_rent.values())).mean())
filtered_houses = list()
houses[["room", "total_price"]].to_csv("../data/房源.csv", index=False)
for index, row in houses.iterrows():
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3 or row["area"] == 0:
continue
if "avg_price" not in row or isnan(row["avg_price"]):
row["avg_price"] = int(row["total_price"] / row["area"])
if row["avg_price"] < 10:
continue
house_price_rent_ratio = int(seaweed_avg_price / row["avg_price"])
if "region_{}".format(row["room"]) in house_type_price_rent:
key = "region_{}".format(row["room"])
if house_type_price_rent[key] * 0.5 < house_price_rent_ratio < house_type_price_rent[key] * 2:
filtered_houses.append(row)
elif "city_{}".format(row["room"]) in house_type_price_rent:
key = "city_{}".format(row["room"])
if house_type_price_rent[key] * 0.25 < house_price_rent_ratio < house_type_price_rent[key] * 4:
filtered_houses.append(row)
elif "region" in house_type_price_rent:
key = "region"
if house_type_price_rent[key] * 0.4 < house_price_rent_ratio < house_type_price_rent[key] * 2:
filtered_houses.append(row)
else:
filtered_houses.append(row)
return pd.DataFrame(filtered_houses)
clf = load("C:\\Users\\wenyan\\PycharmProjects\\Sklearn\\data\\house_type_classify_model.joblib")
def house_type_filter(houses):
"""
通过模型过滤掉合租房源
:param houses:
:return:
"""
X = houses[["room", "area"]]
houses["rent_type_predict"] = clf.predict(X)
houses["prob1"] = clf.predict_proba(X)[:, 1]
houses = houses[(houses.rent_type_predict == 1) & (0.8 < houses.prob1) & (houses.prob1 < 0.99999)]
del houses["rent_type_predict"]
del houses["prob1"]
return houses
def drop_duplicates(houses):
houses["p_month"] = houses["p_date"].apply(lambda x: int(x/100))
houses = houses.drop_duplicates(subset=["area", "avg_price", "p_month", "room", "root", "total_price"])
del houses["p_month"]
return houses
def calc_seaweed_rent(seaweed, year_month):
print("开始计算小区租金{}: {} {} {}".format(year_month, seaweed["city"], seaweed["region"], seaweed["name"]))
start_date, end_date = get_start_date_and_end_date(year_month)
stats = common.get_least_district_stat(seaweed["city"], seaweed["region"], seaweed["name"], seaweed["estate_type2"]
, start_date=start_date, end_date=end_date, limit=1)
start_date = end_date - relativedelta(years=1)
first_date = end_date - relativedelta(months=1)
start_date = int(common.format_date(start_date, "%Y%m%d"))
first_date = int(common.format_date(first_date, "%Y%m%d"))
end_date = int(common.format_date(end_date, "%Y%m%d"))
alias = None
if alias in seaweed:
alias = seaweed["alias"]
houses = pd.DataFrame()
for house_type in range(1, 4):
_houses = common.load_house_data(city=seaweed["city"], region=seaweed["region"], name=seaweed["name"],
cat="rent", room=house_type, rent_type="整租", end_date=end_date,
start_date=first_date, alias=alias, limit=100)
if len(_houses) < 5:
_houses = common.load_house_data(city=seaweed["city"], region=seaweed["region"], name=seaweed["name"],
cat="rent", room=house_type,
rent_type="整租", end_date=end_date, start_date=start_date, alias=alias,
limit=100)
houses = houses.append(_houses, ignore_index=True)
if len(houses) == 0:
return
houses = drop_duplicates(houses)
houses = house_type_filter(houses)
houses = filter_by_price_to_rent(houses, seaweed, year_month)
houses = correct_house(houses, year_month)
houses = weighting(houses)
remove_useless_columns(houses)
houses.dropna(axis=0, how='any', inplace=True)
if len(houses) < 3:
return
if "room" not in houses.columns:
return
houses = houses[houses.avg_price > 0]
houses.sort_values(by='p_date', ascending=False, inplace=True)
agg = houses.groupby(by="room")[['avg_price']].agg(['mean', 'median', 'count', k_means])
total_price_agg = houses.groupby(by="room")[['total_price']].agg(['mean', 'median', 'count', k_means])
agg.columns = ["mean", "median", "count", "k_means"]
total_price_agg.columns = ["mean", "median", "count", "k_means"]
for index, row in agg.iterrows():
query = {
"date": year_month,
"city": seaweed["city"],
"region": seaweed["region"],
"name": seaweed["name"],
"room": int(index)
}
data = dict(row)
for k, v in data.items():
data[k] = int(v)
data["m_date"] = datetime.datetime.utcnow()
data["rent_price"] = data["k_means"]
if len(stats) != 0:
data["avg_price"] = stats[0]["avg_price"]
data["price_rent_ratio"] = int(data["avg_price"] / data["rent_price"])
district_rent.update_one(query, {"$set": data}, True)
for index, row in total_price_agg.iterrows():
query = {
"date": year_month,
"city": seaweed["city"],
"region": seaweed["region"],
"name": seaweed["name"],
"room": int(index)
}
data = dict(row)
for k, v in data.items():
data[k] = int(v)
data["m_date"] = datetime.datetime.utcnow()
data["rent_price"] = data["k_means"]
district_room_rent.update_one(query, {"$set": data}, True)
def main():
pool = multiprocessing.Pool(processes=30)
start_date = common.parse_date("2019-05-01")
now_date = common.localize(datetime.datetime.utcnow())
while start_date < now_date:
year_month = int(common.format_date(start_date - relativedelta(months=1), '%Y%m'))
for city in ["北京"]:
for community in common.get_seaweed(city):
pool.apply_async(calc_seaweed_rent, (community, year_month,))
start_date += relativedelta(months=1)
pool.close()
pool.join()
print("done")
if __name__ == "__main__":
calc_seaweed_rent(common.get_seaweed("北京", "西城", "富国里")[0], 201904)
# main()
View File
+33
View File
@@ -0,0 +1,33 @@
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from common import configs
import common
plt.rcParams['savefig.dpi'] = 400
plt.rcParams['figure.dpi'] = 400
plt.rcParams['figure.figsize'] = (12, 4)
plt.rcParams['font.sans-serif'] = ['SimHei']
region_rent = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia_rent", "region_rent")
price_rent = dict()
for region in common.get_regions("北京"):
df = pd.DataFrame(list(region_rent.find({"cat": "region", "region": region}, {"price_rent_ratio": 1})))
if len(df) > 0:
price_rent[region] = int(df["price_rent_ratio"].mean())
fig, ax = plt.subplots()
x = []
y = []
z = []
avg = np.array(list(price_rent.values())).mean()
for k, v in price_rent.items():
x.append(k)
y.append(v)
z.append(avg)
ax.bar(x, y, width=0.5)
ax.plot(x, y, label='区域售租比')
ax.plot(x, z, label='平均线:' + str(avg))
ax.set(xlabel='区域', ylabel='租售比', title='区域租售比分布图')
ax.grid()
plt.legend()
plt.savefig("D:\\建设银行\\租售比2000以内\\各区域租售比.png")
for item in price_rent.items():
print(str(item[0]) + "\t" + str(item[1]))
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import linear_model
df = pd.read_csv(open("C:\\Users\\wenyan\\PycharmProjects\\Sklearn\\data\\房源.csv"))
fig, ax = plt.subplots()
ax.scatter(df["room"], df["total_price"])
lr = linear_model.LinearRegression(normalize=True)
X = df["room"].values.reshape(len(df["room"]), 1)
Y = df["total_price"].values.ravel()
lr.fit(X, Y)
ax.plot([1, 2, 3], lr.predict([[1], [2], [3]]), 'r')
+17
View File
@@ -0,0 +1,17 @@
import matplotlib.pyplot as plt
import pandas as pd
import common
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
HOUSE_RENT = common.get_mongodb_collection("mongodb://192.168.0.105:27018/", "vip_site", "esf_houseRent")
DISTRICT_RENT = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_room_rent")
stats = list(DISTRICT_RENT.find({"city": "北京", "region": "朝阳", "name": "甜水园"}))
df = pd.DataFrame(stats)
df.dropna()
df = df[["city", "region", "name", "date", "room", "rent_price"]]
df["date"] = df["date"].apply(lambda x: common.parse_date(str(x), '%Y%m'))
room1_df = df[df["room"] == 2]
room1_df = room1_df.sort_values(by="date")
fig, ax = plt.subplots()
ax.plot(room1_df["date"], room1_df["rent_price"], 'h-.')
plt.show()
+36
View File
@@ -0,0 +1,36 @@
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from common import configs
import common
# region_rent = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia_rent", "region_rent")
# price_rent = dict()
# for region in common.get_regions("北京"):
# df = pd.DataFrame(list(region_rent.find({"cat": "region", "region": region}, {"price_rent_ratio": 1})))
# if len(df) > 0:
# price_rent[region] = int(df["price_rent_ratio"].mean())
# plt.rcParams['font.sans-serif'] = ['SimHei']
# district_rent = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia_rent", "district_rent")
# plt.rcParams['savefig.dpi'] = 800
# plt.rcParams['figure.dpi'] = 800
# plt.rcParams['figure.figsize'] = (6, 4)
# for region in common.get_regions("北京"):
# df = pd.DataFrame(list(district_rent.find({"region": region}, {"price_rent_ratio": 1})))
# ratio = np.array(df["price_rent_ratio"])
# result = pd.value_counts(ratio)
# result = result.sort_index()
# fig, ax = plt.subplots()
# ax.plot(result.index, result.values)
# if region in price_rent:
# y = np.arange(int(max(result.values) * 1.2))
# x = np.zeros(len(y)) + price_rent[region]
# ax.plot(x, y, label="区域售租比")
# ax.plot(x * 2, y, '--', label="2倍区域售租比")
# ax.plot(x * 0.4, y, '-', label="0.4倍区域售租比")
# ax.set(xlabel='售租比', ylabel='小区量', title=region + '区小区售租比分布图')
# ax.grid()
# plt.legend()
# plt.savefig("D:\\建设银行\\租售比\\" + region + "租售比分布分布图.png")
numbers = np.array([12000, 12000, 14900, 20000])
print(common.k_means(numbers))