Files
Sklearn/rent/city_region_rent.py
2019-05-07 11:33:15 +08:00

229 lines
8.9 KiB
Python

import calendar
import datetime
import os
import sys
import warnings
from math import isnan
import pandas as pd
from dateutil.relativedelta import relativedelta
from joblib import load
curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)
import common
from common import configs as cfg
warnings.simplefilter("ignore")
def k_means(series):
return common.k_means(series)
clf = load("C:\\Users\\wenyan\\PycharmProjects\\Sklearn\\data\\house_type_classify_model.joblib")
def house_type_filter(houses):
"""
通过模型过滤掉合租房源
:param houses:
:return:
"""
if len(houses) == 0:
return houses
X = houses[["room", "area"]]
houses["rent_type_predict"] = clf.predict(X)
houses["prob1"] = clf.predict_proba(X)[:, 1]
houses = houses[(houses.rent_type_predict == 1) & (houses.prob1 > 0.8)]
del houses["rent_type_predict"]
del houses["prob1"]
return houses
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
region_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "region_rent")
district_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
# 删除无用列
def remove_useless_columns(df):
for col in ["fitment", "direction", "root"]:
if col in df.columns:
del df[col]
def weighting(df):
if 'root' not in df.columns.values.tolist():
return df
root_weight = {
"zhuge.com": 3,
"ke.com": 8,
"lianjia.com": 8
}
weighting_data = list()
for index, row in df.iterrows():
row = dict(row)
weighting_data.append(row)
if row['root'] in root_weight:
for i in range(1, root_weight[row['root']]):
weighting_data.append(row)
print("org:{} dest:{}".format(len(df), len(weighting_data)))
return pd.DataFrame(weighting_data)
def get_start_date_and_end_date(year_month):
date = common.parse_date(str(year_month), '%Y%m')
year = date.year
month = date.month
first_day = datetime.datetime(year=year, month=month, day=1)
month_range = calendar.monthrange(year, month)
end_day = datetime.datetime(year=year, month=month, day=month_range[1])
return first_day, end_day
def calc_city_rent(_city, year_month):
start_date, end_date = get_start_date_and_end_date(year_month)
stats = common.get_least_region_stat(_city, None, start_date=start_date, end_date=end_date)
start_date = int(common.format_date(start_date, "%Y%m%d"))
end_date = int(common.format_date(end_date, "%Y%m%d"))
houses = common.load_house_data(city=_city, region=None, name=None, cat="rent", rent_type="整租",
start_date=start_date, end_date=end_date)
houses = house_type_filter(houses)
houses = weighting(houses)
remove_useless_columns(houses)
houses.dropna(axis=0, how='any', inplace=True)
houses = houses[houses.avg_price > 0]
houses.sort_values(by='p_date', ascending=False, inplace=True)
agg = houses.groupby(by="room")[['avg_price']].agg(['mean', 'median', 'count', k_means, 'min', 'max'])
agg = agg.applymap(int)
agg.columns = ["mean", "median", "count", "k_means", "min", "max"]
for index, row in agg.iterrows():
query = {
"date": year_month,
"cat": "city",
"city": _city,
"room": int(index)
}
data = dict(row)
for k, v in data.items():
data[k] = int(v)
data["m_date"] = datetime.datetime.utcnow()
if data["count"] > cfg.CITY_RENT_MIN_HOUSE_COUNT:
data["rent_price"] = data["k_means"]
if len(stats) > 0:
data["avg_price"] = stats[0]["avg_price"]
data["price_rent_ratio"] = int(data["avg_price"] / data["k_means"])
region_rent.update_one(query, {"$set": data}, True)
def calc_region_rent(_city, _region, year_month):
start_date, end_date = get_start_date_and_end_date(year_month)
stats = common.get_least_region_stat(city=_city, region=_region, start_date=start_date, end_date=end_date, limit=1)
start_date = int(common.format_date(start_date, "%Y%m%d"))
end_date = int(common.format_date(end_date, "%Y%m%d"))
houses = common.load_house_data(city=_city, region=_region, name=None, cat="rent", rent_type="整租",
start_date=start_date, end_date=end_date)
houses = house_type_filter(houses)
houses = weighting(houses)
remove_useless_columns(houses)
houses.dropna(axis=0, how='any', inplace=True)
if len(houses) == 0:
return
houses.sort_values(by='p_date', ascending=False, inplace=True)
agg = houses.groupby(by="room")[['avg_price']].agg(['mean', 'median', 'count', k_means])
agg = agg.applymap(int)
agg.columns = ["mean", "median", "count", "k_means"]
city_rent = dict()
for item in list(region_rent.find({"cat": "city", "city": _city, "date": year_month})):
city_rent[item["room"]] = item["price_rent_ratio"]
for index, row in agg.iterrows():
query = {
"date": year_month,
"cat": "region",
"region": _region,
"city": _city,
"room": int(index)
}
data = dict(row)
for k, v in data.items():
data[k] = int(v)
data["m_date"] = datetime.datetime.utcnow()
if data["count"] > cfg.REGION_RENT_MIN_HOUSE_COUNT:
if len(stats) == 0:
data["rent_price"] = data["k_means"]
else:
region_price = stats[0]["avg_price"]
data["avg_price"] = region_price
data["rent_price"] = data["k_means"]
data["price_rent_ratio"] = int(region_price / data["k_means"])
region_rent.update_one(query, {"$set": data}, True)
def filter_by_price_rent(houses, region_price):
if region_price is None or region_price == 0:
return houses
filtered_houses = list()
for index, row in houses.iterrows():
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3:
continue
if isnan(row["avg_price"]):
row["avg_price"] = int(row["total_price"] / row["area"])
house_price_rent_ratio = int(region_price / row["avg_price"])
if 100 < house_price_rent_ratio < 1500:
filtered_houses.append(row)
return pd.DataFrame(filtered_houses)
def filter_by_price_to_rent(houses, seaweed, year_month):
stats = common.get_least_district_stat(seaweed["city"], seaweed["region"], seaweed["name"], seaweed["estate_type2"]
, limit=1)
if len(stats) == 0:
return houses
seaweed_avg_price = stats[0]["avg_price"]
house_type_price_rent = dict()
for stat in list(region_rent.find({"city": seaweed["city"], "region": seaweed["region"], "date": year_month})):
if stat["cat"] == "city":
house_type_price_rent["city_{}".format(stat["room"])] = stat["price_rent_ratio"]
else:
house_type_price_rent["region_{}".format(stat["room"])] = stat["price_rent_ratio"]
filtered_houses = list()
for index, row in houses.iterrows():
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3:
continue
if isnan(row["avg_price"]):
row["avg_price"] = int(row["total_price"] / row["area"])
house_price_rent_ratio = int(seaweed_avg_price / row["avg_price"])
if "region_{}".format(row["room"]) in house_type_price_rent:
key = "region_{}".format(row["room"])
if house_type_price_rent[key] * 0.5 < house_price_rent_ratio < house_type_price_rent[key] * 2:
filtered_houses.append(row)
else:
print("过滤:" + str(row))
elif "city_{}".format(row["room"]) in house_type_price_rent:
key = "city_{}".format(row["room"])
if house_type_price_rent[key] * 0.25 < house_price_rent_ratio < house_type_price_rent[key] * 4:
filtered_houses.append(row)
else:
print("过滤:" + str(row))
else:
filtered_houses.append(row)
return pd.DataFrame(filtered_houses)
def main():
start_date = common.parse_date("2019-05-01")
now_date = common.localize(datetime.datetime.utcnow())
while start_date < now_date:
year_month = int(common.format_date(start_date - relativedelta(months=1), '%Y%m'))
for city in ["北京"]:
print("{}:{}\t开始计算".format(year_month, city))
calc_city_rent(city, year_month)
for region in common.get_regions(city):
print("{}:{} {}\t开始计算:".format(year_month, city, region))
calc_region_rent(city, region, year_month)
start_date += relativedelta(months=1)
if __name__ == "__main__":
main()