229 lines
8.9 KiB
Python
229 lines
8.9 KiB
Python
import calendar
|
|
import datetime
|
|
import os
|
|
import sys
|
|
import warnings
|
|
from math import isnan
|
|
import pandas as pd
|
|
from dateutil.relativedelta import relativedelta
|
|
from joblib import load
|
|
curPath = os.path.abspath(os.path.dirname(__file__))
|
|
rootPath = os.path.split(curPath)[0]
|
|
sys.path.append(rootPath)
|
|
import common
|
|
from common import configs as cfg
|
|
warnings.simplefilter("ignore")
|
|
|
|
|
|
def k_means(series):
|
|
return common.k_means(series)
|
|
|
|
|
|
clf = load("C:\\Users\\wenyan\\PycharmProjects\\Sklearn\\data\\house_type_classify_model.joblib")
|
|
|
|
|
|
def house_type_filter(houses):
|
|
"""
|
|
通过模型过滤掉合租房源
|
|
:param houses:
|
|
:return:
|
|
"""
|
|
if len(houses) == 0:
|
|
return houses
|
|
X = houses[["room", "area"]]
|
|
houses["rent_type_predict"] = clf.predict(X)
|
|
houses["prob1"] = clf.predict_proba(X)[:, 1]
|
|
houses = houses[(houses.rent_type_predict == 1) & (houses.prob1 > 0.8)]
|
|
del houses["rent_type_predict"]
|
|
del houses["prob1"]
|
|
return houses
|
|
|
|
|
|
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
|
|
region_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "region_rent")
|
|
district_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
|
|
|
|
|
|
# 删除无用列
|
|
def remove_useless_columns(df):
|
|
for col in ["fitment", "direction", "root"]:
|
|
if col in df.columns:
|
|
del df[col]
|
|
|
|
|
|
def weighting(df):
|
|
if 'root' not in df.columns.values.tolist():
|
|
return df
|
|
root_weight = {
|
|
"zhuge.com": 3,
|
|
"ke.com": 8,
|
|
"lianjia.com": 8
|
|
}
|
|
weighting_data = list()
|
|
for index, row in df.iterrows():
|
|
row = dict(row)
|
|
weighting_data.append(row)
|
|
if row['root'] in root_weight:
|
|
for i in range(1, root_weight[row['root']]):
|
|
weighting_data.append(row)
|
|
print("org:{} dest:{}".format(len(df), len(weighting_data)))
|
|
return pd.DataFrame(weighting_data)
|
|
|
|
|
|
def get_start_date_and_end_date(year_month):
|
|
date = common.parse_date(str(year_month), '%Y%m')
|
|
year = date.year
|
|
month = date.month
|
|
first_day = datetime.datetime(year=year, month=month, day=1)
|
|
month_range = calendar.monthrange(year, month)
|
|
end_day = datetime.datetime(year=year, month=month, day=month_range[1])
|
|
return first_day, end_day
|
|
|
|
|
|
def calc_city_rent(_city, year_month):
|
|
start_date, end_date = get_start_date_and_end_date(year_month)
|
|
stats = common.get_least_region_stat(_city, None, start_date=start_date, end_date=end_date)
|
|
start_date = int(common.format_date(start_date, "%Y%m%d"))
|
|
end_date = int(common.format_date(end_date, "%Y%m%d"))
|
|
houses = common.load_house_data(city=_city, region=None, name=None, cat="rent", rent_type="整租",
|
|
start_date=start_date, end_date=end_date)
|
|
houses = house_type_filter(houses)
|
|
houses = weighting(houses)
|
|
remove_useless_columns(houses)
|
|
houses.dropna(axis=0, how='any', inplace=True)
|
|
houses = houses[houses.avg_price > 0]
|
|
houses.sort_values(by='p_date', ascending=False, inplace=True)
|
|
agg = houses.groupby(by="room")[['avg_price']].agg(['mean', 'median', 'count', k_means, 'min', 'max'])
|
|
agg = agg.applymap(int)
|
|
agg.columns = ["mean", "median", "count", "k_means", "min", "max"]
|
|
for index, row in agg.iterrows():
|
|
query = {
|
|
"date": year_month,
|
|
"cat": "city",
|
|
"city": _city,
|
|
"room": int(index)
|
|
}
|
|
data = dict(row)
|
|
for k, v in data.items():
|
|
data[k] = int(v)
|
|
data["m_date"] = datetime.datetime.utcnow()
|
|
if data["count"] > cfg.CITY_RENT_MIN_HOUSE_COUNT:
|
|
data["rent_price"] = data["k_means"]
|
|
if len(stats) > 0:
|
|
data["avg_price"] = stats[0]["avg_price"]
|
|
data["price_rent_ratio"] = int(data["avg_price"] / data["k_means"])
|
|
region_rent.update_one(query, {"$set": data}, True)
|
|
|
|
|
|
def calc_region_rent(_city, _region, year_month):
|
|
start_date, end_date = get_start_date_and_end_date(year_month)
|
|
stats = common.get_least_region_stat(city=_city, region=_region, start_date=start_date, end_date=end_date, limit=1)
|
|
start_date = int(common.format_date(start_date, "%Y%m%d"))
|
|
end_date = int(common.format_date(end_date, "%Y%m%d"))
|
|
houses = common.load_house_data(city=_city, region=_region, name=None, cat="rent", rent_type="整租",
|
|
start_date=start_date, end_date=end_date)
|
|
houses = house_type_filter(houses)
|
|
houses = weighting(houses)
|
|
remove_useless_columns(houses)
|
|
houses.dropna(axis=0, how='any', inplace=True)
|
|
if len(houses) == 0:
|
|
return
|
|
houses.sort_values(by='p_date', ascending=False, inplace=True)
|
|
agg = houses.groupby(by="room")[['avg_price']].agg(['mean', 'median', 'count', k_means])
|
|
agg = agg.applymap(int)
|
|
agg.columns = ["mean", "median", "count", "k_means"]
|
|
city_rent = dict()
|
|
for item in list(region_rent.find({"cat": "city", "city": _city, "date": year_month})):
|
|
city_rent[item["room"]] = item["price_rent_ratio"]
|
|
for index, row in agg.iterrows():
|
|
query = {
|
|
"date": year_month,
|
|
"cat": "region",
|
|
"region": _region,
|
|
"city": _city,
|
|
"room": int(index)
|
|
}
|
|
data = dict(row)
|
|
for k, v in data.items():
|
|
data[k] = int(v)
|
|
data["m_date"] = datetime.datetime.utcnow()
|
|
if data["count"] > cfg.REGION_RENT_MIN_HOUSE_COUNT:
|
|
if len(stats) == 0:
|
|
data["rent_price"] = data["k_means"]
|
|
else:
|
|
region_price = stats[0]["avg_price"]
|
|
data["avg_price"] = region_price
|
|
data["rent_price"] = data["k_means"]
|
|
data["price_rent_ratio"] = int(region_price / data["k_means"])
|
|
region_rent.update_one(query, {"$set": data}, True)
|
|
|
|
|
|
def filter_by_price_rent(houses, region_price):
|
|
if region_price is None or region_price == 0:
|
|
return houses
|
|
filtered_houses = list()
|
|
for index, row in houses.iterrows():
|
|
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3:
|
|
continue
|
|
if isnan(row["avg_price"]):
|
|
row["avg_price"] = int(row["total_price"] / row["area"])
|
|
house_price_rent_ratio = int(region_price / row["avg_price"])
|
|
if 100 < house_price_rent_ratio < 1500:
|
|
filtered_houses.append(row)
|
|
return pd.DataFrame(filtered_houses)
|
|
|
|
|
|
def filter_by_price_to_rent(houses, seaweed, year_month):
|
|
stats = common.get_least_district_stat(seaweed["city"], seaweed["region"], seaweed["name"], seaweed["estate_type2"]
|
|
, limit=1)
|
|
if len(stats) == 0:
|
|
return houses
|
|
seaweed_avg_price = stats[0]["avg_price"]
|
|
house_type_price_rent = dict()
|
|
for stat in list(region_rent.find({"city": seaweed["city"], "region": seaweed["region"], "date": year_month})):
|
|
if stat["cat"] == "city":
|
|
house_type_price_rent["city_{}".format(stat["room"])] = stat["price_rent_ratio"]
|
|
else:
|
|
house_type_price_rent["region_{}".format(stat["room"])] = stat["price_rent_ratio"]
|
|
|
|
filtered_houses = list()
|
|
for index, row in houses.iterrows():
|
|
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3:
|
|
continue
|
|
if isnan(row["avg_price"]):
|
|
row["avg_price"] = int(row["total_price"] / row["area"])
|
|
house_price_rent_ratio = int(seaweed_avg_price / row["avg_price"])
|
|
if "region_{}".format(row["room"]) in house_type_price_rent:
|
|
key = "region_{}".format(row["room"])
|
|
if house_type_price_rent[key] * 0.5 < house_price_rent_ratio < house_type_price_rent[key] * 2:
|
|
filtered_houses.append(row)
|
|
else:
|
|
print("过滤:" + str(row))
|
|
elif "city_{}".format(row["room"]) in house_type_price_rent:
|
|
key = "city_{}".format(row["room"])
|
|
if house_type_price_rent[key] * 0.25 < house_price_rent_ratio < house_type_price_rent[key] * 4:
|
|
filtered_houses.append(row)
|
|
else:
|
|
print("过滤:" + str(row))
|
|
else:
|
|
filtered_houses.append(row)
|
|
return pd.DataFrame(filtered_houses)
|
|
|
|
|
|
def main():
|
|
start_date = common.parse_date("2019-05-01")
|
|
now_date = common.localize(datetime.datetime.utcnow())
|
|
while start_date < now_date:
|
|
year_month = int(common.format_date(start_date - relativedelta(months=1), '%Y%m'))
|
|
for city in ["北京"]:
|
|
print("{}:{}\t开始计算".format(year_month, city))
|
|
calc_city_rent(city, year_month)
|
|
for region in common.get_regions(city):
|
|
print("{}:{} {}\t开始计算:".format(year_month, city, region))
|
|
calc_region_rent(city, region, year_month)
|
|
start_date += relativedelta(months=1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|