git commit -m "Initial commit"

This commit is contained in:
2019-05-07 11:33:15 +08:00
commit 51b2b6baa7
30 changed files with 13620 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
/.idea
+110
View File
@@ -0,0 +1,110 @@
import pandas as pd
import common
from common import configs
from datetime import datetime
import numpy as np
from pymongo import UpdateOne
from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED
def cal(x):
try:
return int(x * 10000)
except Exception as ex:
print(ex)
return 0
district_stat = common.get_mongodb_collection(configs.MONGO_URI, "fangjia_stat", "district_stat")
fill_command = common.get_mongodb_collection(configs.LOG_MONGO_URI, "fangjia_log", "fill_command")
def padding(city):
latest_week_query = {
'weekend': common.get_last_weekend(),
'city': city
}
count = 1
latest_stats = list(district_stat.find(latest_week_query).batch_size(500))
for latest_stat in latest_stats:
print(city + ":" + str(count))
count = count + 1
query = {
'city': latest_stat["city"],
'region': latest_stat["region"],
'district_name': latest_stat["district_name"],
'estate_type': latest_stat["estate_type"]
}
start_date = list(district_stat.find(query).sort([("weekend", 1)]).limit(1))[0]["weekend"]
end_date = list(district_stat.find(query).sort([("weekend", -1)]).limit(1))[0]["weekend"]
weekends = pd.date_range(start_date, end_date, freq='7D')
if len(weekends) < 3:
continue
if len(weekends) == district_stat.count(query):
continue
stats = pd.DataFrame(list(district_stat.find(query).sort([("weekend", 1)])))
stats = stats.set_index('weekend', drop=False)
isUpdate = False
update_count = 0
outer = False
if len(weekends) != len(stats["weekend"]) and len(weekends) > 3:
miss_weekends = [i for i in weekends if i not in list(stats["weekend"])]
for weekend in miss_weekends:
stats = stats.append(pd.Series({"weekend": weekend, "value_type": 10}, name=weekend))
stats.sort_index(inplace=True)
min_avg_price = int(min(stats["avg_price"]))
max_avg_price = int(max(stats["avg_price"]))
try:
stats["avg_price"].interpolate(method='polynomial', order=3, inplace=True, limit_area='inside')
except Exception as ex:
print(str(ex))
print(latest_stat)
continue
for other_field in ["city", "region", "block", "estate_type", "district_name"]:
if latest_stat[other_field] is not None:
stats[other_field].fillna(latest_stat[other_field], inplace=True)
stats["credit"].fillna(0, inplace=True)
stats['new'] = (stats["avg_price"] - stats.shift(1)["avg_price"]) / stats.shift(1)["avg_price"]
stats['new'].fillna(0, inplace=True)
stats['new'] = stats['new'].apply(lambda x: cal(x))
stats['weekend_markup'].fillna(0, inplace=True)
requests = []
for index, row in stats.iterrows():
if row["_id"] is np.nan or row["_id"] is None or row["new"] != int(row["weekend_markup"]):
update = {
"city": row["city"],
"region": row["region"],
"district_name": row["district_name"],
"estate_type": row["estate_type"],
"weekend": row["weekend"]
}
data = {
"credit": row["credit"],
"avg_price": int(row["avg_price"]),
"m_date": datetime.utcnow(),
"weekend_markup": int(row["new"])
}
if row["_id"] is np.nan or row["_id"] is None:
data["c_date"] = datetime.utcnow()
data["calc_value"] = int(row["avg_price"])
data["value_type"] = 8
data["flg"] = 1
requests.append(UpdateOne(update, {'$set': data}, True))
isUpdate = True
update_count = update_count + 1
if row["avg_price"] > max_avg_price or row["avg_price"] < min_avg_price:
outer = True
if isUpdate:
district_stat.bulk_write(requests)
print(latest_stat)
common.send_refitted_trend(latest_stat["city"], latest_stat["region"], latest_stat["district_name"],
latest_stat["estate_type"], latest_stat["weekend"])
latest_stat["update_count"] = update_count
latest_stat["outer"] = outer
fill_command.save(latest_stat)
executor = ThreadPoolExecutor(100)
all_task = [executor.submit(padding, (city)) for city in common.get_all_city()]
wait(all_task, return_when=ALL_COMPLETED)
print("main")
+46
View File
@@ -0,0 +1,46 @@
from common import configs
import common
import pandas as pd
import datetime
#
# def re_match(city, region, name):
# json = common.match(city, name, region)
# print(json)
# if json["credit"] > 0:
# return [json["city"], json["district"], json["name"], json["category"]]
# else:
# return ['', '', '', '']
#
#
open_uri_log = common.get_mongodb_collection(configs.SEAWATER_MONGO_URI, "business", "open_uri_log")
query = {
"user_id": "1007705",
"request_uri": "/property/detail",
"data_status": 1,
'c_date': {'$gt': datetime.datetime.strptime("20190301", "%Y%m%d")}
}
logs = list()
for log in open_uri_log.find(query):
result = log["res_map"]["result"]
logs.append({"name": result["name"], "city": result["city"], "district": result["district"], "avgPrice": result["avgPrice"]})
df = pd.DataFrame(logs)
df.drop_duplicates()
df.to_excel("D:\\亲邻均价.xlsx")
# # logs[["输入城市", "输入区域", "输入名称", 'c_date']] = logs.apply(lambda x:
# # [x["parameter_map"]["city"], x["parameter_map"]["district"],
# # x["parameter_map"]["name"], x['c_date'].strftime("%Y-%m-%d %H:%M:%S")], axis=1, result_type="expand")
# # logs[["匹配城市", "匹配区域", "匹配名称", "匹配类型"]] = logs.apply(lambda x: re_match(x['输入城市'], x['输入区域'], x['输入名称']),
# # axis=1, result_type="expand")
# # del logs['res_map']
# # del logs['parameter_map']
# # del logs['code']
# # del logs['data_status']
# # del logs['ip']
# # del logs['request_uri']
# # del logs['msg']
# # logs.to_excel("D:\\亲邻未匹配数据.xlsx", index=False)
# logs = pd.read_excel("D:\\亲邻未匹配数据.xlsx")
# logs.drop_duplicates(["输入城市", "输入区域", "输入名称"], keep='first', inplace=True)
# logs.to_excel("D:\\亲邻未匹配数据1.xlsx")
+261
View File
@@ -0,0 +1,261 @@
from pymongo import MongoClient
from bson.codec_options import CodecOptions
import datetime
from common import configs
import requests
import common
import pytz
from elasticsearch import Elasticsearch
from elasticsearch import helpers
import pandas as pd
from retrying import retry
from geojson_utils import destination_point
from sklearn.cluster import KMeans
from sklearn import metrics
import numpy as np
es = Elasticsearch([configs.SEAWATER_ES_HOST], http_auth=configs.SEAWATER_ES_HTTP_AUTH, scheme="http",
port=9200, retry_on_timeout=True, timeout=60000)
def find_house(query, index, limit=None):
houses = list()
for doc in helpers.scan(es, query=query, scroll="10m", index=index, doc_type="_doc"):
doc = doc["_source"]
if isinstance(doc["p_date"], str):
doc["p_date"] = int(doc["p_date"])
houses.append(doc)
if limit is None and limit == len(houses):
break
return houses
@retry(stop_max_attempt_number=10, wait_fixed=5000)
def load_house_data(city=None, region=None, name=None, start_date=None, end_date=None, cat=None,
rent_type=None, alias=None, limit=None, room=None):
source = ["city", "region", "district_name", "direction", "fitment", "total_price", "room", "p_date", "area",
"avg_price", "root"]
must = list()
if cat is not None:
must.append({"term": {"cat": cat}})
if rent_type is not None:
must.append({"term": {"rent_type": rent_type}})
if city is not None:
must.append({"term": {"city": city}})
if region is not None:
must.append({"term": {"region": region}})
names = list()
if name is not None:
names.append(name)
if alias is not None and len(alias) > 0:
for alia in alias:
if alia is not None and alia not in names:
names.append(alia)
if len(names) > 0:
must.append({"terms": {"district_name": names}})
if start_date is not None or end_date is not None:
publish_date_range = {}
if start_date is not None:
publish_date_range["gt"] = start_date
if end_date is not None:
publish_date_range["lt"] = end_date
must.append({"range": {"p_date": publish_date_range}})
if room:
must.append({"term": {"room": room}})
else:
must.append({"range": {"room": {"gte": 1, "lt": 4}}})
must.append({"range": {"area": {"gte": 20, "lte": 260}}})
must.append({"range": {"avg_price": {"gte": 1}}})
query = {
"_source": source,
"query": {
"bool": {
"must": must
}
},
"sort": [
{
"publish_date": "DESC"
}
]
}
return pd.DataFrame(find_house(query, "latest_seawater", limit))
def get_last_weekend():
sunday = datetime.datetime.utcnow()
sunday = sunday.replace(hour=0, minute=0, second=0, microsecond=0)
one_day = datetime.timedelta(days=1)
while sunday.weekday() != 6:
sunday -= one_day
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(sunday)
def get_mongodb_collection(uri, db, collection):
return MongoClient(uri).get_database(db).get_collection(collection) \
.with_options(codec_options=CodecOptions(tz_aware=True, tzinfo=pytz.timezone('Asia/Shanghai')))
COMMAND = get_mongodb_collection(configs.LOG_MONGO_URI, "fangjia_log", "calc_command")
SERVER_AREA = get_mongodb_collection(configs.MONGO_URI, "fangjia", "server_area")
DISTRICT_STAT = get_mongodb_collection(configs.MONGO_URI, "fangjia_stat", "district_stat")
REGION_STAT = get_mongodb_collection(configs.MONGO_URI, "fangjia", "region_stat")
REGION_BLOCK = get_mongodb_collection(configs.MONGO_URI, "fangjia", "region_block")
SEAWEED = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia", "seaweed")
def send_refitted_trend(city, region, name, estate_type, weekend):
cmd = {
"city": city,
"region": region,
"name": name,
"estate_type": estate_type,
"calc_date": weekend,
"calc_type": 0,
"category": "sell",
"priority": 3,
"status": 102,
"remark": "week",
"redraw_trend_type": 1,
"ip": "192.168.0.24",
"c_date": datetime.utcnow()
}
COMMAND.save(cmd)
def get_least_district_stat(city, region, name, estate_type, start_date=None, end_date=None, limit=1):
if estate_type is not None and "别墅" in estate_type:
estate_type = "别墅"
else:
estate_type = "小区"
query = {
"city": city,
"region": region,
"district_name": name,
"estate_type": estate_type
}
weekend = {}
if start_date is not None:
weekend["$gte"] = start_date
if end_date is not None:
weekend["$lte"] = end_date
if len(weekend) != 0:
query["weekend"] = weekend
stats = list(DISTRICT_STAT.find(query).sort([("weekend", -1)]).limit(limit))
return stats
def get_least_region_stat(city, region=None, start_date=None, end_date=None, limit=1):
query = {
"city": city,
"region": region,
"block": None
}
weekend = {}
if start_date is not None:
weekend["$gte"] = start_date
if end_date is not None:
weekend["$lte"] = end_date
if len(weekend) != 0:
query["weekend"] = weekend
stats = list(REGION_STAT.find(query).sort([("weekend", -1)]).limit(limit))
return stats
def get_all_city():
return [doc["display"] for doc in SERVER_AREA.find({"display": {"$ne": "全国"}})]
def get_regions(city):
return [region["name"] for region in list(REGION_BLOCK.find({"city": city, "category": "region"}))]
def get_seaweed(city, region=None, name=None):
query = {"cat": "district", "status": 0, "city": city}
if region is not None:
query["region"] = region
if name is not None:
query["name"] = name
keys = {"city": 1, "region": 1, "name": 1, "alias": 1, "estate_type2": 1, "lng2": 1, "lat2": 1,
"finish_building_date": 1}
return list(SEAWEED.find(query, keys))
def get_nearby_seaweed(city, lng, lat, radius):
query = {"cat": "district", "status": 0, "city": city}
box = get_geo_box(lng=lng/10000000000, lat=lat/10000000000, radius=radius)
query["lat2"] = {"$gte": box["min_lat"] * 10000000000, "$lte": box["max_lat"] * 10000000000}
query["lng2"] = {"$gte": box["min_lng"] * 10000000000, "$lte": box["max_lng"] * 10000000000}
keys = {"city": 1, "region": 1, "name": 1, "alias": 1, "estate_type2": 1, "lng2": 1, "lat2": 1,
"finish_building_date": 1}
return list(SEAWEED.find(query, keys))
def match(city, address, region=""):
params = {
"city": city,
"address": address,
"region": region,
"token": "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E\
05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
}
return requests.get("http://open.fangjia.com/address/match", params).json()["result"]
def parse_date(time, fm='%Y-%m-%d'):
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(datetime.datetime.strptime(time, fm))
def localize(time):
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(time)
def format_date(time, fm='%Y%m%d'):
return time.strftime(fm)
def get_geo_box(lng, lat, radius):
central_point = {"type": "Point", "coordinates": [lng, lat]}
right_point = destination_point(central_point, 45, radius/1000)
left_point = destination_point(central_point, 225, radius/1000)
return {"max_lng": right_point["coordinates"][0],
"min_lng": left_point["coordinates"][0],
"min_lat": left_point["coordinates"][1],
"max_lat": right_point["coordinates"][1]}
def k_means(series):
values = np.array(list(series))
length = len(set(series))
if length < 2:
return values.mean()
values.sort()
length = len(values)
if length > 10:
values = values[int(length * 0.2):int(length * 0.8)]
length = len(set(series))
if length <= 2:
return values.mean()
cluster = None
max_score = -1
values = np.array(list(series))
values = values.reshape(len(values), 1)
for n_clusters in range(2, min(len(set(series)), 6)):
_cluster = KMeans(n_clusters=n_clusters, max_iter=10000).fit(values)
score = metrics.silhouette_score(values, _cluster.predict(values), metric='euclidean')
if score > max_score:
cluster = _cluster
max_score = score
cluster_centers = list(cluster.cluster_centers_)
labels_ = cluster.labels_
labels_count = dict()
for label in labels_:
if label in labels_count:
labels_count[label] = labels_count[label] + 1
else:
labels_count[label] = 1
max_cluster_label = max(labels_count.items(), key=lambda x: x[1])[0]
return cluster_centers[max_cluster_label][0]
Binary file not shown.
Binary file not shown.
+24
View File
@@ -0,0 +1,24 @@
# 日志数据库
LOG_MONGO_URI = 'mongodb://192.168.0.42:52002/'
# Seawater数据库
SEAWATER_MONGO_URI = 'mongodb://192.168.0.206:27020/'
# FangJia 数据库
MONGO_URI = 'mongodb://developer:G00IDCDEV12385@192.168.0.43:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
# OpenAPI测试环境数据库地址
TEST_URI = 'mongodb://admin:G00idcMongo85Admin@192.168.0.39:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
# 小区库地址
SEAWEED_URI = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
# ElasticSearch地址
SEAWATER_ES_HOST = '192.168.0.206'
SEAWATER_ES_HTTP_AUTH = ('elastic', 'goojia123456')
# 城市租金样本最小选取量
CITY_RENT_MIN_HOUSE_COUNT = 500
# 区域租金样本最小选取量
REGION_RENT_MIN_HOUSE_COUNT = 10
+24
View File
@@ -0,0 +1,24 @@
# -*- coding: UTF-8 -*-
class Seawater(object):
"""房源实体类"""
def __init__(self, city, region, district_name, area, avg_price, total_price, title,
room, root, url, p_date, fitment, direction, cat, rent_type):
self.city = city
self.region = region
self.district_name = district_name
self.area = area
self.avg_price = avg_price
self.total_price = total_price
self.title = title
self.root = root
self.room = room
self.url = url
self.p_date = p_date
self.fitment = fitment
self.direction = direction
self.cat = cat
self.rent_type = rent_type
Binary file not shown.
File diff suppressed because it is too large Load Diff
+23
View File
@@ -0,0 +1,23 @@
room,total_price
1,7000
1,8500
1,7500
1,6600
2,14000
2,14000
2,12000
2,8200
2,8200
2,14000
2,12000
2,7500
2,8500
2,12000
2,10500
2,8200
2,8000
2,8500
3,14900
3,12000
3,20000
3,12000
1 room total_price
2 1 7000
3 1 8500
4 1 7500
5 1 6600
6 2 14000
7 2 14000
8 2 12000
9 2 8200
10 2 8200
11 2 14000
12 2 12000
13 2 7500
14 2 8500
15 2 12000
16 2 10500
17 2 8200
18 2 8000
19 2 8500
20 3 14900
21 3 12000
22 3 20000
23 3 12000
View File
View File
+71
View File
@@ -0,0 +1,71 @@
import pandas as pd
from common import configs as cfg
import common
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
HOUSE_RENT = common.get_mongodb_collection("mongodb://192.168.0.105:27018/", "vip_site", "esf_houseRent")
DISTRICT_STAT_RENT = common.get_mongodb_collection(cfg.MONGO_URI, "fangjia", "district_stat_rent")
DISTRICT_RENT = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
def find_room_rent(city, region, name, room):
stat = DISTRICT_STAT_RENT.find_one({"city": city, "region": region, "district_name": name, "house_type": room,
"end_month_date": common.parse_date("2019-03-31")})
if stat:
return stat["avg_price"]
else:
return None
def find_sq_rent(city, region, name, room):
stat = DISTRICT_RENT.find_one({"city": city, "region": region, "name": name, "room": room, "date": 201902})
if stat:
return stat["rent_price"]
else:
return None
def find_rent_house(city, region, alias, room):
fields_filter = {"address": 0, "_id": 0, "block": 0, "c_date": 0, "title": 0, "m_date": 0, "rent_type": 0}
houses = pd.DataFrame(list(HOUSE_RENT.find({"city": city, "region": region, "rent_type": "整租",
"district_name": {"$in": alias}, "room": room, "publish_date": {"$gt": common.parse_date("2019-01-01")}}, fields_filter)))
if len(houses) == 0:
return houses
houses["avg_price"] = houses["total_price"] / houses["area"]
houses["avg_price"] = houses["avg_price"].map(int)
houses.drop_duplicates(keep='first', inplace=True)
houses.sort_values(by="area", inplace=True)
return houses
regions = ["朝阳", "海淀", "丰台", "西城", "通州", "石景山", "东城"]
def process():
all_house = pd.DataFrame()
for region in regions:
for seaweed in common.get_seaweed("北京", region):
names = [seaweed["name"]]
if "alias" in seaweed and seaweed["alias"] is not None:
names += seaweed["alias"]
for room in range(1, 3):
houses = find_rent_house(seaweed["city"], seaweed["region"], names, room)
if len(houses) > 1:
all_house = all_house.append(houses)
all_house["publish_date"] = all_house["publish_date"].apply(lambda x: common.format_date(x, '%Y%m%d'))
for col in ["direction", "fitment", "floor_info", "hall", "height", "toilet", "avg_price", "publish_date"]:
del all_house[col]
excel_data = []
for index, row in all_house.iterrows():
row = dict(row)
row["户型租金"] = find_room_rent(row["city"], row["region"], row["district_name"], row["room"])
row["平米租金"] = find_sq_rent(row["city"], row["region"], row["district_name"], row["room"])
print(row)
excel_data.append(row)
excel_data = pd.DataFrame(excel_data)
excel_data = excel_data.dropna(how="any")
excel_data.to_excel("D:\\建行户型房源数据.xlsx", index=False)
process()
+38
View File
@@ -0,0 +1,38 @@
import numpy as np
import pandas as pd
from sklearn import preprocessing
from sklearn import svm
from sklearn.model_selection import GridSearchCV, train_test_split
from sklearn.pipeline import Pipeline
import common
from sklearn.preprocessing import PolynomialFeatures
base_year = common.parse_date("1900-01-01")
def years(now):
year = common.parse_date(str(now), '%Y')
return int((year - base_year).days/365)
train_data = pd.read_csv(open('../data/小区租金样本.csv'))
train_data["build_year"] = train_data["build_year"].apply(lambda param: years(param))
x = train_data[["lat", "lng", "build_year", "room", "avg_price"]]
y = train_data[["rent_price"]]
x = x.astype(np.float64)
y = y.astype(np.float64)
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.2)
c_range = np.logspace(-5, 15, 11, base=2)
gamma_range = np.logspace(-9, 3, 13, base=2)
param_grid = [{'svr__kernel': ['rbf', 'linear'], 'svr__C': c_range, 'svr__gamma': gamma_range}]
svr = svm.SVR()
pipeline = Pipeline([('poly', PolynomialFeatures(3)),
('standard', preprocessing.StandardScaler()),
('minMax', preprocessing.MinMaxScaler()),
('svr', svr)])
grid = GridSearchCV(pipeline, param_grid, cv=3, n_jobs=10)
clf = grid.fit(x_train, y_train.values.ravel())
test_score = grid.score(x_test, y_test)
print('测试得分是%s' % test_score)
print('训练得分是%s' % grid.score(x_train, y_train))
+176
View File
@@ -0,0 +1,176 @@
from sklearn import svm
import re
import pandas as pd
import warnings
import numpy as np
from dateutil.relativedelta import relativedelta
from sklearn.model_selection import GridSearchCV, train_test_split
from sklearn.pipeline import Pipeline
from sklearn import preprocessing
import datetime
import os
import sys
curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)
import common
from common import configs as cfg
warnings.simplefilter("ignore")
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
district_room_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_room_rent")
district_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
district_sell_stat = common.get_mongodb_collection(cfg.MONGO_URI, "fangjia_stat", "district_sell_stat")
def has_geo_info(seaweed):
return "lat2" in seaweed and "lng2" in seaweed and seaweed["lng2"] > 0
def get_build_year(seaweed):
if "finish_building_date" not in seaweed or seaweed["finish_building_date"] is None:
return None
pattern = re.compile(r'\d{4}')
match = pattern.match(str(seaweed["finish_building_date"]))
if not match:
return None
return int(match.group())
def is_common_community(seaweed):
return seaweed["estate_type2"] == "普通住宅"
def get_apply_seaweed(city, year_month):
stats = list(district_sell_stat.find({"city": city, "date": year_month}))
stat_dict = dict()
for stat in stats:
key = stat["city"] + "_" + stat["region"] + "_" + stat["district_name"]
stat_dict[key] = stat["avg_price"]
seaweed_list = list()
for seaweed in common.get_seaweed(city):
if not has_geo_info(seaweed):
continue
if not is_common_community(seaweed):
continue
build_year = get_build_year(seaweed)
if not build_year:
continue
key = seaweed["city"] + "_" + seaweed["region"] + "_" + seaweed["name"]
if key not in stat_dict:
continue
seaweed["build_year"] = build_year
seaweed["lat"] = seaweed["lat2"] / 10000000000
seaweed["lng"] = seaweed["lng2"] / 10000000000
seaweed["avg_price"] = stat_dict[key]
seaweed_list.append(seaweed)
return seaweed_list
def get_rent_table_by_calc_type(calc_type):
if calc_type == "ht":
return district_room_rent
elif calc_type == "sq":
return district_rent
else:
return None
def load_train_data(city, year_month, calc_type):
"""
:param city:
:param year_month:
:param calc_type:计算类型 ht:户型租金 sq: 平米租金
:return:
"""
table_name = get_rent_table_by_calc_type(calc_type)
data = list()
for seaweed in get_apply_seaweed(city, year_month):
query = {"city": seaweed["city"], "region": seaweed["region"], "name": seaweed["name"], "date": year_month,
"calc_model": {"$ne": "svr"}}
stats = list(table_name.find(query))
if len(stats) < 3:
continue
for stat in stats:
item = seaweed.copy()
item["room"] = stat["room"]
item["rent_price"] = stat["rent_price"]
data.append(item)
print(item)
return pd.DataFrame(data)
def has_rent_price(seaweed, room, year_month, calc_type):
table_name = get_rent_table_by_calc_type(calc_type)
query = {"city": seaweed["city"], "region": seaweed["region"], "name": seaweed["name"],
"date": year_month, "room": room}
return table_name.count(query) > 0
def train(city, year_month, calc_type):
"""
:param city:
:param year_month:
:param calc_type:计算类型 ht:户型租金 sq: 平米租金
:return:
"""
train_data = load_train_data(city, year_month, calc_type)
x = train_data[["lat", "lng", "build_year", "room", "avg_price"]]
y = train_data[["rent_price"]]
x = x.astype(np.float64)
y = y.astype(np.float64)
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.3)
c_range = np.logspace(-5, 15, 11, base=2)
gamma_range = np.logspace(-9, 3, 13, base=2)
param_grid = [{'svr__kernel': ['rbf', 'linear'], 'svr__C': c_range, 'svr__gamma': gamma_range}]
svr = svm.SVR()
pipeline = Pipeline([('standard', preprocessing.StandardScaler()),
('minMax', preprocessing.MinMaxScaler()),
('svr', svr)])
grid = GridSearchCV(pipeline, param_grid, cv=3, n_jobs=10)
clf = grid.fit(x_train, y_train.values.ravel())
score = grid.score(x_test, y_test)
print('精度为%s' % score)
predict_seaweed = list()
for seaweed in get_apply_seaweed(city, year_month):
for room in range(1, 4):
if not has_rent_price(seaweed, room, year_month, calc_type):
item = seaweed.copy()
item["room"] = room
predict_seaweed.append(item)
predict_df = pd.DataFrame(predict_seaweed)
result = clf.predict(predict_df[["lat", "lng", "build_year", "room", "avg_price"]])
result = pd.DataFrame([[int(val)] for val in result])
result.columns = ["rent"]
predict_df = predict_df.join(result)
predict_df = predict_df[["city", "region", "name", "room", "rent"]]
table_name = get_rent_table_by_calc_type(calc_type)
for index, row in predict_df.iterrows():
if row["rent"] <= 0:
continue
query = {
"date": year_month,
"city": row["city"],
"region": row["region"],
"name": row["name"],
"room": int(row["room"])
}
data = dict()
data["m_date"] = datetime.datetime.utcnow()
data["rent_price"] = row["rent"]
data["calc_model"] = "svr"
table_name.update_one(query, {"$set": data}, True)
print(row)
if __name__ == "__main__":
start_date = common.parse_date("2019-05-01")
now_date = common.localize(datetime.datetime.utcnow())
while start_date < now_date:
year_month = int(common.format_date(start_date - relativedelta(months=1), '%Y%m'))
for city in ["北京"]:
train("北京", year_month, "ht")
train("北京", year_month, "sq")
start_date += relativedelta(months=1)
+228
View File
@@ -0,0 +1,228 @@
import calendar
import datetime
import os
import sys
import warnings
from math import isnan
import pandas as pd
from dateutil.relativedelta import relativedelta
from joblib import load
curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)
import common
from common import configs as cfg
warnings.simplefilter("ignore")
def k_means(series):
return common.k_means(series)
clf = load("C:\\Users\\wenyan\\PycharmProjects\\Sklearn\\data\\house_type_classify_model.joblib")
def house_type_filter(houses):
"""
通过模型过滤掉合租房源
:param houses:
:return:
"""
if len(houses) == 0:
return houses
X = houses[["room", "area"]]
houses["rent_type_predict"] = clf.predict(X)
houses["prob1"] = clf.predict_proba(X)[:, 1]
houses = houses[(houses.rent_type_predict == 1) & (houses.prob1 > 0.8)]
del houses["rent_type_predict"]
del houses["prob1"]
return houses
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
region_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "region_rent")
district_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
# 删除无用列
def remove_useless_columns(df):
for col in ["fitment", "direction", "root"]:
if col in df.columns:
del df[col]
def weighting(df):
if 'root' not in df.columns.values.tolist():
return df
root_weight = {
"zhuge.com": 3,
"ke.com": 8,
"lianjia.com": 8
}
weighting_data = list()
for index, row in df.iterrows():
row = dict(row)
weighting_data.append(row)
if row['root'] in root_weight:
for i in range(1, root_weight[row['root']]):
weighting_data.append(row)
print("org:{} dest:{}".format(len(df), len(weighting_data)))
return pd.DataFrame(weighting_data)
def get_start_date_and_end_date(year_month):
date = common.parse_date(str(year_month), '%Y%m')
year = date.year
month = date.month
first_day = datetime.datetime(year=year, month=month, day=1)
month_range = calendar.monthrange(year, month)
end_day = datetime.datetime(year=year, month=month, day=month_range[1])
return first_day, end_day
def calc_city_rent(_city, year_month):
start_date, end_date = get_start_date_and_end_date(year_month)
stats = common.get_least_region_stat(_city, None, start_date=start_date, end_date=end_date)
start_date = int(common.format_date(start_date, "%Y%m%d"))
end_date = int(common.format_date(end_date, "%Y%m%d"))
houses = common.load_house_data(city=_city, region=None, name=None, cat="rent", rent_type="整租",
start_date=start_date, end_date=end_date)
houses = house_type_filter(houses)
houses = weighting(houses)
remove_useless_columns(houses)
houses.dropna(axis=0, how='any', inplace=True)
houses = houses[houses.avg_price > 0]
houses.sort_values(by='p_date', ascending=False, inplace=True)
agg = houses.groupby(by="room")[['avg_price']].agg(['mean', 'median', 'count', k_means, 'min', 'max'])
agg = agg.applymap(int)
agg.columns = ["mean", "median", "count", "k_means", "min", "max"]
for index, row in agg.iterrows():
query = {
"date": year_month,
"cat": "city",
"city": _city,
"room": int(index)
}
data = dict(row)
for k, v in data.items():
data[k] = int(v)
data["m_date"] = datetime.datetime.utcnow()
if data["count"] > cfg.CITY_RENT_MIN_HOUSE_COUNT:
data["rent_price"] = data["k_means"]
if len(stats) > 0:
data["avg_price"] = stats[0]["avg_price"]
data["price_rent_ratio"] = int(data["avg_price"] / data["k_means"])
region_rent.update_one(query, {"$set": data}, True)
def calc_region_rent(_city, _region, year_month):
start_date, end_date = get_start_date_and_end_date(year_month)
stats = common.get_least_region_stat(city=_city, region=_region, start_date=start_date, end_date=end_date, limit=1)
start_date = int(common.format_date(start_date, "%Y%m%d"))
end_date = int(common.format_date(end_date, "%Y%m%d"))
houses = common.load_house_data(city=_city, region=_region, name=None, cat="rent", rent_type="整租",
start_date=start_date, end_date=end_date)
houses = house_type_filter(houses)
houses = weighting(houses)
remove_useless_columns(houses)
houses.dropna(axis=0, how='any', inplace=True)
if len(houses) == 0:
return
houses.sort_values(by='p_date', ascending=False, inplace=True)
agg = houses.groupby(by="room")[['avg_price']].agg(['mean', 'median', 'count', k_means])
agg = agg.applymap(int)
agg.columns = ["mean", "median", "count", "k_means"]
city_rent = dict()
for item in list(region_rent.find({"cat": "city", "city": _city, "date": year_month})):
city_rent[item["room"]] = item["price_rent_ratio"]
for index, row in agg.iterrows():
query = {
"date": year_month,
"cat": "region",
"region": _region,
"city": _city,
"room": int(index)
}
data = dict(row)
for k, v in data.items():
data[k] = int(v)
data["m_date"] = datetime.datetime.utcnow()
if data["count"] > cfg.REGION_RENT_MIN_HOUSE_COUNT:
if len(stats) == 0:
data["rent_price"] = data["k_means"]
else:
region_price = stats[0]["avg_price"]
data["avg_price"] = region_price
data["rent_price"] = data["k_means"]
data["price_rent_ratio"] = int(region_price / data["k_means"])
region_rent.update_one(query, {"$set": data}, True)
def filter_by_price_rent(houses, region_price):
if region_price is None or region_price == 0:
return houses
filtered_houses = list()
for index, row in houses.iterrows():
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3:
continue
if isnan(row["avg_price"]):
row["avg_price"] = int(row["total_price"] / row["area"])
house_price_rent_ratio = int(region_price / row["avg_price"])
if 100 < house_price_rent_ratio < 1500:
filtered_houses.append(row)
return pd.DataFrame(filtered_houses)
def filter_by_price_to_rent(houses, seaweed, year_month):
stats = common.get_least_district_stat(seaweed["city"], seaweed["region"], seaweed["name"], seaweed["estate_type2"]
, limit=1)
if len(stats) == 0:
return houses
seaweed_avg_price = stats[0]["avg_price"]
house_type_price_rent = dict()
for stat in list(region_rent.find({"city": seaweed["city"], "region": seaweed["region"], "date": year_month})):
if stat["cat"] == "city":
house_type_price_rent["city_{}".format(stat["room"])] = stat["price_rent_ratio"]
else:
house_type_price_rent["region_{}".format(stat["room"])] = stat["price_rent_ratio"]
filtered_houses = list()
for index, row in houses.iterrows():
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3:
continue
if isnan(row["avg_price"]):
row["avg_price"] = int(row["total_price"] / row["area"])
house_price_rent_ratio = int(seaweed_avg_price / row["avg_price"])
if "region_{}".format(row["room"]) in house_type_price_rent:
key = "region_{}".format(row["room"])
if house_type_price_rent[key] * 0.5 < house_price_rent_ratio < house_type_price_rent[key] * 2:
filtered_houses.append(row)
else:
print("过滤:" + str(row))
elif "city_{}".format(row["room"]) in house_type_price_rent:
key = "city_{}".format(row["room"])
if house_type_price_rent[key] * 0.25 < house_price_rent_ratio < house_type_price_rent[key] * 4:
filtered_houses.append(row)
else:
print("过滤:" + str(row))
else:
filtered_houses.append(row)
return pd.DataFrame(filtered_houses)
def main():
start_date = common.parse_date("2019-05-01")
now_date = common.localize(datetime.datetime.utcnow())
while start_date < now_date:
year_month = int(common.format_date(start_date - relativedelta(months=1), '%Y%m'))
for city in ["北京"]:
print("{}:{}\t开始计算".format(year_month, city))
calc_city_rent(city, year_month)
for region in common.get_regions(city):
print("{}:{} {}\t开始计算:".format(year_month, city, region))
calc_region_rent(city, region, year_month)
start_date += relativedelta(months=1)
if __name__ == "__main__":
main()
+261
View File
@@ -0,0 +1,261 @@
import calendar
import datetime
import multiprocessing
import os
import sys
import warnings
from math import isnan
from joblib import load
import pandas as pd
from dateutil.relativedelta import relativedelta
curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)
import common
warnings.simplefilter("ignore")
def k_means(series):
return common.k_means(series)
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
region_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "region_rent")
district_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
district_room_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_room_rent")
# 删除无用列
def remove_useless_columns(df):
for col in ["fitment", "direction", "root"]:
if col in df.columns:
del df[col]
def weighting(df):
root_weight = {
"zhuge.com": 2,
"5i5j.com": 3,
"ke.com": 8,
"lianjia.com": 8
}
weighting_data = list()
for index, row in df.iterrows():
row = dict(row)
weighting_data.append(row)
if 'root' in row and row['root'] in root_weight:
for i in range(1, root_weight[row['root']]):
weighting_data.append(row)
return pd.DataFrame(weighting_data)
def get_region_markup(_city, _region, _start, _end, _room):
_room = 2
start_stat = region_rent.find_one(
{"cat": "region", "city": _city, "region": _region, "date": _start, "room": _room})
end_stat = region_rent.find_one({"cat": "region", "city": _city, "region": _region, "date": _end, "room": _room})
if not start_stat or not end_stat:
return 0
return (end_stat["rent_price"] - start_stat["rent_price"]) / start_stat["rent_price"]
def correct_house(_houses, _year_month):
calc_year_month = common.parse_date(str(_year_month), '%Y%m')
for index, house in _houses.iterrows():
p_date = common.parse_date(str(house["p_date"]), '%Y%m%d')
days = (calc_year_month - p_date).days
if days > 180:
markup = get_region_markup(house["city"], house["region"], int(house["p_date"] / 100), _year_month, house["room"])
_houses.at[index, "avg_price"] = house["avg_price"] * (1 + markup)
_houses.at[index, "total_price"] = house["total_price"] * (1 + markup)
return _houses
def get_start_date_and_end_date(year_month):
date = common.parse_date(str(year_month), '%Y%m')
year = date.year
month = date.month
first_day = datetime.datetime(year=year, month=month, day=1)
month_range = calendar.monthrange(year, month)
end_day = datetime.datetime(year=year, month=month, day=month_range[1])
return first_day, end_day
def filter_by_price_rent(houses, region_price):
if region_rent is None:
return houses
filtered_houses = list()
for index, row in houses.iterrows():
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3:
continue
if isnan(row["avg_price"]):
row["avg_price"] = int(row["total_price"] / row["area"])
house_price_rent_ratio = int(region_price / row["avg_price"])
if 100 < house_price_rent_ratio < 1000:
filtered_houses.append(row)
return pd.DataFrame(filtered_houses)
def filter_by_price_to_rent(houses, seaweed, year_month):
stats = common.get_least_district_stat(seaweed["city"], seaweed["region"], seaweed["name"], seaweed["estate_type2"]
, limit=1)
if len(stats) == 0:
return houses
seaweed_avg_price = stats[0]["avg_price"]
house_type_price_rent = dict()
for stat in list(region_rent.find({"city": seaweed["city"], "region": seaweed["region"], "date": year_month})):
if stat["cat"] == "city":
house_type_price_rent["city_{}".format(stat["room"])] = stat["price_rent_ratio"]
else:
house_type_price_rent["region_{}".format(stat["room"])] = stat["price_rent_ratio"]
if len(house_type_price_rent) > 0:
house_type_price_rent["region"] = int(pd.Series(list(house_type_price_rent.values())).mean())
filtered_houses = list()
houses[["room", "total_price"]].to_csv("../data/房源.csv", index=False)
for index, row in houses.iterrows():
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3 or row["area"] == 0:
continue
if "avg_price" not in row or isnan(row["avg_price"]):
row["avg_price"] = int(row["total_price"] / row["area"])
if row["avg_price"] < 10:
continue
house_price_rent_ratio = int(seaweed_avg_price / row["avg_price"])
if "region_{}".format(row["room"]) in house_type_price_rent:
key = "region_{}".format(row["room"])
if house_type_price_rent[key] * 0.5 < house_price_rent_ratio < house_type_price_rent[key] * 2:
filtered_houses.append(row)
elif "city_{}".format(row["room"]) in house_type_price_rent:
key = "city_{}".format(row["room"])
if house_type_price_rent[key] * 0.25 < house_price_rent_ratio < house_type_price_rent[key] * 4:
filtered_houses.append(row)
elif "region" in house_type_price_rent:
key = "region"
if house_type_price_rent[key] * 0.4 < house_price_rent_ratio < house_type_price_rent[key] * 2:
filtered_houses.append(row)
else:
filtered_houses.append(row)
return pd.DataFrame(filtered_houses)
clf = load("C:\\Users\\wenyan\\PycharmProjects\\Sklearn\\data\\house_type_classify_model.joblib")
def house_type_filter(houses):
"""
通过模型过滤掉合租房源
:param houses:
:return:
"""
X = houses[["room", "area"]]
houses["rent_type_predict"] = clf.predict(X)
houses["prob1"] = clf.predict_proba(X)[:, 1]
houses = houses[(houses.rent_type_predict == 1) & (0.8 < houses.prob1) & (houses.prob1 < 0.99999)]
del houses["rent_type_predict"]
del houses["prob1"]
return houses
def drop_duplicates(houses):
houses["p_month"] = houses["p_date"].apply(lambda x: int(x/100))
houses = houses.drop_duplicates(subset=["area", "avg_price", "p_month", "room", "root", "total_price"])
del houses["p_month"]
return houses
def calc_seaweed_rent(seaweed, year_month):
print("开始计算小区租金{}: {} {} {}".format(year_month, seaweed["city"], seaweed["region"], seaweed["name"]))
start_date, end_date = get_start_date_and_end_date(year_month)
stats = common.get_least_district_stat(seaweed["city"], seaweed["region"], seaweed["name"], seaweed["estate_type2"]
, start_date=start_date, end_date=end_date, limit=1)
start_date = end_date - relativedelta(years=1)
first_date = end_date - relativedelta(months=1)
start_date = int(common.format_date(start_date, "%Y%m%d"))
first_date = int(common.format_date(first_date, "%Y%m%d"))
end_date = int(common.format_date(end_date, "%Y%m%d"))
alias = None
if alias in seaweed:
alias = seaweed["alias"]
houses = pd.DataFrame()
for house_type in range(1, 4):
_houses = common.load_house_data(city=seaweed["city"], region=seaweed["region"], name=seaweed["name"],
cat="rent", room=house_type, rent_type="整租", end_date=end_date,
start_date=first_date, alias=alias, limit=100)
if len(_houses) < 5:
_houses = common.load_house_data(city=seaweed["city"], region=seaweed["region"], name=seaweed["name"],
cat="rent", room=house_type,
rent_type="整租", end_date=end_date, start_date=start_date, alias=alias,
limit=100)
houses = houses.append(_houses, ignore_index=True)
if len(houses) == 0:
return
houses = drop_duplicates(houses)
houses = house_type_filter(houses)
houses = filter_by_price_to_rent(houses, seaweed, year_month)
houses = correct_house(houses, year_month)
houses = weighting(houses)
remove_useless_columns(houses)
houses.dropna(axis=0, how='any', inplace=True)
if len(houses) < 3:
return
if "room" not in houses.columns:
return
houses = houses[houses.avg_price > 0]
houses.sort_values(by='p_date', ascending=False, inplace=True)
agg = houses.groupby(by="room")[['avg_price']].agg(['mean', 'median', 'count', k_means])
total_price_agg = houses.groupby(by="room")[['total_price']].agg(['mean', 'median', 'count', k_means])
agg.columns = ["mean", "median", "count", "k_means"]
total_price_agg.columns = ["mean", "median", "count", "k_means"]
for index, row in agg.iterrows():
query = {
"date": year_month,
"city": seaweed["city"],
"region": seaweed["region"],
"name": seaweed["name"],
"room": int(index)
}
data = dict(row)
for k, v in data.items():
data[k] = int(v)
data["m_date"] = datetime.datetime.utcnow()
data["rent_price"] = data["k_means"]
if len(stats) != 0:
data["avg_price"] = stats[0]["avg_price"]
data["price_rent_ratio"] = int(data["avg_price"] / data["rent_price"])
district_rent.update_one(query, {"$set": data}, True)
for index, row in total_price_agg.iterrows():
query = {
"date": year_month,
"city": seaweed["city"],
"region": seaweed["region"],
"name": seaweed["name"],
"room": int(index)
}
data = dict(row)
for k, v in data.items():
data[k] = int(v)
data["m_date"] = datetime.datetime.utcnow()
data["rent_price"] = data["k_means"]
district_room_rent.update_one(query, {"$set": data}, True)
def main():
pool = multiprocessing.Pool(processes=30)
start_date = common.parse_date("2019-05-01")
now_date = common.localize(datetime.datetime.utcnow())
while start_date < now_date:
year_month = int(common.format_date(start_date - relativedelta(months=1), '%Y%m'))
for city in ["北京"]:
for community in common.get_seaweed(city):
pool.apply_async(calc_seaweed_rent, (community, year_month,))
start_date += relativedelta(months=1)
pool.close()
pool.join()
print("done")
if __name__ == "__main__":
calc_seaweed_rent(common.get_seaweed("北京", "西城", "富国里")[0], 201904)
# main()
View File
+33
View File
@@ -0,0 +1,33 @@
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from common import configs
import common
plt.rcParams['savefig.dpi'] = 400
plt.rcParams['figure.dpi'] = 400
plt.rcParams['figure.figsize'] = (12, 4)
plt.rcParams['font.sans-serif'] = ['SimHei']
region_rent = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia_rent", "region_rent")
price_rent = dict()
for region in common.get_regions("北京"):
df = pd.DataFrame(list(region_rent.find({"cat": "region", "region": region}, {"price_rent_ratio": 1})))
if len(df) > 0:
price_rent[region] = int(df["price_rent_ratio"].mean())
fig, ax = plt.subplots()
x = []
y = []
z = []
avg = np.array(list(price_rent.values())).mean()
for k, v in price_rent.items():
x.append(k)
y.append(v)
z.append(avg)
ax.bar(x, y, width=0.5)
ax.plot(x, y, label='区域售租比')
ax.plot(x, z, label='平均线:' + str(avg))
ax.set(xlabel='区域', ylabel='租售比', title='区域租售比分布图')
ax.grid()
plt.legend()
plt.savefig("D:\\建设银行\\租售比2000以内\\各区域租售比.png")
for item in price_rent.items():
print(str(item[0]) + "\t" + str(item[1]))
+12
View File
@@ -0,0 +1,12 @@
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import linear_model
df = pd.read_csv(open("C:\\Users\\wenyan\\PycharmProjects\\Sklearn\\data\\房源.csv"))
fig, ax = plt.subplots()
ax.scatter(df["room"], df["total_price"])
lr = linear_model.LinearRegression(normalize=True)
X = df["room"].values.reshape(len(df["room"]), 1)
Y = df["total_price"].values.ravel()
lr.fit(X, Y)
ax.plot([1, 2, 3], lr.predict([[1], [2], [3]]), 'r')
+17
View File
@@ -0,0 +1,17 @@
import matplotlib.pyplot as plt
import pandas as pd
import common
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
HOUSE_RENT = common.get_mongodb_collection("mongodb://192.168.0.105:27018/", "vip_site", "esf_houseRent")
DISTRICT_RENT = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_room_rent")
stats = list(DISTRICT_RENT.find({"city": "北京", "region": "朝阳", "name": "甜水园"}))
df = pd.DataFrame(stats)
df.dropna()
df = df[["city", "region", "name", "date", "room", "rent_price"]]
df["date"] = df["date"].apply(lambda x: common.parse_date(str(x), '%Y%m'))
room1_df = df[df["room"] == 2]
room1_df = room1_df.sort_values(by="date")
fig, ax = plt.subplots()
ax.plot(room1_df["date"], room1_df["rent_price"], 'h-.')
plt.show()
+36
View File
@@ -0,0 +1,36 @@
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from common import configs
import common
# region_rent = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia_rent", "region_rent")
# price_rent = dict()
# for region in common.get_regions("北京"):
# df = pd.DataFrame(list(region_rent.find({"cat": "region", "region": region}, {"price_rent_ratio": 1})))
# if len(df) > 0:
# price_rent[region] = int(df["price_rent_ratio"].mean())
# plt.rcParams['font.sans-serif'] = ['SimHei']
# district_rent = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia_rent", "district_rent")
# plt.rcParams['savefig.dpi'] = 800
# plt.rcParams['figure.dpi'] = 800
# plt.rcParams['figure.figsize'] = (6, 4)
# for region in common.get_regions("北京"):
# df = pd.DataFrame(list(district_rent.find({"region": region}, {"price_rent_ratio": 1})))
# ratio = np.array(df["price_rent_ratio"])
# result = pd.value_counts(ratio)
# result = result.sort_index()
# fig, ax = plt.subplots()
# ax.plot(result.index, result.values)
# if region in price_rent:
# y = np.arange(int(max(result.values) * 1.2))
# x = np.zeros(len(y)) + price_rent[region]
# ax.plot(x, y, label="区域售租比")
# ax.plot(x * 2, y, '--', label="2倍区域售租比")
# ax.plot(x * 0.4, y, '-', label="0.4倍区域售租比")
# ax.set(xlabel='售租比', ylabel='小区量', title=region + '区小区售租比分布图')
# ax.grid()
# plt.legend()
# plt.savefig("D:\\建设银行\\租售比\\" + region + "租售比分布分布图.png")
numbers = np.array([12000, 12000, 14900, 20000])
print(common.k_means(numbers))
+253
View File
@@ -0,0 +1,253 @@
absl-py==0.7.0
alabaster==0.7.10
alembic==1.0.3
anaconda-client==1.6.14
anaconda-navigator==1.9.6
anaconda-project==0.8.2
aniso8601==4.0.1
APScheduler==3.5.0
asn1crypto==0.24.0
astor==0.7.1
astroid==1.6.3
astropy==3.0.2
attrs==18.1.0
Babel==2.5.3
backcall==0.1.0
backports.shutil-get-terminal-size==1.0.0
beautifulsoup4==4.6.0
bitarray==0.8.1
bkcharts==0.2
blaze==0.11.3
bleach==2.1.3
blinker==1.4
bokeh==0.12.16
boto==2.48.0
Bottleneck==1.2.1
brotlipy==0.7.0
certifi==2019.3.9
cffi==1.11.5
chardet==3.0.4
click==6.7
cloudpickle==0.5.3
clyent==1.2.2
colorama==0.3.9
comtypes==1.1.4
conda==4.6.8
conda-build==3.10.5
conda-verify==2.0.0
contextlib2==0.5.5
coverage==3.7.1
cronmon==0.1.0
cryptography==2.6.1
cycler==0.10.0
Cython==0.28.2
cytoolz==0.9.0.1
dask==0.17.5
datashape==0.5.4
decorator==4.3.0
distributed==1.21.8
docutils==0.14
dogpile.cache==0.6.5
elasticsearch==6.3.1
entrypoints==0.2.3
et-xmlfile==1.0.1
eventlet==0.23.0
fastcache==1.0.2
filelock==3.0.4
Flask==0.12.2
Flask-APScheduler==1.7.1
Flask-Cors==3.0.4
flask-marshmallow==0.8.0
Flask-Migrate==2.3.0
Flask-RESTful==0.3.6
Flask-Script==2.0.6
Flask-SocketIO==3.0.1
Flask-SQLAlchemy==2.3.2
Flask-Testing==0.7.1
gast==0.2.2
geojson==2.4.1
geojson-utils==0.0.2
gevent==1.3.0
glob2==0.6
greenlet==0.4.13
grpcio==1.12.1
h11==0.7.0
h2==3.1.0
h5py==2.7.1
heapdict==1.0.0
hpack==3.0.0
html5lib==1.0.1
hyperframe==5.2.0
idna==2.6
imageio==2.3.0
imagesize==1.0.0
ipykernel==4.8.2
ipython==6.4.0
ipython-genutils==0.2.0
ipywidgets==7.2.1
isort==4.3.4
itsdangerous==0.24
jdcal==1.4
jedi==0.12.0
Jinja2==2.10
jmespath==0.9.3
jsonschema==2.6.0
jupyter==1.0.0
jupyter-client==5.2.3
jupyter-console==5.2.0
jupyter-core==4.4.0
jupyterlab==0.32.1
jupyterlab-launcher==0.10.5
kaitaistruct==0.8
Keras==2.2.4
Keras-Applications==1.0.6
Keras-Preprocessing==1.0.5
kiwisolver==1.0.1
lazy-object-proxy==1.3.1
ldap3==2.5.2
llvmlite==0.23.1
locket==0.2.0
lxml==4.2.1
Mako==1.0.7
Markdown==3.0.1
MarkupSafe==1.0
marshmallow==3.0.0b5
marshmallow-sqlalchemy==0.13.2
matplotlib==2.2.2
mccabe==0.6.1
menuinst==1.4.14
mistune==0.8.3
mitmproxy==4.0.4
mkl-fft==1.0.10
mkl-random==1.0.2
more-itertools==4.1.0
mpmath==1.0.0
msgpack-python==0.5.6
multipledispatch==0.5.0
navigator-updater==0.2.1
nbconvert==5.3.1
nbformat==4.4.0
networkx==2.1
nltk==3.3
nose==1.3.4
notebook==5.5.0
numba==0.38.0
numexpr==2.6.9
numpy==1.16.2
numpydoc==0.8.0
odo==0.5.1
olefile==0.45.1
openpyxl==2.5.3
packaging==17.1
pandas==0.23.0
pandocfilters==1.4.2
parso==0.2.0
partd==0.3.8
passlib==1.7.1
path.py==11.0.1
pathlib2==2.3.2
patsy==0.5.0
pep8==1.7.1
pickleshare==0.7.4
Pillow==5.1.0
pkginfo==1.4.2
pluggy==0.6.0
ply==3.11
prompt-toolkit==1.0.15
protobuf==3.6.0
psutil==5.4.5
py==1.5.3
pyasn1==0.4.5
pycodestyle==2.4.0
pycosat==0.6.3
pycparser==2.18
pycrypto==2.6.1
pycurl==7.43.0.2
pydivert==2.1.0
pyflakes==1.6.0
Pygments==2.2.0
pylint==1.8.4
pymongo==3.7.1
pyodbc==4.0.23
pyOpenSSL==18.0.0
pyparsing==2.2.0
pyperclip==1.6.5
pypinyin==0.33.2
PySocks==1.6.8
pytest==3.0.2
pytest-arraydiff==0.2
pytest-astropy==0.3.0
pytest-cov==2.3.1
pytest-doctestplus==0.1.3
pytest-openfiles==0.3.0
pytest-ordering==0.5
pytest-remotedata==0.2.1
python-dateutil==2.7.2
python-editor==1.0.3
python-engineio==2.2.0
python-socketio==2.0.0
pytz==2018.4
PyWavelets==0.5.2
pywin32==223
pywinpty==0.5.1
PyYAML==3.12
pyzmq==17.0.0
QtAwesome==0.4.4
qtconsole==4.3.1
QtPy==1.4.1
redis==2.10.6
redis-py-cluster==1.3.6
requests==2.18.4
retrying==1.3.3
rope==0.10.7
ruamel-yaml==0.15.35
ruamel.yaml==0.15.88
scikit-image==0.13.1
scikit-learn==0.20.3
scipy==1.2.1
seaborn==0.8.1
Send2Trash==1.5.0
simplegeneric==0.8.1
singledispatch==3.4.0.3
six==1.11.0
snowballstemmer==1.2.1
sortedcollections==0.6.1
sortedcontainers==1.5.10
Sphinx==1.7.4
sphinxcontrib-websupport==1.0.1
spyder==3.2.8
SQLAlchemy==1.2.0
statsmodels==0.9.0
sympy==1.1.1
tables==3.4.3
tblib==1.3.2
tensorboard==1.12.0
tensorflow==1.12.0
termcolor==1.1.0
terminado==0.8.1
testpath==0.3.1
toolz==0.9.0
tornado==5.0.2
traitlets==4.3.2
typing==3.6.4
tzlocal==1.5.1
unicodecsv==0.14.1
urllib3==1.22
urwid==2.0.1
wcwidth==0.1.7
webencodings==0.5.1
webtest==2.0.32
Werkzeug==0.14.1
widgetsnbextension==3.2.1
win-inet-pton==1.0.1
win-unicode-console==0.5
wincertstore==0.2
wrapt==1.10.11
wsproto==0.11.0
wtforms==2.2.1
xlrd==1.1.0
XlsxWriter==1.0.4
xlwings==0.11.8
xlwt==1.3.0
zict==0.1.3
joblib==0.13.2
+40
View File
@@ -0,0 +1,40 @@
import sys
import os
curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)
import common
from common import configs as cfg
from sync import utils
prod_district_stat = common.get_mongodb_collection(cfg.MONGO_URI, "fangjia_stat", "district_stat")
test_district_stat = common.get_mongodb_collection(cfg.TEST_URI, "fangjia_stat", "district_stat")
for index in prod_district_stat.list_indexes():
keys = list()
for key in index["key"]:
keys.append((key, index["key"][key]))
print(index)
if "unique" in index:
test_district_stat.create_index(keys, background=True, unique=index["unique"])
else:
test_district_stat.create_index(keys, background=True)
for seaweed in utils.get_all_test_seaweed():
sync_condition = {
"city": seaweed["city"],
"region": seaweed["region"],
"district_name": seaweed["name"],
"weekend": {"$gt": common.parse_date("2018-03-01")}
}
print(sync_condition)
total = prod_district_stat.count(sync_condition)
count = 0
for dbo in prod_district_stat.find(sync_condition, no_cursor_timeout=True).batch_size(1000):
count += 1
test_district_stat.save(dbo)
if count % 1000 == 0:
print(str(count) + "/" + str(total))
+31
View File
@@ -0,0 +1,31 @@
import common
from common import configs as cfg
prod_district_stat = common.get_mongodb_collection(cfg.MONGO_URI, "fangjia", "district_stat_rent")
test_district_stat = common.get_mongodb_collection(cfg.TEST_URI, "fangjia", "district_stat_rent")
for index in prod_district_stat.list_indexes():
keys = list()
for key in index["key"]:
keys.append((key, index["key"][key]))
print(index)
if "unique" in index:
test_district_stat.create_index(keys, background=True, unique=index["unique"])
else:
test_district_stat.create_index(keys, background=True)
sync_condition = {
"city": "上海",
"region": "长宁",
"end_month_date": {"$gt": common.parse_date("2018-03-01")}
}
total = prod_district_stat.count(sync_condition)
count = 0
for dbo in prod_district_stat.find(sync_condition, no_cursor_timeout=True).batch_size(1000):
count += 1
test_district_stat.save(dbo)
if count % 1000 == 0:
print(str(count) + "/" + str(total))
View File
Binary file not shown.
Binary file not shown.
+6
View File
@@ -0,0 +1,6 @@
import common
seaweed = common.get_mongodb_collection("mongodb://192.168.0.39:52002/", "fangjia", "seaweed")
def get_all_test_seaweed():
return list(seaweed.find())