git commit -m "Initial commit"
This commit is contained in:
@@ -0,0 +1 @@
|
||||
/.idea
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
import pandas as pd
|
||||
import common
|
||||
from common import configs
|
||||
from datetime import datetime
|
||||
import numpy as np
|
||||
from pymongo import UpdateOne
|
||||
from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED
|
||||
|
||||
|
||||
def cal(x):
|
||||
try:
|
||||
return int(x * 10000)
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
return 0
|
||||
|
||||
|
||||
district_stat = common.get_mongodb_collection(configs.MONGO_URI, "fangjia_stat", "district_stat")
|
||||
fill_command = common.get_mongodb_collection(configs.LOG_MONGO_URI, "fangjia_log", "fill_command")
|
||||
|
||||
|
||||
def padding(city):
|
||||
latest_week_query = {
|
||||
'weekend': common.get_last_weekend(),
|
||||
'city': city
|
||||
}
|
||||
count = 1
|
||||
latest_stats = list(district_stat.find(latest_week_query).batch_size(500))
|
||||
for latest_stat in latest_stats:
|
||||
print(city + ":" + str(count))
|
||||
count = count + 1
|
||||
query = {
|
||||
'city': latest_stat["city"],
|
||||
'region': latest_stat["region"],
|
||||
'district_name': latest_stat["district_name"],
|
||||
'estate_type': latest_stat["estate_type"]
|
||||
}
|
||||
start_date = list(district_stat.find(query).sort([("weekend", 1)]).limit(1))[0]["weekend"]
|
||||
end_date = list(district_stat.find(query).sort([("weekend", -1)]).limit(1))[0]["weekend"]
|
||||
weekends = pd.date_range(start_date, end_date, freq='7D')
|
||||
if len(weekends) < 3:
|
||||
continue
|
||||
if len(weekends) == district_stat.count(query):
|
||||
continue
|
||||
stats = pd.DataFrame(list(district_stat.find(query).sort([("weekend", 1)])))
|
||||
stats = stats.set_index('weekend', drop=False)
|
||||
isUpdate = False
|
||||
update_count = 0
|
||||
outer = False
|
||||
if len(weekends) != len(stats["weekend"]) and len(weekends) > 3:
|
||||
miss_weekends = [i for i in weekends if i not in list(stats["weekend"])]
|
||||
for weekend in miss_weekends:
|
||||
stats = stats.append(pd.Series({"weekend": weekend, "value_type": 10}, name=weekend))
|
||||
stats.sort_index(inplace=True)
|
||||
min_avg_price = int(min(stats["avg_price"]))
|
||||
max_avg_price = int(max(stats["avg_price"]))
|
||||
try:
|
||||
stats["avg_price"].interpolate(method='polynomial', order=3, inplace=True, limit_area='inside')
|
||||
except Exception as ex:
|
||||
print(str(ex))
|
||||
print(latest_stat)
|
||||
continue
|
||||
for other_field in ["city", "region", "block", "estate_type", "district_name"]:
|
||||
if latest_stat[other_field] is not None:
|
||||
stats[other_field].fillna(latest_stat[other_field], inplace=True)
|
||||
stats["credit"].fillna(0, inplace=True)
|
||||
stats['new'] = (stats["avg_price"] - stats.shift(1)["avg_price"]) / stats.shift(1)["avg_price"]
|
||||
stats['new'].fillna(0, inplace=True)
|
||||
stats['new'] = stats['new'].apply(lambda x: cal(x))
|
||||
stats['weekend_markup'].fillna(0, inplace=True)
|
||||
requests = []
|
||||
for index, row in stats.iterrows():
|
||||
if row["_id"] is np.nan or row["_id"] is None or row["new"] != int(row["weekend_markup"]):
|
||||
update = {
|
||||
"city": row["city"],
|
||||
"region": row["region"],
|
||||
"district_name": row["district_name"],
|
||||
"estate_type": row["estate_type"],
|
||||
"weekend": row["weekend"]
|
||||
}
|
||||
data = {
|
||||
"credit": row["credit"],
|
||||
"avg_price": int(row["avg_price"]),
|
||||
"m_date": datetime.utcnow(),
|
||||
"weekend_markup": int(row["new"])
|
||||
}
|
||||
if row["_id"] is np.nan or row["_id"] is None:
|
||||
data["c_date"] = datetime.utcnow()
|
||||
data["calc_value"] = int(row["avg_price"])
|
||||
data["value_type"] = 8
|
||||
data["flg"] = 1
|
||||
requests.append(UpdateOne(update, {'$set': data}, True))
|
||||
isUpdate = True
|
||||
update_count = update_count + 1
|
||||
if row["avg_price"] > max_avg_price or row["avg_price"] < min_avg_price:
|
||||
outer = True
|
||||
if isUpdate:
|
||||
district_stat.bulk_write(requests)
|
||||
print(latest_stat)
|
||||
common.send_refitted_trend(latest_stat["city"], latest_stat["region"], latest_stat["district_name"],
|
||||
latest_stat["estate_type"], latest_stat["weekend"])
|
||||
latest_stat["update_count"] = update_count
|
||||
latest_stat["outer"] = outer
|
||||
fill_command.save(latest_stat)
|
||||
|
||||
|
||||
executor = ThreadPoolExecutor(100)
|
||||
all_task = [executor.submit(padding, (city)) for city in common.get_all_city()]
|
||||
wait(all_task, return_when=ALL_COMPLETED)
|
||||
print("main")
|
||||
@@ -0,0 +1,46 @@
|
||||
from common import configs
|
||||
import common
|
||||
import pandas as pd
|
||||
import datetime
|
||||
|
||||
#
|
||||
# def re_match(city, region, name):
|
||||
# json = common.match(city, name, region)
|
||||
# print(json)
|
||||
# if json["credit"] > 0:
|
||||
# return [json["city"], json["district"], json["name"], json["category"]]
|
||||
# else:
|
||||
# return ['', '', '', '']
|
||||
#
|
||||
#
|
||||
open_uri_log = common.get_mongodb_collection(configs.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
query = {
|
||||
"user_id": "1007705",
|
||||
"request_uri": "/property/detail",
|
||||
"data_status": 1,
|
||||
'c_date': {'$gt': datetime.datetime.strptime("20190301", "%Y%m%d")}
|
||||
}
|
||||
logs = list()
|
||||
for log in open_uri_log.find(query):
|
||||
result = log["res_map"]["result"]
|
||||
logs.append({"name": result["name"], "city": result["city"], "district": result["district"], "avgPrice": result["avgPrice"]})
|
||||
df = pd.DataFrame(logs)
|
||||
df.drop_duplicates()
|
||||
df.to_excel("D:\\亲邻均价.xlsx")
|
||||
|
||||
# # logs[["输入城市", "输入区域", "输入名称", 'c_date']] = logs.apply(lambda x:
|
||||
# # [x["parameter_map"]["city"], x["parameter_map"]["district"],
|
||||
# # x["parameter_map"]["name"], x['c_date'].strftime("%Y-%m-%d %H:%M:%S")], axis=1, result_type="expand")
|
||||
# # logs[["匹配城市", "匹配区域", "匹配名称", "匹配类型"]] = logs.apply(lambda x: re_match(x['输入城市'], x['输入区域'], x['输入名称']),
|
||||
# # axis=1, result_type="expand")
|
||||
# # del logs['res_map']
|
||||
# # del logs['parameter_map']
|
||||
# # del logs['code']
|
||||
# # del logs['data_status']
|
||||
# # del logs['ip']
|
||||
# # del logs['request_uri']
|
||||
# # del logs['msg']
|
||||
# # logs.to_excel("D:\\亲邻未匹配数据.xlsx", index=False)
|
||||
# logs = pd.read_excel("D:\\亲邻未匹配数据.xlsx")
|
||||
# logs.drop_duplicates(["输入城市", "输入区域", "输入名称"], keep='first', inplace=True)
|
||||
# logs.to_excel("D:\\亲邻未匹配数据1.xlsx")
|
||||
@@ -0,0 +1,261 @@
|
||||
from pymongo import MongoClient
|
||||
from bson.codec_options import CodecOptions
|
||||
import datetime
|
||||
from common import configs
|
||||
import requests
|
||||
import common
|
||||
import pytz
|
||||
from elasticsearch import Elasticsearch
|
||||
from elasticsearch import helpers
|
||||
import pandas as pd
|
||||
from retrying import retry
|
||||
from geojson_utils import destination_point
|
||||
from sklearn.cluster import KMeans
|
||||
from sklearn import metrics
|
||||
import numpy as np
|
||||
|
||||
es = Elasticsearch([configs.SEAWATER_ES_HOST], http_auth=configs.SEAWATER_ES_HTTP_AUTH, scheme="http",
|
||||
port=9200, retry_on_timeout=True, timeout=60000)
|
||||
|
||||
|
||||
def find_house(query, index, limit=None):
|
||||
houses = list()
|
||||
for doc in helpers.scan(es, query=query, scroll="10m", index=index, doc_type="_doc"):
|
||||
doc = doc["_source"]
|
||||
if isinstance(doc["p_date"], str):
|
||||
doc["p_date"] = int(doc["p_date"])
|
||||
houses.append(doc)
|
||||
if limit is None and limit == len(houses):
|
||||
break
|
||||
return houses
|
||||
|
||||
|
||||
@retry(stop_max_attempt_number=10, wait_fixed=5000)
|
||||
def load_house_data(city=None, region=None, name=None, start_date=None, end_date=None, cat=None,
|
||||
rent_type=None, alias=None, limit=None, room=None):
|
||||
source = ["city", "region", "district_name", "direction", "fitment", "total_price", "room", "p_date", "area",
|
||||
"avg_price", "root"]
|
||||
must = list()
|
||||
if cat is not None:
|
||||
must.append({"term": {"cat": cat}})
|
||||
if rent_type is not None:
|
||||
must.append({"term": {"rent_type": rent_type}})
|
||||
if city is not None:
|
||||
must.append({"term": {"city": city}})
|
||||
if region is not None:
|
||||
must.append({"term": {"region": region}})
|
||||
names = list()
|
||||
if name is not None:
|
||||
names.append(name)
|
||||
if alias is not None and len(alias) > 0:
|
||||
for alia in alias:
|
||||
if alia is not None and alia not in names:
|
||||
names.append(alia)
|
||||
if len(names) > 0:
|
||||
must.append({"terms": {"district_name": names}})
|
||||
if start_date is not None or end_date is not None:
|
||||
publish_date_range = {}
|
||||
if start_date is not None:
|
||||
publish_date_range["gt"] = start_date
|
||||
if end_date is not None:
|
||||
publish_date_range["lt"] = end_date
|
||||
must.append({"range": {"p_date": publish_date_range}})
|
||||
if room:
|
||||
must.append({"term": {"room": room}})
|
||||
else:
|
||||
must.append({"range": {"room": {"gte": 1, "lt": 4}}})
|
||||
must.append({"range": {"area": {"gte": 20, "lte": 260}}})
|
||||
must.append({"range": {"avg_price": {"gte": 1}}})
|
||||
query = {
|
||||
"_source": source,
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": must
|
||||
}
|
||||
},
|
||||
"sort": [
|
||||
{
|
||||
"publish_date": "DESC"
|
||||
}
|
||||
]
|
||||
}
|
||||
return pd.DataFrame(find_house(query, "latest_seawater", limit))
|
||||
|
||||
|
||||
def get_last_weekend():
|
||||
sunday = datetime.datetime.utcnow()
|
||||
sunday = sunday.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
one_day = datetime.timedelta(days=1)
|
||||
while sunday.weekday() != 6:
|
||||
sunday -= one_day
|
||||
pacific = pytz.timezone('Asia/Shanghai')
|
||||
return pacific.localize(sunday)
|
||||
|
||||
|
||||
def get_mongodb_collection(uri, db, collection):
|
||||
return MongoClient(uri).get_database(db).get_collection(collection) \
|
||||
.with_options(codec_options=CodecOptions(tz_aware=True, tzinfo=pytz.timezone('Asia/Shanghai')))
|
||||
|
||||
|
||||
COMMAND = get_mongodb_collection(configs.LOG_MONGO_URI, "fangjia_log", "calc_command")
|
||||
SERVER_AREA = get_mongodb_collection(configs.MONGO_URI, "fangjia", "server_area")
|
||||
DISTRICT_STAT = get_mongodb_collection(configs.MONGO_URI, "fangjia_stat", "district_stat")
|
||||
REGION_STAT = get_mongodb_collection(configs.MONGO_URI, "fangjia", "region_stat")
|
||||
REGION_BLOCK = get_mongodb_collection(configs.MONGO_URI, "fangjia", "region_block")
|
||||
SEAWEED = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia", "seaweed")
|
||||
|
||||
|
||||
def send_refitted_trend(city, region, name, estate_type, weekend):
|
||||
cmd = {
|
||||
"city": city,
|
||||
"region": region,
|
||||
"name": name,
|
||||
"estate_type": estate_type,
|
||||
"calc_date": weekend,
|
||||
"calc_type": 0,
|
||||
"category": "sell",
|
||||
"priority": 3,
|
||||
"status": 102,
|
||||
"remark": "week",
|
||||
"redraw_trend_type": 1,
|
||||
"ip": "192.168.0.24",
|
||||
"c_date": datetime.utcnow()
|
||||
}
|
||||
COMMAND.save(cmd)
|
||||
|
||||
|
||||
def get_least_district_stat(city, region, name, estate_type, start_date=None, end_date=None, limit=1):
|
||||
if estate_type is not None and "别墅" in estate_type:
|
||||
estate_type = "别墅"
|
||||
else:
|
||||
estate_type = "小区"
|
||||
query = {
|
||||
"city": city,
|
||||
"region": region,
|
||||
"district_name": name,
|
||||
"estate_type": estate_type
|
||||
}
|
||||
weekend = {}
|
||||
if start_date is not None:
|
||||
weekend["$gte"] = start_date
|
||||
if end_date is not None:
|
||||
weekend["$lte"] = end_date
|
||||
if len(weekend) != 0:
|
||||
query["weekend"] = weekend
|
||||
stats = list(DISTRICT_STAT.find(query).sort([("weekend", -1)]).limit(limit))
|
||||
return stats
|
||||
|
||||
|
||||
def get_least_region_stat(city, region=None, start_date=None, end_date=None, limit=1):
|
||||
query = {
|
||||
"city": city,
|
||||
"region": region,
|
||||
"block": None
|
||||
}
|
||||
weekend = {}
|
||||
if start_date is not None:
|
||||
weekend["$gte"] = start_date
|
||||
if end_date is not None:
|
||||
weekend["$lte"] = end_date
|
||||
if len(weekend) != 0:
|
||||
query["weekend"] = weekend
|
||||
stats = list(REGION_STAT.find(query).sort([("weekend", -1)]).limit(limit))
|
||||
return stats
|
||||
|
||||
|
||||
def get_all_city():
|
||||
return [doc["display"] for doc in SERVER_AREA.find({"display": {"$ne": "全国"}})]
|
||||
|
||||
|
||||
def get_regions(city):
|
||||
return [region["name"] for region in list(REGION_BLOCK.find({"city": city, "category": "region"}))]
|
||||
|
||||
|
||||
def get_seaweed(city, region=None, name=None):
|
||||
query = {"cat": "district", "status": 0, "city": city}
|
||||
if region is not None:
|
||||
query["region"] = region
|
||||
if name is not None:
|
||||
query["name"] = name
|
||||
keys = {"city": 1, "region": 1, "name": 1, "alias": 1, "estate_type2": 1, "lng2": 1, "lat2": 1,
|
||||
"finish_building_date": 1}
|
||||
return list(SEAWEED.find(query, keys))
|
||||
|
||||
|
||||
def get_nearby_seaweed(city, lng, lat, radius):
|
||||
query = {"cat": "district", "status": 0, "city": city}
|
||||
box = get_geo_box(lng=lng/10000000000, lat=lat/10000000000, radius=radius)
|
||||
query["lat2"] = {"$gte": box["min_lat"] * 10000000000, "$lte": box["max_lat"] * 10000000000}
|
||||
query["lng2"] = {"$gte": box["min_lng"] * 10000000000, "$lte": box["max_lng"] * 10000000000}
|
||||
keys = {"city": 1, "region": 1, "name": 1, "alias": 1, "estate_type2": 1, "lng2": 1, "lat2": 1,
|
||||
"finish_building_date": 1}
|
||||
return list(SEAWEED.find(query, keys))
|
||||
|
||||
|
||||
def match(city, address, region=""):
|
||||
params = {
|
||||
"city": city,
|
||||
"address": address,
|
||||
"region": region,
|
||||
"token": "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E\
|
||||
05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
}
|
||||
return requests.get("http://open.fangjia.com/address/match", params).json()["result"]
|
||||
|
||||
|
||||
def parse_date(time, fm='%Y-%m-%d'):
|
||||
pacific = pytz.timezone('Asia/Shanghai')
|
||||
return pacific.localize(datetime.datetime.strptime(time, fm))
|
||||
|
||||
|
||||
def localize(time):
|
||||
pacific = pytz.timezone('Asia/Shanghai')
|
||||
return pacific.localize(time)
|
||||
|
||||
|
||||
def format_date(time, fm='%Y%m%d'):
|
||||
return time.strftime(fm)
|
||||
|
||||
|
||||
def get_geo_box(lng, lat, radius):
|
||||
central_point = {"type": "Point", "coordinates": [lng, lat]}
|
||||
right_point = destination_point(central_point, 45, radius/1000)
|
||||
left_point = destination_point(central_point, 225, radius/1000)
|
||||
return {"max_lng": right_point["coordinates"][0],
|
||||
"min_lng": left_point["coordinates"][0],
|
||||
"min_lat": left_point["coordinates"][1],
|
||||
"max_lat": right_point["coordinates"][1]}
|
||||
|
||||
|
||||
def k_means(series):
|
||||
values = np.array(list(series))
|
||||
length = len(set(series))
|
||||
if length < 2:
|
||||
return values.mean()
|
||||
values.sort()
|
||||
length = len(values)
|
||||
if length > 10:
|
||||
values = values[int(length * 0.2):int(length * 0.8)]
|
||||
length = len(set(series))
|
||||
if length <= 2:
|
||||
return values.mean()
|
||||
cluster = None
|
||||
max_score = -1
|
||||
values = np.array(list(series))
|
||||
values = values.reshape(len(values), 1)
|
||||
for n_clusters in range(2, min(len(set(series)), 6)):
|
||||
_cluster = KMeans(n_clusters=n_clusters, max_iter=10000).fit(values)
|
||||
score = metrics.silhouette_score(values, _cluster.predict(values), metric='euclidean')
|
||||
if score > max_score:
|
||||
cluster = _cluster
|
||||
max_score = score
|
||||
cluster_centers = list(cluster.cluster_centers_)
|
||||
labels_ = cluster.labels_
|
||||
labels_count = dict()
|
||||
for label in labels_:
|
||||
if label in labels_count:
|
||||
labels_count[label] = labels_count[label] + 1
|
||||
else:
|
||||
labels_count[label] = 1
|
||||
max_cluster_label = max(labels_count.items(), key=lambda x: x[1])[0]
|
||||
return cluster_centers[max_cluster_label][0]
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,24 @@
|
||||
# 日志数据库
|
||||
LOG_MONGO_URI = 'mongodb://192.168.0.42:52002/'
|
||||
# Seawater数据库
|
||||
SEAWATER_MONGO_URI = 'mongodb://192.168.0.206:27020/'
|
||||
# FangJia 数据库
|
||||
MONGO_URI = 'mongodb://developer:G00IDCDEV12385@192.168.0.43:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
|
||||
# OpenAPI测试环境数据库地址
|
||||
TEST_URI = 'mongodb://admin:G00idcMongo85Admin@192.168.0.39:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
|
||||
# 小区库地址
|
||||
SEAWEED_URI = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
|
||||
# ElasticSearch地址
|
||||
SEAWATER_ES_HOST = '192.168.0.206'
|
||||
SEAWATER_ES_HTTP_AUTH = ('elastic', 'goojia123456')
|
||||
# 城市租金样本最小选取量
|
||||
CITY_RENT_MIN_HOUSE_COUNT = 500
|
||||
# 区域租金样本最小选取量
|
||||
REGION_RENT_MIN_HOUSE_COUNT = 10
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
|
||||
|
||||
class Seawater(object):
|
||||
"""房源实体类"""
|
||||
|
||||
def __init__(self, city, region, district_name, area, avg_price, total_price, title,
|
||||
room, root, url, p_date, fitment, direction, cat, rent_type):
|
||||
self.city = city
|
||||
self.region = region
|
||||
self.district_name = district_name
|
||||
self.area = area
|
||||
self.avg_price = avg_price
|
||||
self.total_price = total_price
|
||||
self.title = title
|
||||
self.root = root
|
||||
self.room = room
|
||||
self.url = url
|
||||
self.p_date = p_date
|
||||
self.fitment = fitment
|
||||
self.direction = direction
|
||||
self.cat = cat
|
||||
self.rent_type = rent_type
|
||||
|
||||
Binary file not shown.
+11929
File diff suppressed because it is too large
Load Diff
+23
@@ -0,0 +1,23 @@
|
||||
room,total_price
|
||||
1,7000
|
||||
1,8500
|
||||
1,7500
|
||||
1,6600
|
||||
2,14000
|
||||
2,14000
|
||||
2,12000
|
||||
2,8200
|
||||
2,8200
|
||||
2,14000
|
||||
2,12000
|
||||
2,7500
|
||||
2,8500
|
||||
2,12000
|
||||
2,10500
|
||||
2,8200
|
||||
2,8000
|
||||
2,8500
|
||||
3,14900
|
||||
3,12000
|
||||
3,20000
|
||||
3,12000
|
||||
|
@@ -0,0 +1,71 @@
|
||||
import pandas as pd
|
||||
from common import configs as cfg
|
||||
|
||||
import common
|
||||
|
||||
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
|
||||
HOUSE_RENT = common.get_mongodb_collection("mongodb://192.168.0.105:27018/", "vip_site", "esf_houseRent")
|
||||
DISTRICT_STAT_RENT = common.get_mongodb_collection(cfg.MONGO_URI, "fangjia", "district_stat_rent")
|
||||
DISTRICT_RENT = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
|
||||
|
||||
|
||||
def find_room_rent(city, region, name, room):
|
||||
stat = DISTRICT_STAT_RENT.find_one({"city": city, "region": region, "district_name": name, "house_type": room,
|
||||
"end_month_date": common.parse_date("2019-03-31")})
|
||||
if stat:
|
||||
return stat["avg_price"]
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def find_sq_rent(city, region, name, room):
|
||||
stat = DISTRICT_RENT.find_one({"city": city, "region": region, "name": name, "room": room, "date": 201902})
|
||||
if stat:
|
||||
return stat["rent_price"]
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def find_rent_house(city, region, alias, room):
|
||||
fields_filter = {"address": 0, "_id": 0, "block": 0, "c_date": 0, "title": 0, "m_date": 0, "rent_type": 0}
|
||||
houses = pd.DataFrame(list(HOUSE_RENT.find({"city": city, "region": region, "rent_type": "整租",
|
||||
"district_name": {"$in": alias}, "room": room, "publish_date": {"$gt": common.parse_date("2019-01-01")}}, fields_filter)))
|
||||
if len(houses) == 0:
|
||||
return houses
|
||||
houses["avg_price"] = houses["total_price"] / houses["area"]
|
||||
houses["avg_price"] = houses["avg_price"].map(int)
|
||||
houses.drop_duplicates(keep='first', inplace=True)
|
||||
houses.sort_values(by="area", inplace=True)
|
||||
return houses
|
||||
|
||||
|
||||
regions = ["朝阳", "海淀", "丰台", "西城", "通州", "石景山", "东城"]
|
||||
|
||||
|
||||
def process():
|
||||
all_house = pd.DataFrame()
|
||||
for region in regions:
|
||||
for seaweed in common.get_seaweed("北京", region):
|
||||
names = [seaweed["name"]]
|
||||
if "alias" in seaweed and seaweed["alias"] is not None:
|
||||
names += seaweed["alias"]
|
||||
for room in range(1, 3):
|
||||
houses = find_rent_house(seaweed["city"], seaweed["region"], names, room)
|
||||
if len(houses) > 1:
|
||||
all_house = all_house.append(houses)
|
||||
all_house["publish_date"] = all_house["publish_date"].apply(lambda x: common.format_date(x, '%Y%m%d'))
|
||||
for col in ["direction", "fitment", "floor_info", "hall", "height", "toilet", "avg_price", "publish_date"]:
|
||||
del all_house[col]
|
||||
excel_data = []
|
||||
for index, row in all_house.iterrows():
|
||||
row = dict(row)
|
||||
row["户型租金"] = find_room_rent(row["city"], row["region"], row["district_name"], row["room"])
|
||||
row["平米租金"] = find_sq_rent(row["city"], row["region"], row["district_name"], row["room"])
|
||||
print(row)
|
||||
excel_data.append(row)
|
||||
excel_data = pd.DataFrame(excel_data)
|
||||
excel_data = excel_data.dropna(how="any")
|
||||
excel_data.to_excel("D:\\建行户型房源数据.xlsx", index=False)
|
||||
|
||||
|
||||
process()
|
||||
@@ -0,0 +1,38 @@
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn import preprocessing
|
||||
from sklearn import svm
|
||||
from sklearn.model_selection import GridSearchCV, train_test_split
|
||||
from sklearn.pipeline import Pipeline
|
||||
import common
|
||||
from sklearn.preprocessing import PolynomialFeatures
|
||||
|
||||
|
||||
base_year = common.parse_date("1900-01-01")
|
||||
|
||||
|
||||
def years(now):
|
||||
year = common.parse_date(str(now), '%Y')
|
||||
return int((year - base_year).days/365)
|
||||
|
||||
|
||||
train_data = pd.read_csv(open('../data/小区租金样本.csv'))
|
||||
train_data["build_year"] = train_data["build_year"].apply(lambda param: years(param))
|
||||
x = train_data[["lat", "lng", "build_year", "room", "avg_price"]]
|
||||
y = train_data[["rent_price"]]
|
||||
x = x.astype(np.float64)
|
||||
y = y.astype(np.float64)
|
||||
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.2)
|
||||
c_range = np.logspace(-5, 15, 11, base=2)
|
||||
gamma_range = np.logspace(-9, 3, 13, base=2)
|
||||
param_grid = [{'svr__kernel': ['rbf', 'linear'], 'svr__C': c_range, 'svr__gamma': gamma_range}]
|
||||
svr = svm.SVR()
|
||||
pipeline = Pipeline([('poly', PolynomialFeatures(3)),
|
||||
('standard', preprocessing.StandardScaler()),
|
||||
('minMax', preprocessing.MinMaxScaler()),
|
||||
('svr', svr)])
|
||||
grid = GridSearchCV(pipeline, param_grid, cv=3, n_jobs=10)
|
||||
clf = grid.fit(x_train, y_train.values.ravel())
|
||||
test_score = grid.score(x_test, y_test)
|
||||
print('测试得分是%s' % test_score)
|
||||
print('训练得分是%s' % grid.score(x_train, y_train))
|
||||
@@ -0,0 +1,176 @@
|
||||
from sklearn import svm
|
||||
import re
|
||||
import pandas as pd
|
||||
import warnings
|
||||
import numpy as np
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from sklearn.model_selection import GridSearchCV, train_test_split
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn import preprocessing
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
curPath = os.path.abspath(os.path.dirname(__file__))
|
||||
rootPath = os.path.split(curPath)[0]
|
||||
sys.path.append(rootPath)
|
||||
import common
|
||||
from common import configs as cfg
|
||||
warnings.simplefilter("ignore")
|
||||
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
|
||||
district_room_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_room_rent")
|
||||
district_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
|
||||
district_sell_stat = common.get_mongodb_collection(cfg.MONGO_URI, "fangjia_stat", "district_sell_stat")
|
||||
|
||||
|
||||
def has_geo_info(seaweed):
|
||||
return "lat2" in seaweed and "lng2" in seaweed and seaweed["lng2"] > 0
|
||||
|
||||
|
||||
def get_build_year(seaweed):
|
||||
if "finish_building_date" not in seaweed or seaweed["finish_building_date"] is None:
|
||||
return None
|
||||
pattern = re.compile(r'\d{4}')
|
||||
match = pattern.match(str(seaweed["finish_building_date"]))
|
||||
if not match:
|
||||
return None
|
||||
return int(match.group())
|
||||
|
||||
|
||||
def is_common_community(seaweed):
|
||||
return seaweed["estate_type2"] == "普通住宅"
|
||||
|
||||
|
||||
def get_apply_seaweed(city, year_month):
|
||||
stats = list(district_sell_stat.find({"city": city, "date": year_month}))
|
||||
stat_dict = dict()
|
||||
for stat in stats:
|
||||
key = stat["city"] + "_" + stat["region"] + "_" + stat["district_name"]
|
||||
stat_dict[key] = stat["avg_price"]
|
||||
seaweed_list = list()
|
||||
for seaweed in common.get_seaweed(city):
|
||||
if not has_geo_info(seaweed):
|
||||
continue
|
||||
if not is_common_community(seaweed):
|
||||
continue
|
||||
build_year = get_build_year(seaweed)
|
||||
if not build_year:
|
||||
continue
|
||||
key = seaweed["city"] + "_" + seaweed["region"] + "_" + seaweed["name"]
|
||||
if key not in stat_dict:
|
||||
continue
|
||||
seaweed["build_year"] = build_year
|
||||
seaweed["lat"] = seaweed["lat2"] / 10000000000
|
||||
seaweed["lng"] = seaweed["lng2"] / 10000000000
|
||||
seaweed["avg_price"] = stat_dict[key]
|
||||
seaweed_list.append(seaweed)
|
||||
return seaweed_list
|
||||
|
||||
|
||||
def get_rent_table_by_calc_type(calc_type):
|
||||
if calc_type == "ht":
|
||||
return district_room_rent
|
||||
elif calc_type == "sq":
|
||||
return district_rent
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def load_train_data(city, year_month, calc_type):
|
||||
"""
|
||||
|
||||
:param city:
|
||||
:param year_month:
|
||||
:param calc_type:计算类型 ht:户型租金 sq: 平米租金
|
||||
:return:
|
||||
"""
|
||||
table_name = get_rent_table_by_calc_type(calc_type)
|
||||
data = list()
|
||||
for seaweed in get_apply_seaweed(city, year_month):
|
||||
query = {"city": seaweed["city"], "region": seaweed["region"], "name": seaweed["name"], "date": year_month,
|
||||
"calc_model": {"$ne": "svr"}}
|
||||
stats = list(table_name.find(query))
|
||||
if len(stats) < 3:
|
||||
continue
|
||||
for stat in stats:
|
||||
item = seaweed.copy()
|
||||
item["room"] = stat["room"]
|
||||
item["rent_price"] = stat["rent_price"]
|
||||
data.append(item)
|
||||
print(item)
|
||||
return pd.DataFrame(data)
|
||||
|
||||
|
||||
def has_rent_price(seaweed, room, year_month, calc_type):
|
||||
table_name = get_rent_table_by_calc_type(calc_type)
|
||||
query = {"city": seaweed["city"], "region": seaweed["region"], "name": seaweed["name"],
|
||||
"date": year_month, "room": room}
|
||||
return table_name.count(query) > 0
|
||||
|
||||
|
||||
def train(city, year_month, calc_type):
|
||||
"""
|
||||
|
||||
:param city:
|
||||
:param year_month:
|
||||
:param calc_type:计算类型 ht:户型租金 sq: 平米租金
|
||||
:return:
|
||||
"""
|
||||
train_data = load_train_data(city, year_month, calc_type)
|
||||
x = train_data[["lat", "lng", "build_year", "room", "avg_price"]]
|
||||
y = train_data[["rent_price"]]
|
||||
x = x.astype(np.float64)
|
||||
y = y.astype(np.float64)
|
||||
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.3)
|
||||
c_range = np.logspace(-5, 15, 11, base=2)
|
||||
gamma_range = np.logspace(-9, 3, 13, base=2)
|
||||
param_grid = [{'svr__kernel': ['rbf', 'linear'], 'svr__C': c_range, 'svr__gamma': gamma_range}]
|
||||
svr = svm.SVR()
|
||||
pipeline = Pipeline([('standard', preprocessing.StandardScaler()),
|
||||
('minMax', preprocessing.MinMaxScaler()),
|
||||
('svr', svr)])
|
||||
grid = GridSearchCV(pipeline, param_grid, cv=3, n_jobs=10)
|
||||
clf = grid.fit(x_train, y_train.values.ravel())
|
||||
score = grid.score(x_test, y_test)
|
||||
print('精度为%s' % score)
|
||||
predict_seaweed = list()
|
||||
for seaweed in get_apply_seaweed(city, year_month):
|
||||
for room in range(1, 4):
|
||||
if not has_rent_price(seaweed, room, year_month, calc_type):
|
||||
item = seaweed.copy()
|
||||
item["room"] = room
|
||||
predict_seaweed.append(item)
|
||||
predict_df = pd.DataFrame(predict_seaweed)
|
||||
result = clf.predict(predict_df[["lat", "lng", "build_year", "room", "avg_price"]])
|
||||
result = pd.DataFrame([[int(val)] for val in result])
|
||||
result.columns = ["rent"]
|
||||
predict_df = predict_df.join(result)
|
||||
predict_df = predict_df[["city", "region", "name", "room", "rent"]]
|
||||
table_name = get_rent_table_by_calc_type(calc_type)
|
||||
for index, row in predict_df.iterrows():
|
||||
if row["rent"] <= 0:
|
||||
continue
|
||||
query = {
|
||||
"date": year_month,
|
||||
"city": row["city"],
|
||||
"region": row["region"],
|
||||
"name": row["name"],
|
||||
"room": int(row["room"])
|
||||
}
|
||||
data = dict()
|
||||
data["m_date"] = datetime.datetime.utcnow()
|
||||
data["rent_price"] = row["rent"]
|
||||
data["calc_model"] = "svr"
|
||||
table_name.update_one(query, {"$set": data}, True)
|
||||
print(row)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
start_date = common.parse_date("2019-05-01")
|
||||
now_date = common.localize(datetime.datetime.utcnow())
|
||||
while start_date < now_date:
|
||||
year_month = int(common.format_date(start_date - relativedelta(months=1), '%Y%m'))
|
||||
for city in ["北京"]:
|
||||
train("北京", year_month, "ht")
|
||||
train("北京", year_month, "sq")
|
||||
start_date += relativedelta(months=1)
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
import calendar
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
from math import isnan
|
||||
import pandas as pd
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from joblib import load
|
||||
curPath = os.path.abspath(os.path.dirname(__file__))
|
||||
rootPath = os.path.split(curPath)[0]
|
||||
sys.path.append(rootPath)
|
||||
import common
|
||||
from common import configs as cfg
|
||||
warnings.simplefilter("ignore")
|
||||
|
||||
|
||||
def k_means(series):
|
||||
return common.k_means(series)
|
||||
|
||||
|
||||
clf = load("C:\\Users\\wenyan\\PycharmProjects\\Sklearn\\data\\house_type_classify_model.joblib")
|
||||
|
||||
|
||||
def house_type_filter(houses):
|
||||
"""
|
||||
通过模型过滤掉合租房源
|
||||
:param houses:
|
||||
:return:
|
||||
"""
|
||||
if len(houses) == 0:
|
||||
return houses
|
||||
X = houses[["room", "area"]]
|
||||
houses["rent_type_predict"] = clf.predict(X)
|
||||
houses["prob1"] = clf.predict_proba(X)[:, 1]
|
||||
houses = houses[(houses.rent_type_predict == 1) & (houses.prob1 > 0.8)]
|
||||
del houses["rent_type_predict"]
|
||||
del houses["prob1"]
|
||||
return houses
|
||||
|
||||
|
||||
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
|
||||
region_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "region_rent")
|
||||
district_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
|
||||
|
||||
|
||||
# 删除无用列
|
||||
def remove_useless_columns(df):
|
||||
for col in ["fitment", "direction", "root"]:
|
||||
if col in df.columns:
|
||||
del df[col]
|
||||
|
||||
|
||||
def weighting(df):
|
||||
if 'root' not in df.columns.values.tolist():
|
||||
return df
|
||||
root_weight = {
|
||||
"zhuge.com": 3,
|
||||
"ke.com": 8,
|
||||
"lianjia.com": 8
|
||||
}
|
||||
weighting_data = list()
|
||||
for index, row in df.iterrows():
|
||||
row = dict(row)
|
||||
weighting_data.append(row)
|
||||
if row['root'] in root_weight:
|
||||
for i in range(1, root_weight[row['root']]):
|
||||
weighting_data.append(row)
|
||||
print("org:{} dest:{}".format(len(df), len(weighting_data)))
|
||||
return pd.DataFrame(weighting_data)
|
||||
|
||||
|
||||
def get_start_date_and_end_date(year_month):
|
||||
date = common.parse_date(str(year_month), '%Y%m')
|
||||
year = date.year
|
||||
month = date.month
|
||||
first_day = datetime.datetime(year=year, month=month, day=1)
|
||||
month_range = calendar.monthrange(year, month)
|
||||
end_day = datetime.datetime(year=year, month=month, day=month_range[1])
|
||||
return first_day, end_day
|
||||
|
||||
|
||||
def calc_city_rent(_city, year_month):
|
||||
start_date, end_date = get_start_date_and_end_date(year_month)
|
||||
stats = common.get_least_region_stat(_city, None, start_date=start_date, end_date=end_date)
|
||||
start_date = int(common.format_date(start_date, "%Y%m%d"))
|
||||
end_date = int(common.format_date(end_date, "%Y%m%d"))
|
||||
houses = common.load_house_data(city=_city, region=None, name=None, cat="rent", rent_type="整租",
|
||||
start_date=start_date, end_date=end_date)
|
||||
houses = house_type_filter(houses)
|
||||
houses = weighting(houses)
|
||||
remove_useless_columns(houses)
|
||||
houses.dropna(axis=0, how='any', inplace=True)
|
||||
houses = houses[houses.avg_price > 0]
|
||||
houses.sort_values(by='p_date', ascending=False, inplace=True)
|
||||
agg = houses.groupby(by="room")[['avg_price']].agg(['mean', 'median', 'count', k_means, 'min', 'max'])
|
||||
agg = agg.applymap(int)
|
||||
agg.columns = ["mean", "median", "count", "k_means", "min", "max"]
|
||||
for index, row in agg.iterrows():
|
||||
query = {
|
||||
"date": year_month,
|
||||
"cat": "city",
|
||||
"city": _city,
|
||||
"room": int(index)
|
||||
}
|
||||
data = dict(row)
|
||||
for k, v in data.items():
|
||||
data[k] = int(v)
|
||||
data["m_date"] = datetime.datetime.utcnow()
|
||||
if data["count"] > cfg.CITY_RENT_MIN_HOUSE_COUNT:
|
||||
data["rent_price"] = data["k_means"]
|
||||
if len(stats) > 0:
|
||||
data["avg_price"] = stats[0]["avg_price"]
|
||||
data["price_rent_ratio"] = int(data["avg_price"] / data["k_means"])
|
||||
region_rent.update_one(query, {"$set": data}, True)
|
||||
|
||||
|
||||
def calc_region_rent(_city, _region, year_month):
|
||||
start_date, end_date = get_start_date_and_end_date(year_month)
|
||||
stats = common.get_least_region_stat(city=_city, region=_region, start_date=start_date, end_date=end_date, limit=1)
|
||||
start_date = int(common.format_date(start_date, "%Y%m%d"))
|
||||
end_date = int(common.format_date(end_date, "%Y%m%d"))
|
||||
houses = common.load_house_data(city=_city, region=_region, name=None, cat="rent", rent_type="整租",
|
||||
start_date=start_date, end_date=end_date)
|
||||
houses = house_type_filter(houses)
|
||||
houses = weighting(houses)
|
||||
remove_useless_columns(houses)
|
||||
houses.dropna(axis=0, how='any', inplace=True)
|
||||
if len(houses) == 0:
|
||||
return
|
||||
houses.sort_values(by='p_date', ascending=False, inplace=True)
|
||||
agg = houses.groupby(by="room")[['avg_price']].agg(['mean', 'median', 'count', k_means])
|
||||
agg = agg.applymap(int)
|
||||
agg.columns = ["mean", "median", "count", "k_means"]
|
||||
city_rent = dict()
|
||||
for item in list(region_rent.find({"cat": "city", "city": _city, "date": year_month})):
|
||||
city_rent[item["room"]] = item["price_rent_ratio"]
|
||||
for index, row in agg.iterrows():
|
||||
query = {
|
||||
"date": year_month,
|
||||
"cat": "region",
|
||||
"region": _region,
|
||||
"city": _city,
|
||||
"room": int(index)
|
||||
}
|
||||
data = dict(row)
|
||||
for k, v in data.items():
|
||||
data[k] = int(v)
|
||||
data["m_date"] = datetime.datetime.utcnow()
|
||||
if data["count"] > cfg.REGION_RENT_MIN_HOUSE_COUNT:
|
||||
if len(stats) == 0:
|
||||
data["rent_price"] = data["k_means"]
|
||||
else:
|
||||
region_price = stats[0]["avg_price"]
|
||||
data["avg_price"] = region_price
|
||||
data["rent_price"] = data["k_means"]
|
||||
data["price_rent_ratio"] = int(region_price / data["k_means"])
|
||||
region_rent.update_one(query, {"$set": data}, True)
|
||||
|
||||
|
||||
def filter_by_price_rent(houses, region_price):
|
||||
if region_price is None or region_price == 0:
|
||||
return houses
|
||||
filtered_houses = list()
|
||||
for index, row in houses.iterrows():
|
||||
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3:
|
||||
continue
|
||||
if isnan(row["avg_price"]):
|
||||
row["avg_price"] = int(row["total_price"] / row["area"])
|
||||
house_price_rent_ratio = int(region_price / row["avg_price"])
|
||||
if 100 < house_price_rent_ratio < 1500:
|
||||
filtered_houses.append(row)
|
||||
return pd.DataFrame(filtered_houses)
|
||||
|
||||
|
||||
def filter_by_price_to_rent(houses, seaweed, year_month):
|
||||
stats = common.get_least_district_stat(seaweed["city"], seaweed["region"], seaweed["name"], seaweed["estate_type2"]
|
||||
, limit=1)
|
||||
if len(stats) == 0:
|
||||
return houses
|
||||
seaweed_avg_price = stats[0]["avg_price"]
|
||||
house_type_price_rent = dict()
|
||||
for stat in list(region_rent.find({"city": seaweed["city"], "region": seaweed["region"], "date": year_month})):
|
||||
if stat["cat"] == "city":
|
||||
house_type_price_rent["city_{}".format(stat["room"])] = stat["price_rent_ratio"]
|
||||
else:
|
||||
house_type_price_rent["region_{}".format(stat["room"])] = stat["price_rent_ratio"]
|
||||
|
||||
filtered_houses = list()
|
||||
for index, row in houses.iterrows():
|
||||
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3:
|
||||
continue
|
||||
if isnan(row["avg_price"]):
|
||||
row["avg_price"] = int(row["total_price"] / row["area"])
|
||||
house_price_rent_ratio = int(seaweed_avg_price / row["avg_price"])
|
||||
if "region_{}".format(row["room"]) in house_type_price_rent:
|
||||
key = "region_{}".format(row["room"])
|
||||
if house_type_price_rent[key] * 0.5 < house_price_rent_ratio < house_type_price_rent[key] * 2:
|
||||
filtered_houses.append(row)
|
||||
else:
|
||||
print("过滤:" + str(row))
|
||||
elif "city_{}".format(row["room"]) in house_type_price_rent:
|
||||
key = "city_{}".format(row["room"])
|
||||
if house_type_price_rent[key] * 0.25 < house_price_rent_ratio < house_type_price_rent[key] * 4:
|
||||
filtered_houses.append(row)
|
||||
else:
|
||||
print("过滤:" + str(row))
|
||||
else:
|
||||
filtered_houses.append(row)
|
||||
return pd.DataFrame(filtered_houses)
|
||||
|
||||
|
||||
def main():
|
||||
start_date = common.parse_date("2019-05-01")
|
||||
now_date = common.localize(datetime.datetime.utcnow())
|
||||
while start_date < now_date:
|
||||
year_month = int(common.format_date(start_date - relativedelta(months=1), '%Y%m'))
|
||||
for city in ["北京"]:
|
||||
print("{}:{}\t开始计算".format(year_month, city))
|
||||
calc_city_rent(city, year_month)
|
||||
for region in common.get_regions(city):
|
||||
print("{}:{} {}\t开始计算:".format(year_month, city, region))
|
||||
calc_region_rent(city, region, year_month)
|
||||
start_date += relativedelta(months=1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,261 @@
|
||||
import calendar
|
||||
import datetime
|
||||
import multiprocessing
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
from math import isnan
|
||||
from joblib import load
|
||||
import pandas as pd
|
||||
from dateutil.relativedelta import relativedelta
|
||||
|
||||
curPath = os.path.abspath(os.path.dirname(__file__))
|
||||
rootPath = os.path.split(curPath)[0]
|
||||
sys.path.append(rootPath)
|
||||
import common
|
||||
warnings.simplefilter("ignore")
|
||||
|
||||
|
||||
def k_means(series):
|
||||
return common.k_means(series)
|
||||
|
||||
|
||||
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
|
||||
region_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "region_rent")
|
||||
district_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_rent")
|
||||
district_room_rent = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_room_rent")
|
||||
|
||||
|
||||
# 删除无用列
|
||||
def remove_useless_columns(df):
|
||||
for col in ["fitment", "direction", "root"]:
|
||||
if col in df.columns:
|
||||
del df[col]
|
||||
|
||||
|
||||
def weighting(df):
|
||||
root_weight = {
|
||||
"zhuge.com": 2,
|
||||
"5i5j.com": 3,
|
||||
"ke.com": 8,
|
||||
"lianjia.com": 8
|
||||
}
|
||||
weighting_data = list()
|
||||
for index, row in df.iterrows():
|
||||
row = dict(row)
|
||||
weighting_data.append(row)
|
||||
if 'root' in row and row['root'] in root_weight:
|
||||
for i in range(1, root_weight[row['root']]):
|
||||
weighting_data.append(row)
|
||||
return pd.DataFrame(weighting_data)
|
||||
|
||||
|
||||
def get_region_markup(_city, _region, _start, _end, _room):
|
||||
_room = 2
|
||||
start_stat = region_rent.find_one(
|
||||
{"cat": "region", "city": _city, "region": _region, "date": _start, "room": _room})
|
||||
end_stat = region_rent.find_one({"cat": "region", "city": _city, "region": _region, "date": _end, "room": _room})
|
||||
if not start_stat or not end_stat:
|
||||
return 0
|
||||
return (end_stat["rent_price"] - start_stat["rent_price"]) / start_stat["rent_price"]
|
||||
|
||||
|
||||
def correct_house(_houses, _year_month):
|
||||
calc_year_month = common.parse_date(str(_year_month), '%Y%m')
|
||||
for index, house in _houses.iterrows():
|
||||
p_date = common.parse_date(str(house["p_date"]), '%Y%m%d')
|
||||
days = (calc_year_month - p_date).days
|
||||
if days > 180:
|
||||
markup = get_region_markup(house["city"], house["region"], int(house["p_date"] / 100), _year_month, house["room"])
|
||||
_houses.at[index, "avg_price"] = house["avg_price"] * (1 + markup)
|
||||
_houses.at[index, "total_price"] = house["total_price"] * (1 + markup)
|
||||
return _houses
|
||||
|
||||
|
||||
def get_start_date_and_end_date(year_month):
|
||||
date = common.parse_date(str(year_month), '%Y%m')
|
||||
year = date.year
|
||||
month = date.month
|
||||
first_day = datetime.datetime(year=year, month=month, day=1)
|
||||
month_range = calendar.monthrange(year, month)
|
||||
end_day = datetime.datetime(year=year, month=month, day=month_range[1])
|
||||
return first_day, end_day
|
||||
|
||||
|
||||
def filter_by_price_rent(houses, region_price):
|
||||
if region_rent is None:
|
||||
return houses
|
||||
filtered_houses = list()
|
||||
for index, row in houses.iterrows():
|
||||
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3:
|
||||
continue
|
||||
if isnan(row["avg_price"]):
|
||||
row["avg_price"] = int(row["total_price"] / row["area"])
|
||||
house_price_rent_ratio = int(region_price / row["avg_price"])
|
||||
if 100 < house_price_rent_ratio < 1000:
|
||||
filtered_houses.append(row)
|
||||
return pd.DataFrame(filtered_houses)
|
||||
|
||||
|
||||
def filter_by_price_to_rent(houses, seaweed, year_month):
|
||||
stats = common.get_least_district_stat(seaweed["city"], seaweed["region"], seaweed["name"], seaweed["estate_type2"]
|
||||
, limit=1)
|
||||
if len(stats) == 0:
|
||||
return houses
|
||||
seaweed_avg_price = stats[0]["avg_price"]
|
||||
house_type_price_rent = dict()
|
||||
for stat in list(region_rent.find({"city": seaweed["city"], "region": seaweed["region"], "date": year_month})):
|
||||
if stat["cat"] == "city":
|
||||
house_type_price_rent["city_{}".format(stat["room"])] = stat["price_rent_ratio"]
|
||||
else:
|
||||
house_type_price_rent["region_{}".format(stat["room"])] = stat["price_rent_ratio"]
|
||||
if len(house_type_price_rent) > 0:
|
||||
house_type_price_rent["region"] = int(pd.Series(list(house_type_price_rent.values())).mean())
|
||||
filtered_houses = list()
|
||||
houses[["room", "total_price"]].to_csv("../data/房源.csv", index=False)
|
||||
for index, row in houses.iterrows():
|
||||
if isnan(row["area"]) is None or row["total_price"] is None or row["area"] < 3 or row["area"] == 0:
|
||||
continue
|
||||
if "avg_price" not in row or isnan(row["avg_price"]):
|
||||
row["avg_price"] = int(row["total_price"] / row["area"])
|
||||
if row["avg_price"] < 10:
|
||||
continue
|
||||
house_price_rent_ratio = int(seaweed_avg_price / row["avg_price"])
|
||||
if "region_{}".format(row["room"]) in house_type_price_rent:
|
||||
key = "region_{}".format(row["room"])
|
||||
if house_type_price_rent[key] * 0.5 < house_price_rent_ratio < house_type_price_rent[key] * 2:
|
||||
filtered_houses.append(row)
|
||||
elif "city_{}".format(row["room"]) in house_type_price_rent:
|
||||
key = "city_{}".format(row["room"])
|
||||
if house_type_price_rent[key] * 0.25 < house_price_rent_ratio < house_type_price_rent[key] * 4:
|
||||
filtered_houses.append(row)
|
||||
elif "region" in house_type_price_rent:
|
||||
key = "region"
|
||||
if house_type_price_rent[key] * 0.4 < house_price_rent_ratio < house_type_price_rent[key] * 2:
|
||||
filtered_houses.append(row)
|
||||
else:
|
||||
filtered_houses.append(row)
|
||||
return pd.DataFrame(filtered_houses)
|
||||
|
||||
|
||||
clf = load("C:\\Users\\wenyan\\PycharmProjects\\Sklearn\\data\\house_type_classify_model.joblib")
|
||||
|
||||
|
||||
def house_type_filter(houses):
|
||||
"""
|
||||
通过模型过滤掉合租房源
|
||||
:param houses:
|
||||
:return:
|
||||
"""
|
||||
X = houses[["room", "area"]]
|
||||
houses["rent_type_predict"] = clf.predict(X)
|
||||
houses["prob1"] = clf.predict_proba(X)[:, 1]
|
||||
houses = houses[(houses.rent_type_predict == 1) & (0.8 < houses.prob1) & (houses.prob1 < 0.99999)]
|
||||
del houses["rent_type_predict"]
|
||||
del houses["prob1"]
|
||||
return houses
|
||||
|
||||
|
||||
def drop_duplicates(houses):
|
||||
houses["p_month"] = houses["p_date"].apply(lambda x: int(x/100))
|
||||
houses = houses.drop_duplicates(subset=["area", "avg_price", "p_month", "room", "root", "total_price"])
|
||||
del houses["p_month"]
|
||||
return houses
|
||||
|
||||
|
||||
def calc_seaweed_rent(seaweed, year_month):
|
||||
print("开始计算小区租金{}: {} {} {}".format(year_month, seaweed["city"], seaweed["region"], seaweed["name"]))
|
||||
start_date, end_date = get_start_date_and_end_date(year_month)
|
||||
stats = common.get_least_district_stat(seaweed["city"], seaweed["region"], seaweed["name"], seaweed["estate_type2"]
|
||||
, start_date=start_date, end_date=end_date, limit=1)
|
||||
start_date = end_date - relativedelta(years=1)
|
||||
first_date = end_date - relativedelta(months=1)
|
||||
start_date = int(common.format_date(start_date, "%Y%m%d"))
|
||||
first_date = int(common.format_date(first_date, "%Y%m%d"))
|
||||
end_date = int(common.format_date(end_date, "%Y%m%d"))
|
||||
alias = None
|
||||
if alias in seaweed:
|
||||
alias = seaweed["alias"]
|
||||
houses = pd.DataFrame()
|
||||
for house_type in range(1, 4):
|
||||
_houses = common.load_house_data(city=seaweed["city"], region=seaweed["region"], name=seaweed["name"],
|
||||
cat="rent", room=house_type, rent_type="整租", end_date=end_date,
|
||||
start_date=first_date, alias=alias, limit=100)
|
||||
if len(_houses) < 5:
|
||||
_houses = common.load_house_data(city=seaweed["city"], region=seaweed["region"], name=seaweed["name"],
|
||||
cat="rent", room=house_type,
|
||||
rent_type="整租", end_date=end_date, start_date=start_date, alias=alias,
|
||||
limit=100)
|
||||
houses = houses.append(_houses, ignore_index=True)
|
||||
|
||||
if len(houses) == 0:
|
||||
return
|
||||
houses = drop_duplicates(houses)
|
||||
houses = house_type_filter(houses)
|
||||
houses = filter_by_price_to_rent(houses, seaweed, year_month)
|
||||
houses = correct_house(houses, year_month)
|
||||
houses = weighting(houses)
|
||||
remove_useless_columns(houses)
|
||||
houses.dropna(axis=0, how='any', inplace=True)
|
||||
if len(houses) < 3:
|
||||
return
|
||||
if "room" not in houses.columns:
|
||||
return
|
||||
houses = houses[houses.avg_price > 0]
|
||||
houses.sort_values(by='p_date', ascending=False, inplace=True)
|
||||
agg = houses.groupby(by="room")[['avg_price']].agg(['mean', 'median', 'count', k_means])
|
||||
total_price_agg = houses.groupby(by="room")[['total_price']].agg(['mean', 'median', 'count', k_means])
|
||||
agg.columns = ["mean", "median", "count", "k_means"]
|
||||
total_price_agg.columns = ["mean", "median", "count", "k_means"]
|
||||
for index, row in agg.iterrows():
|
||||
query = {
|
||||
"date": year_month,
|
||||
"city": seaweed["city"],
|
||||
"region": seaweed["region"],
|
||||
"name": seaweed["name"],
|
||||
"room": int(index)
|
||||
}
|
||||
data = dict(row)
|
||||
for k, v in data.items():
|
||||
data[k] = int(v)
|
||||
data["m_date"] = datetime.datetime.utcnow()
|
||||
data["rent_price"] = data["k_means"]
|
||||
if len(stats) != 0:
|
||||
data["avg_price"] = stats[0]["avg_price"]
|
||||
data["price_rent_ratio"] = int(data["avg_price"] / data["rent_price"])
|
||||
district_rent.update_one(query, {"$set": data}, True)
|
||||
|
||||
for index, row in total_price_agg.iterrows():
|
||||
query = {
|
||||
"date": year_month,
|
||||
"city": seaweed["city"],
|
||||
"region": seaweed["region"],
|
||||
"name": seaweed["name"],
|
||||
"room": int(index)
|
||||
}
|
||||
data = dict(row)
|
||||
for k, v in data.items():
|
||||
data[k] = int(v)
|
||||
data["m_date"] = datetime.datetime.utcnow()
|
||||
data["rent_price"] = data["k_means"]
|
||||
district_room_rent.update_one(query, {"$set": data}, True)
|
||||
|
||||
|
||||
def main():
|
||||
pool = multiprocessing.Pool(processes=30)
|
||||
start_date = common.parse_date("2019-05-01")
|
||||
now_date = common.localize(datetime.datetime.utcnow())
|
||||
while start_date < now_date:
|
||||
year_month = int(common.format_date(start_date - relativedelta(months=1), '%Y%m'))
|
||||
for city in ["北京"]:
|
||||
for community in common.get_seaweed(city):
|
||||
pool.apply_async(calc_seaweed_rent, (community, year_month,))
|
||||
start_date += relativedelta(months=1)
|
||||
pool.close()
|
||||
pool.join()
|
||||
print("done")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
calc_seaweed_rent(common.get_seaweed("北京", "西城", "富国里")[0], 201904)
|
||||
# main()
|
||||
@@ -0,0 +1,33 @@
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from common import configs
|
||||
import common
|
||||
plt.rcParams['savefig.dpi'] = 400
|
||||
plt.rcParams['figure.dpi'] = 400
|
||||
plt.rcParams['figure.figsize'] = (12, 4)
|
||||
plt.rcParams['font.sans-serif'] = ['SimHei']
|
||||
region_rent = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia_rent", "region_rent")
|
||||
price_rent = dict()
|
||||
for region in common.get_regions("北京"):
|
||||
df = pd.DataFrame(list(region_rent.find({"cat": "region", "region": region}, {"price_rent_ratio": 1})))
|
||||
if len(df) > 0:
|
||||
price_rent[region] = int(df["price_rent_ratio"].mean())
|
||||
fig, ax = plt.subplots()
|
||||
x = []
|
||||
y = []
|
||||
z = []
|
||||
avg = np.array(list(price_rent.values())).mean()
|
||||
for k, v in price_rent.items():
|
||||
x.append(k)
|
||||
y.append(v)
|
||||
z.append(avg)
|
||||
ax.bar(x, y, width=0.5)
|
||||
ax.plot(x, y, label='区域售租比')
|
||||
ax.plot(x, z, label='平均线:' + str(avg))
|
||||
ax.set(xlabel='区域', ylabel='租售比', title='区域租售比分布图')
|
||||
ax.grid()
|
||||
plt.legend()
|
||||
plt.savefig("D:\\建设银行\\租售比2000以内\\各区域租售比.png")
|
||||
for item in price_rent.items():
|
||||
print(str(item[0]) + "\t" + str(item[1]))
|
||||
@@ -0,0 +1,12 @@
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn import linear_model
|
||||
df = pd.read_csv(open("C:\\Users\\wenyan\\PycharmProjects\\Sklearn\\data\\房源.csv"))
|
||||
fig, ax = plt.subplots()
|
||||
ax.scatter(df["room"], df["total_price"])
|
||||
|
||||
lr = linear_model.LinearRegression(normalize=True)
|
||||
X = df["room"].values.reshape(len(df["room"]), 1)
|
||||
Y = df["total_price"].values.ravel()
|
||||
lr.fit(X, Y)
|
||||
ax.plot([1, 2, 3], lr.predict([[1], [2], [3]]), 'r')
|
||||
@@ -0,0 +1,17 @@
|
||||
import matplotlib.pyplot as plt
|
||||
import pandas as pd
|
||||
import common
|
||||
|
||||
mongo_uri = 'mongodb://developer:G00IDCDEV12385@192.168.0.136:28017/?connectTimeoutMS=60000&socketTimeoutMS=10000000'
|
||||
HOUSE_RENT = common.get_mongodb_collection("mongodb://192.168.0.105:27018/", "vip_site", "esf_houseRent")
|
||||
DISTRICT_RENT = common.get_mongodb_collection(mongo_uri, "fangjia_rent", "district_room_rent")
|
||||
stats = list(DISTRICT_RENT.find({"city": "北京", "region": "朝阳", "name": "甜水园"}))
|
||||
df = pd.DataFrame(stats)
|
||||
df.dropna()
|
||||
df = df[["city", "region", "name", "date", "room", "rent_price"]]
|
||||
df["date"] = df["date"].apply(lambda x: common.parse_date(str(x), '%Y%m'))
|
||||
room1_df = df[df["room"] == 2]
|
||||
room1_df = room1_df.sort_values(by="date")
|
||||
fig, ax = plt.subplots()
|
||||
ax.plot(room1_df["date"], room1_df["rent_price"], 'h-.')
|
||||
plt.show()
|
||||
@@ -0,0 +1,36 @@
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from common import configs
|
||||
import common
|
||||
|
||||
# region_rent = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia_rent", "region_rent")
|
||||
# price_rent = dict()
|
||||
# for region in common.get_regions("北京"):
|
||||
# df = pd.DataFrame(list(region_rent.find({"cat": "region", "region": region}, {"price_rent_ratio": 1})))
|
||||
# if len(df) > 0:
|
||||
# price_rent[region] = int(df["price_rent_ratio"].mean())
|
||||
# plt.rcParams['font.sans-serif'] = ['SimHei']
|
||||
# district_rent = common.get_mongodb_collection(configs.SEAWEED_URI, "fangjia_rent", "district_rent")
|
||||
# plt.rcParams['savefig.dpi'] = 800
|
||||
# plt.rcParams['figure.dpi'] = 800
|
||||
# plt.rcParams['figure.figsize'] = (6, 4)
|
||||
# for region in common.get_regions("北京"):
|
||||
# df = pd.DataFrame(list(district_rent.find({"region": region}, {"price_rent_ratio": 1})))
|
||||
# ratio = np.array(df["price_rent_ratio"])
|
||||
# result = pd.value_counts(ratio)
|
||||
# result = result.sort_index()
|
||||
# fig, ax = plt.subplots()
|
||||
# ax.plot(result.index, result.values)
|
||||
# if region in price_rent:
|
||||
# y = np.arange(int(max(result.values) * 1.2))
|
||||
# x = np.zeros(len(y)) + price_rent[region]
|
||||
# ax.plot(x, y, label="区域售租比")
|
||||
# ax.plot(x * 2, y, '--', label="2倍区域售租比")
|
||||
# ax.plot(x * 0.4, y, '-', label="0.4倍区域售租比")
|
||||
# ax.set(xlabel='售租比', ylabel='小区量', title=region + '区小区售租比分布图')
|
||||
# ax.grid()
|
||||
# plt.legend()
|
||||
# plt.savefig("D:\\建设银行\\租售比\\" + region + "租售比分布分布图.png")
|
||||
numbers = np.array([12000, 12000, 14900, 20000])
|
||||
print(common.k_means(numbers))
|
||||
@@ -0,0 +1,253 @@
|
||||
absl-py==0.7.0
|
||||
alabaster==0.7.10
|
||||
alembic==1.0.3
|
||||
anaconda-client==1.6.14
|
||||
anaconda-navigator==1.9.6
|
||||
anaconda-project==0.8.2
|
||||
aniso8601==4.0.1
|
||||
APScheduler==3.5.0
|
||||
asn1crypto==0.24.0
|
||||
astor==0.7.1
|
||||
astroid==1.6.3
|
||||
astropy==3.0.2
|
||||
attrs==18.1.0
|
||||
Babel==2.5.3
|
||||
backcall==0.1.0
|
||||
backports.shutil-get-terminal-size==1.0.0
|
||||
beautifulsoup4==4.6.0
|
||||
bitarray==0.8.1
|
||||
bkcharts==0.2
|
||||
blaze==0.11.3
|
||||
bleach==2.1.3
|
||||
blinker==1.4
|
||||
bokeh==0.12.16
|
||||
boto==2.48.0
|
||||
Bottleneck==1.2.1
|
||||
brotlipy==0.7.0
|
||||
certifi==2019.3.9
|
||||
cffi==1.11.5
|
||||
chardet==3.0.4
|
||||
click==6.7
|
||||
cloudpickle==0.5.3
|
||||
clyent==1.2.2
|
||||
colorama==0.3.9
|
||||
comtypes==1.1.4
|
||||
conda==4.6.8
|
||||
conda-build==3.10.5
|
||||
conda-verify==2.0.0
|
||||
contextlib2==0.5.5
|
||||
coverage==3.7.1
|
||||
cronmon==0.1.0
|
||||
cryptography==2.6.1
|
||||
cycler==0.10.0
|
||||
Cython==0.28.2
|
||||
cytoolz==0.9.0.1
|
||||
dask==0.17.5
|
||||
datashape==0.5.4
|
||||
decorator==4.3.0
|
||||
distributed==1.21.8
|
||||
docutils==0.14
|
||||
dogpile.cache==0.6.5
|
||||
elasticsearch==6.3.1
|
||||
entrypoints==0.2.3
|
||||
et-xmlfile==1.0.1
|
||||
eventlet==0.23.0
|
||||
fastcache==1.0.2
|
||||
filelock==3.0.4
|
||||
Flask==0.12.2
|
||||
Flask-APScheduler==1.7.1
|
||||
Flask-Cors==3.0.4
|
||||
flask-marshmallow==0.8.0
|
||||
Flask-Migrate==2.3.0
|
||||
Flask-RESTful==0.3.6
|
||||
Flask-Script==2.0.6
|
||||
Flask-SocketIO==3.0.1
|
||||
Flask-SQLAlchemy==2.3.2
|
||||
Flask-Testing==0.7.1
|
||||
gast==0.2.2
|
||||
geojson==2.4.1
|
||||
geojson-utils==0.0.2
|
||||
gevent==1.3.0
|
||||
glob2==0.6
|
||||
greenlet==0.4.13
|
||||
grpcio==1.12.1
|
||||
h11==0.7.0
|
||||
h2==3.1.0
|
||||
h5py==2.7.1
|
||||
heapdict==1.0.0
|
||||
hpack==3.0.0
|
||||
html5lib==1.0.1
|
||||
hyperframe==5.2.0
|
||||
idna==2.6
|
||||
imageio==2.3.0
|
||||
imagesize==1.0.0
|
||||
ipykernel==4.8.2
|
||||
ipython==6.4.0
|
||||
ipython-genutils==0.2.0
|
||||
ipywidgets==7.2.1
|
||||
isort==4.3.4
|
||||
itsdangerous==0.24
|
||||
jdcal==1.4
|
||||
jedi==0.12.0
|
||||
Jinja2==2.10
|
||||
jmespath==0.9.3
|
||||
jsonschema==2.6.0
|
||||
jupyter==1.0.0
|
||||
jupyter-client==5.2.3
|
||||
jupyter-console==5.2.0
|
||||
jupyter-core==4.4.0
|
||||
jupyterlab==0.32.1
|
||||
jupyterlab-launcher==0.10.5
|
||||
kaitaistruct==0.8
|
||||
Keras==2.2.4
|
||||
Keras-Applications==1.0.6
|
||||
Keras-Preprocessing==1.0.5
|
||||
kiwisolver==1.0.1
|
||||
lazy-object-proxy==1.3.1
|
||||
ldap3==2.5.2
|
||||
llvmlite==0.23.1
|
||||
locket==0.2.0
|
||||
lxml==4.2.1
|
||||
Mako==1.0.7
|
||||
Markdown==3.0.1
|
||||
MarkupSafe==1.0
|
||||
marshmallow==3.0.0b5
|
||||
marshmallow-sqlalchemy==0.13.2
|
||||
matplotlib==2.2.2
|
||||
mccabe==0.6.1
|
||||
menuinst==1.4.14
|
||||
mistune==0.8.3
|
||||
mitmproxy==4.0.4
|
||||
mkl-fft==1.0.10
|
||||
mkl-random==1.0.2
|
||||
more-itertools==4.1.0
|
||||
mpmath==1.0.0
|
||||
msgpack-python==0.5.6
|
||||
multipledispatch==0.5.0
|
||||
navigator-updater==0.2.1
|
||||
nbconvert==5.3.1
|
||||
nbformat==4.4.0
|
||||
networkx==2.1
|
||||
nltk==3.3
|
||||
nose==1.3.4
|
||||
notebook==5.5.0
|
||||
numba==0.38.0
|
||||
numexpr==2.6.9
|
||||
numpy==1.16.2
|
||||
numpydoc==0.8.0
|
||||
odo==0.5.1
|
||||
olefile==0.45.1
|
||||
openpyxl==2.5.3
|
||||
packaging==17.1
|
||||
pandas==0.23.0
|
||||
pandocfilters==1.4.2
|
||||
parso==0.2.0
|
||||
partd==0.3.8
|
||||
passlib==1.7.1
|
||||
path.py==11.0.1
|
||||
pathlib2==2.3.2
|
||||
patsy==0.5.0
|
||||
pep8==1.7.1
|
||||
pickleshare==0.7.4
|
||||
Pillow==5.1.0
|
||||
pkginfo==1.4.2
|
||||
pluggy==0.6.0
|
||||
ply==3.11
|
||||
prompt-toolkit==1.0.15
|
||||
protobuf==3.6.0
|
||||
psutil==5.4.5
|
||||
py==1.5.3
|
||||
pyasn1==0.4.5
|
||||
pycodestyle==2.4.0
|
||||
pycosat==0.6.3
|
||||
pycparser==2.18
|
||||
pycrypto==2.6.1
|
||||
pycurl==7.43.0.2
|
||||
pydivert==2.1.0
|
||||
pyflakes==1.6.0
|
||||
Pygments==2.2.0
|
||||
pylint==1.8.4
|
||||
pymongo==3.7.1
|
||||
pyodbc==4.0.23
|
||||
pyOpenSSL==18.0.0
|
||||
pyparsing==2.2.0
|
||||
pyperclip==1.6.5
|
||||
pypinyin==0.33.2
|
||||
PySocks==1.6.8
|
||||
pytest==3.0.2
|
||||
pytest-arraydiff==0.2
|
||||
pytest-astropy==0.3.0
|
||||
pytest-cov==2.3.1
|
||||
pytest-doctestplus==0.1.3
|
||||
pytest-openfiles==0.3.0
|
||||
pytest-ordering==0.5
|
||||
pytest-remotedata==0.2.1
|
||||
python-dateutil==2.7.2
|
||||
python-editor==1.0.3
|
||||
python-engineio==2.2.0
|
||||
python-socketio==2.0.0
|
||||
pytz==2018.4
|
||||
PyWavelets==0.5.2
|
||||
pywin32==223
|
||||
pywinpty==0.5.1
|
||||
PyYAML==3.12
|
||||
pyzmq==17.0.0
|
||||
QtAwesome==0.4.4
|
||||
qtconsole==4.3.1
|
||||
QtPy==1.4.1
|
||||
redis==2.10.6
|
||||
redis-py-cluster==1.3.6
|
||||
requests==2.18.4
|
||||
retrying==1.3.3
|
||||
rope==0.10.7
|
||||
ruamel-yaml==0.15.35
|
||||
ruamel.yaml==0.15.88
|
||||
scikit-image==0.13.1
|
||||
scikit-learn==0.20.3
|
||||
scipy==1.2.1
|
||||
seaborn==0.8.1
|
||||
Send2Trash==1.5.0
|
||||
simplegeneric==0.8.1
|
||||
singledispatch==3.4.0.3
|
||||
six==1.11.0
|
||||
snowballstemmer==1.2.1
|
||||
sortedcollections==0.6.1
|
||||
sortedcontainers==1.5.10
|
||||
Sphinx==1.7.4
|
||||
sphinxcontrib-websupport==1.0.1
|
||||
spyder==3.2.8
|
||||
SQLAlchemy==1.2.0
|
||||
statsmodels==0.9.0
|
||||
sympy==1.1.1
|
||||
tables==3.4.3
|
||||
tblib==1.3.2
|
||||
tensorboard==1.12.0
|
||||
tensorflow==1.12.0
|
||||
termcolor==1.1.0
|
||||
terminado==0.8.1
|
||||
testpath==0.3.1
|
||||
toolz==0.9.0
|
||||
tornado==5.0.2
|
||||
traitlets==4.3.2
|
||||
typing==3.6.4
|
||||
tzlocal==1.5.1
|
||||
unicodecsv==0.14.1
|
||||
urllib3==1.22
|
||||
urwid==2.0.1
|
||||
wcwidth==0.1.7
|
||||
webencodings==0.5.1
|
||||
webtest==2.0.32
|
||||
Werkzeug==0.14.1
|
||||
widgetsnbextension==3.2.1
|
||||
win-inet-pton==1.0.1
|
||||
win-unicode-console==0.5
|
||||
wincertstore==0.2
|
||||
wrapt==1.10.11
|
||||
wsproto==0.11.0
|
||||
wtforms==2.2.1
|
||||
xlrd==1.1.0
|
||||
XlsxWriter==1.0.4
|
||||
xlwings==0.11.8
|
||||
xlwt==1.3.0
|
||||
zict==0.1.3
|
||||
joblib==0.13.2
|
||||
@@ -0,0 +1,40 @@
|
||||
import sys
|
||||
import os
|
||||
curPath = os.path.abspath(os.path.dirname(__file__))
|
||||
rootPath = os.path.split(curPath)[0]
|
||||
sys.path.append(rootPath)
|
||||
|
||||
|
||||
import common
|
||||
from common import configs as cfg
|
||||
from sync import utils
|
||||
|
||||
|
||||
prod_district_stat = common.get_mongodb_collection(cfg.MONGO_URI, "fangjia_stat", "district_stat")
|
||||
test_district_stat = common.get_mongodb_collection(cfg.TEST_URI, "fangjia_stat", "district_stat")
|
||||
for index in prod_district_stat.list_indexes():
|
||||
keys = list()
|
||||
for key in index["key"]:
|
||||
keys.append((key, index["key"][key]))
|
||||
print(index)
|
||||
if "unique" in index:
|
||||
test_district_stat.create_index(keys, background=True, unique=index["unique"])
|
||||
else:
|
||||
test_district_stat.create_index(keys, background=True)
|
||||
|
||||
for seaweed in utils.get_all_test_seaweed():
|
||||
sync_condition = {
|
||||
"city": seaweed["city"],
|
||||
"region": seaweed["region"],
|
||||
"district_name": seaweed["name"],
|
||||
"weekend": {"$gt": common.parse_date("2018-03-01")}
|
||||
}
|
||||
print(sync_condition)
|
||||
total = prod_district_stat.count(sync_condition)
|
||||
count = 0
|
||||
for dbo in prod_district_stat.find(sync_condition, no_cursor_timeout=True).batch_size(1000):
|
||||
count += 1
|
||||
test_district_stat.save(dbo)
|
||||
if count % 1000 == 0:
|
||||
print(str(count) + "/" + str(total))
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import common
|
||||
from common import configs as cfg
|
||||
|
||||
|
||||
prod_district_stat = common.get_mongodb_collection(cfg.MONGO_URI, "fangjia", "district_stat_rent")
|
||||
test_district_stat = common.get_mongodb_collection(cfg.TEST_URI, "fangjia", "district_stat_rent")
|
||||
for index in prod_district_stat.list_indexes():
|
||||
keys = list()
|
||||
for key in index["key"]:
|
||||
keys.append((key, index["key"][key]))
|
||||
print(index)
|
||||
if "unique" in index:
|
||||
test_district_stat.create_index(keys, background=True, unique=index["unique"])
|
||||
else:
|
||||
test_district_stat.create_index(keys, background=True)
|
||||
|
||||
|
||||
|
||||
sync_condition = {
|
||||
"city": "上海",
|
||||
"region": "长宁",
|
||||
"end_month_date": {"$gt": common.parse_date("2018-03-01")}
|
||||
}
|
||||
|
||||
total = prod_district_stat.count(sync_condition)
|
||||
count = 0
|
||||
for dbo in prod_district_stat.find(sync_condition, no_cursor_timeout=True).batch_size(1000):
|
||||
count += 1
|
||||
test_district_stat.save(dbo)
|
||||
if count % 1000 == 0:
|
||||
print(str(count) + "/" + str(total))
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,6 @@
|
||||
import common
|
||||
seaweed = common.get_mongodb_collection("mongodb://192.168.0.39:52002/", "fangjia", "seaweed")
|
||||
|
||||
|
||||
def get_all_test_seaweed():
|
||||
return list(seaweed.find())
|
||||
Reference in New Issue
Block a user