git commit -m "Initial commit"

This commit is contained in:
2019-05-07 11:33:15 +08:00
commit 51b2b6baa7
30 changed files with 13620 additions and 0 deletions
+110
View File
@@ -0,0 +1,110 @@
import pandas as pd
import common
from common import configs
from datetime import datetime
import numpy as np
from pymongo import UpdateOne
from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED
def cal(x):
try:
return int(x * 10000)
except Exception as ex:
print(ex)
return 0
district_stat = common.get_mongodb_collection(configs.MONGO_URI, "fangjia_stat", "district_stat")
fill_command = common.get_mongodb_collection(configs.LOG_MONGO_URI, "fangjia_log", "fill_command")
def padding(city):
latest_week_query = {
'weekend': common.get_last_weekend(),
'city': city
}
count = 1
latest_stats = list(district_stat.find(latest_week_query).batch_size(500))
for latest_stat in latest_stats:
print(city + ":" + str(count))
count = count + 1
query = {
'city': latest_stat["city"],
'region': latest_stat["region"],
'district_name': latest_stat["district_name"],
'estate_type': latest_stat["estate_type"]
}
start_date = list(district_stat.find(query).sort([("weekend", 1)]).limit(1))[0]["weekend"]
end_date = list(district_stat.find(query).sort([("weekend", -1)]).limit(1))[0]["weekend"]
weekends = pd.date_range(start_date, end_date, freq='7D')
if len(weekends) < 3:
continue
if len(weekends) == district_stat.count(query):
continue
stats = pd.DataFrame(list(district_stat.find(query).sort([("weekend", 1)])))
stats = stats.set_index('weekend', drop=False)
isUpdate = False
update_count = 0
outer = False
if len(weekends) != len(stats["weekend"]) and len(weekends) > 3:
miss_weekends = [i for i in weekends if i not in list(stats["weekend"])]
for weekend in miss_weekends:
stats = stats.append(pd.Series({"weekend": weekend, "value_type": 10}, name=weekend))
stats.sort_index(inplace=True)
min_avg_price = int(min(stats["avg_price"]))
max_avg_price = int(max(stats["avg_price"]))
try:
stats["avg_price"].interpolate(method='polynomial', order=3, inplace=True, limit_area='inside')
except Exception as ex:
print(str(ex))
print(latest_stat)
continue
for other_field in ["city", "region", "block", "estate_type", "district_name"]:
if latest_stat[other_field] is not None:
stats[other_field].fillna(latest_stat[other_field], inplace=True)
stats["credit"].fillna(0, inplace=True)
stats['new'] = (stats["avg_price"] - stats.shift(1)["avg_price"]) / stats.shift(1)["avg_price"]
stats['new'].fillna(0, inplace=True)
stats['new'] = stats['new'].apply(lambda x: cal(x))
stats['weekend_markup'].fillna(0, inplace=True)
requests = []
for index, row in stats.iterrows():
if row["_id"] is np.nan or row["_id"] is None or row["new"] != int(row["weekend_markup"]):
update = {
"city": row["city"],
"region": row["region"],
"district_name": row["district_name"],
"estate_type": row["estate_type"],
"weekend": row["weekend"]
}
data = {
"credit": row["credit"],
"avg_price": int(row["avg_price"]),
"m_date": datetime.utcnow(),
"weekend_markup": int(row["new"])
}
if row["_id"] is np.nan or row["_id"] is None:
data["c_date"] = datetime.utcnow()
data["calc_value"] = int(row["avg_price"])
data["value_type"] = 8
data["flg"] = 1
requests.append(UpdateOne(update, {'$set': data}, True))
isUpdate = True
update_count = update_count + 1
if row["avg_price"] > max_avg_price or row["avg_price"] < min_avg_price:
outer = True
if isUpdate:
district_stat.bulk_write(requests)
print(latest_stat)
common.send_refitted_trend(latest_stat["city"], latest_stat["region"], latest_stat["district_name"],
latest_stat["estate_type"], latest_stat["weekend"])
latest_stat["update_count"] = update_count
latest_stat["outer"] = outer
fill_command.save(latest_stat)
executor = ThreadPoolExecutor(100)
all_task = [executor.submit(padding, (city)) for city in common.get_all_city()]
wait(all_task, return_when=ALL_COMPLETED)
print("main")