版本修改
This commit is contained in:
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,136 @@
|
||||
"""
|
||||
统计小区数量、近半年第三方均价、近半年成交信息
|
||||
"""
|
||||
import logging
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
city_num = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
if city_num is not None:
|
||||
log.info("输出参数" + city_num)
|
||||
|
||||
df = pd.DataFrame()
|
||||
if city_num is not None and int(city_num) == 92:
|
||||
df = pd.read_excel('/opt/airflow/dags/repo/dags/stat/evaluate/data/估值92城市数据统计.xlsx')
|
||||
else:
|
||||
df = pd.read_excel('/opt/airflow/dags/repo/dags/stat/evaluate/data/估值数据统计.xlsx')
|
||||
year_month_day = dateutils.format_date(datetime.datetime.now())
|
||||
path = "/home/airflow/data/evaluate"
|
||||
aim_file_name = path + '/估值数据统计结果' + year_month_day + '.xlsx'
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat') # 成交
|
||||
average_statistics_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'average_statistics') # 第三方均价
|
||||
deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price')
|
||||
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'mapping_alias': 1})
|
||||
.batch_size(10000)))
|
||||
seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'})
|
||||
|
||||
cities = df[['城市']].drop_duplicates(keep='first')['城市'].values.tolist()
|
||||
|
||||
|
||||
def count_stat():
|
||||
"""
|
||||
统计小区数量
|
||||
"""
|
||||
result_df = pd.DataFrame(seaweed_conn.aggregate([
|
||||
{'$match': {'cat': 'district', 'status': 0}},
|
||||
{'$group': {'_id': '$city', 'community_count': {'$sum': 1}}}
|
||||
]))
|
||||
result_rename_df = result_df.rename(columns={'_id': '城市', 'community_count': '小区量'})
|
||||
new_df = pd.merge(df, result_rename_df, how='left', on=['城市'])
|
||||
new_df.to_excel(aim_file_name, index=False)
|
||||
|
||||
|
||||
def third_avg_stat():
|
||||
"""
|
||||
近半年有第三方均价小区数量及占比
|
||||
"""
|
||||
new_df = pd.read_excel(aim_file_name)
|
||||
s_date = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-180), fm='%Y%m')
|
||||
average_statistics_df = pd.DataFrame(
|
||||
list(average_statistics_conn.find({'s_date': {'$gte': int(s_date)}, 'category': 'district'}).batch_size(10000)))
|
||||
average_statistics_new_df = average_statistics_df[
|
||||
['city', 'name', 'region', '58tc_esf_price', 'cityhouse_esf_price', 'soufang_esf_price', 'lianjia_esf_price',
|
||||
'5i5j_esf_price', 'maitian_esf_price', '365house_esf_price', 'beike_esf_price', 'qfang_esf_price',
|
||||
'fangtan_esf_price',
|
||||
'tuboshi_esf_price', 'zhugezhaofang_esf_price', 'zhongyuan_esf_price', 'jiwu_esf_price', 'anjuke_esf_price']]
|
||||
|
||||
average_statistics_new_df.loc[:, 'has_esf_price'] = average_statistics_new_df['58tc_esf_price'] \
|
||||
.combine_first(average_statistics_new_df['cityhouse_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['soufang_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['lianjia_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['5i5j_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['maitian_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['365house_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['beike_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['qfang_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['fangtan_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['tuboshi_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['zhugezhaofang_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['zhongyuan_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['jiwu_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['anjuke_esf_price'])
|
||||
average_statistics_new_df.fillna(0, inplace=True)
|
||||
average_statistics_new_df.loc[:, ['flag']] = average_statistics_new_df['has_esf_price'].apply(
|
||||
lambda x: True if x > 0 else False)
|
||||
average_statistics_new_df = average_statistics_new_df[average_statistics_new_df['flag'] == True]
|
||||
average_statistics_new_df = average_statistics_new_df[['city', 'region', 'name']]
|
||||
average_statistics_new_df = average_statistics_new_df.drop_duplicates(keep='first')
|
||||
average_statistics_new_df['近半年有第三方均价小区数量'] = average_statistics_new_df.groupby('city')['city'].transform('size')
|
||||
average_statistics_new_df = average_statistics_new_df[['city', '近半年有第三方均价小区数量']].drop_duplicates(keep='first')
|
||||
average_statistics_new_df = average_statistics_new_df.rename(columns={'city': '城市'})
|
||||
new_df = pd.merge(new_df, average_statistics_new_df, how='left', on=['城市'])
|
||||
new_df['近半年有第三方均价小区占比'] = round(new_df['近半年有第三方均价小区数量'] * 100 / new_df['小区量'], 2)
|
||||
new_df.to_excel(aim_file_name, index=False)
|
||||
|
||||
|
||||
def deal_stat():
|
||||
"""
|
||||
近半年成交量及小区数量统计
|
||||
"""
|
||||
new_df = pd.read_excel(aim_file_name)
|
||||
# 小区别名升级为单独df
|
||||
actions = list()
|
||||
for index, item in seaweed_df.iterrows():
|
||||
if type(item['alias']) == list:
|
||||
for name in item['alias']:
|
||||
actions.append({'_id': item['_id'], 'city': item['city'], 'name': name, 'region': item['region']})
|
||||
alias_df = pd.DataFrame(actions)
|
||||
seaweed_alias_df = pd.concat([seaweed_df[['_id', 'city', 'region', 'name']], alias_df])
|
||||
deal_price_df = pd.DataFrame(list(deal_price_conn.find(
|
||||
{'trade_date': {'$gte': dateutils.add_months(dateutils.datetime.datetime.now(), days=-180)}},
|
||||
{'city': 1, 'region': 1, 'district_name': 1, '_id': 0}).batch_size(10000)))
|
||||
deal_price_df = deal_price_df.rename(columns={'district_name': 'name'})
|
||||
deal_price_seaweed_df = pd.merge(deal_price_df, seaweed_alias_df, how='left', on=['city', 'region', 'name'])
|
||||
# 剔除没有配到的小区
|
||||
deal_price_seaweed_notnull_df = deal_price_seaweed_df.dropna(subset=['_id'])
|
||||
deal_price_seaweed_notnull_df.loc[:, '近半年成交数据量'] = deal_price_seaweed_notnull_df.groupby(['city'])[
|
||||
'city'].transform('size')
|
||||
deal_price_half_year_df = deal_price_seaweed_notnull_df[['city', '近半年成交数据量']].drop_duplicates(keep='first')
|
||||
deal_price_half_year_df = deal_price_half_year_df.rename(columns={'city': '城市'})
|
||||
new_df = pd.merge(new_df, deal_price_half_year_df, how='left', on=['城市'])
|
||||
seaweed_half_year_df = deal_price_seaweed_notnull_df[['city', '_id']]
|
||||
seaweed_half_year_df = seaweed_half_year_df.drop_duplicates(subset=['city', '_id'], keep='first')
|
||||
seaweed_half_year_df['近半年有成交小区数量'] = seaweed_half_year_df.groupby(['city'])['city'].transform('size')
|
||||
seaweed_half_year_rename_df = seaweed_half_year_df[['city', '近半年有成交小区数量']].rename(columns={'city': '城市'})
|
||||
seaweed_half_year_rename_df = seaweed_half_year_rename_df.drop_duplicates(keep='first')
|
||||
new_df = pd.merge(new_df, seaweed_half_year_rename_df, how='left', on=['城市'])
|
||||
new_df.to_excel(aim_file_name, index=False)
|
||||
|
||||
|
||||
log.info("开始统计小区数量、近半年第三方均价、近半年成交信息")
|
||||
count_stat()
|
||||
# third_avg_stat()
|
||||
deal_stat()
|
||||
log.info("完成统计小区数量、近半年第三方均价、近半年成交信息")
|
||||
@@ -0,0 +1,137 @@
|
||||
"""
|
||||
统计近三个月、近一年样本量及小区属性
|
||||
"""
|
||||
import logging
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils, commutils
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
sell_num = sys.argv[1] if len(sys.argv) > 1 else "2"
|
||||
sell_num = int(sell_num)
|
||||
log.info("输出参数" + str(sell_num))
|
||||
|
||||
year_month_day = dateutils.format_date(datetime.datetime.now())
|
||||
aim_file_name = '/home/airflow/data/evaluate/估值数据统计结果' + year_month_day + '.xlsx'
|
||||
start_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-93))
|
||||
one_year_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-366))
|
||||
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'mapping_alias': 1})
|
||||
.batch_size(10000)))
|
||||
seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'})
|
||||
|
||||
|
||||
def get_sell(df, daysBefore=365):
|
||||
cities = df[['城市']].drop_duplicates(keep='first')['城市'].values.tolist()
|
||||
city_one_year_count = list()
|
||||
for city in cities:
|
||||
flag = True
|
||||
for item in city_one_year_count:
|
||||
if item['城市'] == city:
|
||||
flag = False
|
||||
break
|
||||
if flag == False:
|
||||
continue
|
||||
seaweed_temp_df = seaweed_df[seaweed_df['city'] == city]
|
||||
count = 0
|
||||
total_num = 0
|
||||
all_num = 0
|
||||
for index, item in seaweed_temp_df.iterrows():
|
||||
names = list()
|
||||
region = item['region']
|
||||
names.append(item['name'])
|
||||
if type(item['alias']) == list:
|
||||
for alias_item in item['alias']:
|
||||
names.append(alias_item)
|
||||
names_str = "','".join(names)
|
||||
names_str = "'" + names_str + "'"
|
||||
try:
|
||||
num = commutils.get_sample(city, region, names_str, daysBefore=daysBefore)
|
||||
if num > sell_num:
|
||||
count += 1
|
||||
total_num += num
|
||||
all_num += num
|
||||
except Exception as e:
|
||||
print(item)
|
||||
print(e)
|
||||
city_one_year_count.append({'城市': city, '近一年有样本小区数量': count, '近一年的样本总量': total_num, '总数量': all_num})
|
||||
print(city + "已处理完, 近一年有样本小区数量" + str(count) + " 近一年的样本总量" + str(total_num))
|
||||
|
||||
result_df = pd.DataFrame(city_one_year_count)
|
||||
return result_df
|
||||
|
||||
|
||||
def sell_stat():
|
||||
"""
|
||||
最近一年有样本小区占比 最近3月有样本小区占比
|
||||
"""
|
||||
df = pd.read_excel(aim_file_name)
|
||||
result_df = get_sell(df, 365)
|
||||
result_df = result_df[['城市', '近一年有样本小区数量', '近一年的样本总量']]
|
||||
df = pd.merge(df, result_df, how='left', on=['城市'])
|
||||
df['近一年有样本小区占比'] = round(df['近一年有样本小区数量'] * 100 / df['小区量'], 2)
|
||||
|
||||
result_df = get_sell(df, 93)
|
||||
result_df = result_df.rename(columns={'近一年有样本小区数量': '近三月有样本小区数量'})
|
||||
result_df = result_df[['城市', '近三月有样本小区数量']]
|
||||
df = pd.merge(df, result_df, how='left', on=['城市'])
|
||||
df['近三月有样本小区占比'] = round(df['近三月有样本小区数量'] * 100 / df['小区量'], 2)
|
||||
|
||||
df.to_excel(aim_file_name, index=False)
|
||||
|
||||
|
||||
def property_stat():
|
||||
"""
|
||||
建筑年代覆盖率 物业类型覆盖率 建筑类型覆盖率
|
||||
"""
|
||||
df = pd.read_excel(aim_file_name)
|
||||
df['建筑年代'] = None
|
||||
df['物业类型'] = None
|
||||
df['建筑类型'] = None
|
||||
for index, item in df.iterrows():
|
||||
city = item['城市']
|
||||
print(city)
|
||||
count = seaweed_conn.count(
|
||||
{'cat': 'district', 'status': 0, 'city': city, 'finish_building_date': {'$ne': None}})
|
||||
df.loc[index, '建筑年代'] = round(count * 100 / item['小区量'], 2)
|
||||
|
||||
estate_type2_count = seaweed_conn.count(
|
||||
{'cat': 'district', 'status': 0, 'city': city, 'estate_type2': {'$ne': None}})
|
||||
df.loc[index, '物业类型'] = round(estate_type2_count * 100 / item['小区量'], 2)
|
||||
|
||||
build_type_count = seaweed_conn.count(
|
||||
{'cat': 'district', 'status': 0, 'city': city, 'build_type': {'$ne': None}})
|
||||
df.loc[index, '建筑类型'] = round(build_type_count * 100 / item['小区量'], 2)
|
||||
df.to_excel(aim_file_name, index=False)
|
||||
|
||||
|
||||
def merge_aim():
|
||||
df = pd.read_excel(aim_file_name)
|
||||
need_df = pd.read_excel('/opt/airflow/dags/repo/dags/stat/evaluate/data/估值二期项目数据投标.xlsx')
|
||||
df = pd.merge(df, need_df, how='left', on=['城市'])
|
||||
df.to_excel(aim_file_name, index=False)
|
||||
|
||||
def up_to_nextcloud():
|
||||
"""
|
||||
上传统计信息至nextcloud
|
||||
"""
|
||||
from nextcloud import NextCloud
|
||||
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
|
||||
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
|
||||
NEXTCLOUD_PASSWORD = "goojia123456"
|
||||
aim_file_path = '共享/二期估值数据每周统计/估值数据统计结果' + year_month_day + '.xlsx'
|
||||
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
|
||||
nxc.upload_file(aim_file_name, aim_file_path)
|
||||
|
||||
log.info("开始统计统计近三个月、近一年样本量及小区属性")
|
||||
sell_stat()
|
||||
property_stat()
|
||||
merge_aim()
|
||||
log.info("完成统计统计近三个月、近一年样本量及小区属性")
|
||||
log.info("开始上传至云盘")
|
||||
up_to_nextcloud()
|
||||
log.info("云盘上传完成")
|
||||
Reference in New Issue
Block a user