Files
airflow/dags/stat/evaluate/district_sell_property.py
T
2022-08-16 15:10:34 +08:00

152 lines
7.1 KiB
Python

"""
统计近三个月、近一年样本量及小区属性
"""
import logging
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils
import pandas as pd
import datetime
import sys
log = logging.getLogger(__name__)
sell_num = sys.argv[1] if len(sys.argv) > 1 else "2"
sell_num = int(sell_num)
log.info("输出参数" + str(sell_num))
year_month_day = dateutils.format_date(datetime.datetime.now())
aim_file_name = '/home/airflow/data/evaluate/估值数据统计结果' + year_month_day + '.xlsx'
start_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-93))
one_year_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-366))
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'mapping_alias': 1})
.batch_size(10000)))
seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'})
def sell_stat():
"""
最近一年有样本小区占比 最近3月有样本小区占比
"""
df = pd.read_excel(aim_file_name)
sell_seaweed_df = es206_client.scan_by_sql_to_data_frame(
"select city, region, district_name, p_date from latest_seawater where p_date > " + start_time + " and cat='sell' ")
# 小区别名升级为单独df
actions = list()
for index, item in seaweed_df.iterrows():
if type(item['alias']) == list:
for name in item['alias']:
actions.append({'_id': item['_id'], 'city': item['city'], 'name': name, 'region': item['region']})
alias_df = pd.DataFrame(actions)
sell_seaweed_rename_df = sell_seaweed_df[['city', 'region', 'district_name']]
sell_seaweed_rename_df = sell_seaweed_rename_df.rename(columns={'district_name': 'name'})
sell_seaweed_merge_df = pd.merge(sell_seaweed_rename_df, alias_df, how='left', on=['city', 'region', 'name'])
sell_seaweed_merge_df['count'] = sell_seaweed_merge_df[sell_seaweed_merge_df['_id'].notna()].groupby('_id')[
'_id'].transform('size')
sell_seaweed_merge_df = sell_seaweed_merge_df[['city', 'region', '_id', 'count']]
sell_seaweed_merge_df = sell_seaweed_merge_df.drop_duplicates(subset=['city', 'region', '_id'])
sell_seaweed_ok_df = sell_seaweed_merge_df[sell_seaweed_merge_df['count'] > sell_num]
sell_seaweed_ok_df['city_seaweed_count'] = sell_seaweed_ok_df.groupby('city')['city'].transform('size')
city_seaweed_count_df = sell_seaweed_ok_df[['city', 'city_seaweed_count']].drop_duplicates(
subset=['city', 'city_seaweed_count'])
city_seaweed_count_df = city_seaweed_count_df.rename(columns={'city': '城市', 'city_seaweed_count': '最近3月有样本小区占比'})
df = pd.merge(df, city_seaweed_count_df, how='left', on=['城市'])
df['最近3月有样本小区占比'] = round(df['最近3月有样本小区占比'] * 100 / df['小区量'], 2)
df.to_excel(aim_file_name, index=False)
cities = df[['城市']].drop_duplicates(keep='first')['城市'].values.tolist()
city_one_year_count = list()
for city in cities:
flag = True
for item in city_one_year_count:
if item['城市'] == city:
flag = False
break
if flag == False:
continue
seaweed_temp_df = seaweed_df[seaweed_df['city'] == city]
count = 0
total_num = 0
all_num = 0
for index, item in seaweed_temp_df.iterrows():
names = list()
region = item['region']
names.append(item['name'])
if type(item['alias']) == list:
for alias_item in item['alias']:
names.append(alias_item)
names_str = "','".join(names)
names_str = "'" + names_str + "'"
try:
sql = "select count(*) from latest_seawater where city='" + city + "' and region='" + region + "' and cat='sell' and district_name in(" + names_str + ") and p_date > " + one_year_time
num = es206_client.search_by_sql(sql)['hits']['total']['value']
if num > sell_num:
count += 1
total_num += num
all_num += num
except Exception as e:
print(item)
print(e)
city_one_year_count.append({'城市': city, '近一年有样本小区数量': count, '近一年的样本总量': total_num, '总数量': all_num})
print(city + "已处理完, 近一年有样本小区数量" + str(count) + " 近一年的样本总量" + str(total_num))
result_df = pd.DataFrame(city_one_year_count)
result_df = result_df[['城市', '近一年有样本小区数量', '近一年的样本总量']]
df = pd.merge(df, result_df, how='left', on=['城市'])
df['最近一年有样本小区占比'] = round(df['近一年有样本小区数量'] * 100 / df['小区量'], 2)
df.to_excel(aim_file_name, index=False)
def property_stat():
"""
建筑年代覆盖率 物业类型覆盖率 建筑类型覆盖率
"""
df = pd.read_excel(aim_file_name)
df['建筑年代'] = None
df['物业类型'] = None
df['建筑类型'] = None
for index, item in df.iterrows():
city = item['城市']
print(city)
count = seaweed_conn.count(
{'cat': 'district', 'status': 0, 'city': city, 'finish_building_date': {'$ne': None}})
df.loc[index, '建筑年代'] = round(count * 100 / item['小区量'], 2)
estate_type2_count = seaweed_conn.count(
{'cat': 'district', 'status': 0, 'city': city, 'estate_type2': {'$ne': None}})
df.loc[index, '物业类型'] = round(estate_type2_count * 100 / item['小区量'], 2)
build_type_count = seaweed_conn.count(
{'cat': 'district', 'status': 0, 'city': city, 'build_type': {'$ne': None}})
df.loc[index, '建筑类型'] = round(build_type_count * 100 / item['小区量'], 2)
df.to_excel(aim_file_name, index=False)
def merge_aim():
df = pd.read_excel(aim_file_name)
need_df = pd.read_excel('/opt/airflow/dags/repo/dags/stat/evaluate/data/估值二期项目数据投标.xlsx')
df = pd.merge(df, need_df, how='left', on=['城市'])
df.to_excel(aim_file_name, index=False)
def up_to_nextcloud():
"""
上传统计信息至nextcloud
"""
from nextcloud import NextCloud
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
NEXTCLOUD_PASSWORD = "goojia123456"
aim_file_path = '共享/二期估值数据每周统计/估值数据统计结果' + year_month_day + '.xlsx'
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
nxc.upload_file(aim_file_name, aim_file_path)
log.info("开始统计统计近三个月、近一年样本量及小区属性")
sell_stat()
property_stat()
merge_aim()
log.info("完成统计统计近三个月、近一年样本量及小区属性")
log.info("开始上传至云盘")
up_to_nextcloud()
log.info("云盘上传完成")