Files
airflow/dags/jh/price/price_exception.py
T
2024-05-20 13:16:59 +08:00

158 lines
7.1 KiB
Python

"""
价格异常检测脚本
"""
from goojia_utils import dateutils, pgsql, MAIN_MONGO_CLIENT, data_utils
import pandas as pd
import logging
import os
import sys
import requests
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
# 获取独家小区
exclusive_seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0, 'mark_map.exclusive': '是'}, {'city': 1, 'region': 1, 'name': 1, 'mark_map.exclusive': 1}).batch_size(100000)))
exclusive_seaweed_df = data_utils.flatten_json(exclusive_seaweed_df, 'mark_map')
exclusive_seaweed_df = exclusive_seaweed_df.rename(columns={'name': 'commname', 'exclusive': '独家小区'})
def simple_calc(result_df):
"""
新增是否有样本列
:param result_df:
:return:
"""
month_before = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-2), '%Y-%m-%d')
simple_df = pd.DataFrame()
actions = list()
count = 0
for index, item in result_df.iterrows():
actions.append(item['commid'])
if len(actions) == 800:
ins = "','".join(actions)
ins = "'" + ins + "'"
simple_temp_df = pgsql.scan_by_sql_to_data_frame(
"select distinct gujia_comm_id as commid from base.t0831_bigdata_forsale_gj_h where " +
" anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")")
simple_df = pd.concat([simple_df, simple_temp_df])
actions.clear()
count += 800
if len(actions) > 0:
ins = "','".join(actions)
ins = "'" + ins + "'"
simple_temp_df = pgsql.scan_by_sql_to_data_frame(
"select distinct gujia_comm_id as commid from base.t0831_bigdata_forsale_gj_h where " +
" anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")")
simple_df = pd.concat([simple_df, simple_temp_df])
actions.clear()
simple_df['has_sample'] = 1
result_df = pd.merge(result_df, simple_df, on=['commid'], how='left')
return result_df
def send_msg():
infos = "【自动检测异常价格】异常价格数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情"
data = {
"msgtype": "text",
"text": {
"content": infos
}
}
# 发送消息至群
requests.post(
url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
json=data)
def up_to_nextcloud():
"""
上传统计信息至nextcloud
"""
from nextcloud import NextCloud
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
NEXTCLOUD_PASSWORD = "goojia123456"
nextcloud_file_name = '共享/建行/' + file_name
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
nxc.upload_file(aim_file_name, nextcloud_file_name)
log = logging.getLogger(__name__)
path = "/home/airflow/data/jh/price"
if not os.path.exists(path):
os.makedirs(path)
month_now = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now()), '%Y%m')
month_last = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-1), '%Y%m')
month_before = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-2), '%Y%m')
type = sys.argv[1] if len(sys.argv) > 1 else ''
log.info("输出类型" + type)
# 新增支持上一月与前月对比
if type == month_last:
month_now = month_last
month_last = month_before
df = pgsql.scan_by_sql_to_data_frame("SELECT gujia_bss_prc_id as id, urbn_nm as cityname, gujia_admndiv_nm as distname, gujia_bds_id as commid, comm_nm as commname, gujia_bds_clmtd as calc_type, gujia_bds_cfdc_dgr as confidence, gujia_comm_basic_prc as avg_price, gujiabds_clc_yrmo_val as year_month FROM t_app_p9pha_plda.avm_community_base_price_gj_h WHERE gujiabds_clc_yrmo_val IN('"+month_now+"', '"+month_last+"') " +
" ORDER BY urbn_nm, gujia_admndiv_nm, gujia_bds_id, comm_nm, gujia_bds_clmtd")
six_map = dict()
five_map = dict()
for index, item in df.iterrows():
year_month = item['year_month']
key = item['cityname'] + "," + item['distname'] + "," + item['commid'] + "," + item['commname']
if year_month == int(month_last):
five_map[key] = str(item['avg_price']) + "|" + str(item['calc_type']) + '|' + str(item['confidence'])
elif year_month == int(month_now):
six_map[key] = str(item['avg_price']) + "|" + str(item['calc_type']) + '|' + str(item['confidence'])
actions = list()
city_seaweed_dict = dict()
for key in six_map:
if key in five_map and key in six_map:
try:
five_price = float(five_map[key].split("|")[0])
six_price = float(six_map[key].split("|")[0])
if five_price and six_price:
info = key.split(",")
if abs(six_price - five_price) * 100 / five_price > 10:
actions.append({'city': info[0], 'region': info[1], 'commid': info[2], 'commname': info[3],
'five_price': five_price,'five_calc_type': five_map[key].split("|")[1],
'six_price': six_price,'six_calc_type': six_map[key].split("|")[1], '差异值': abs(six_price - five_price) * 100 / five_price,
'涨跌幅': (six_price - five_price) * 100 / five_price, 'confidence': six_map[key].split("|")[2]})
if info[0] not in city_seaweed_dict:
city_seaweed_dict[info[0]] = 1
else:
city_seaweed_dict[info[0]] += 1
except Exception as e:
log.error(e)
log.error(five_map[key] + ' ' + six_map[key])
result_df = pd.DataFrame(actions)
result_df['confidence'] = result_df['confidence'].astype(float)
file_name = month_now +'VS'+ month_last +'月价格波动大于10%小区.xlsx'
aim_file_name = path + '/' + file_name
excel_writer = pd.ExcelWriter(aim_file_name)
if type == 'sample':
result_df = simple_calc(result_df)
result_df = pd.merge(result_df, exclusive_seaweed_df, on=['city', 'region', 'commname'], how='left')
result_df = result_df.drop_duplicates(subset=['commid'], keep='first')
result_df = result_df.rename(columns={'city': '城市', 'region': '区域', 'commid': '小区ID', 'commname': '小区名', 'five_price': '上一月价格',
'five_calc_type': '上一月计算方式', 'six_price': '当月价格', 'six_calc_type': '当月计算方式', 'confidence': '置信度'})
result_df.to_excel(excel_writer, index=False, sheet_name='价格差异小区')
log.info(len(result_df))
city_count_df = result_df.groupby(by=["城市"])['小区ID'].count().reset_index()
for index, item in city_count_df.iterrows():
city_count_df.loc[index, '占比'] = round(item['小区ID'] * 100 / city_seaweed_dict[item['城市']], 2)
city_count_df = city_count_df.rename(columns={'小区ID': '小区数量'})
city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度')
#excel_writer.save()
excel_writer.close()
up_to_nextcloud()
send_msg()