Merge remote-tracking branch 'origin/master'
This commit is contained in:
@@ -20,5 +20,5 @@ for doc in es206_client.scan_by_sql("select code from fangjia_log where code >=
|
||||
count += 1
|
||||
if count > 2:
|
||||
print('【OPEN日志监控】最近两分钟超过两条错误,请检查日志!')
|
||||
dingutils.send_text('最近两分钟超过两条错误,请检查OPEN日志!', 'ab4231000995a8c90d8d1e91dfd6930c3896dd1c3bbf1f0af19e19004cf6c98c')
|
||||
dingutils.send_text('【OPEN日志监控】最近两分钟超过两条错误,请检查OPEN日志!', 'ab4231000995a8c90d8d1e91dfd6930c3896dd1c3bbf1f0af19e19004cf6c98c')
|
||||
break
|
||||
@@ -14,86 +14,92 @@ last_day = last.strftime('%Y%m%d')
|
||||
|
||||
log.info(f"系统当前时间 : {now}")
|
||||
|
||||
# 区域维度,统计2个月内样本数据量
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + str(
|
||||
last_day) + ' and p_date<' + str(now_day)
|
||||
log.info(f"城市区域维度 sql : {sql}")
|
||||
df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
df_region = pd.DataFrame(df.groupby(by=['city', 'region'])['_id'].count()).reset_index()
|
||||
df_region = df_region.rename(columns={'_id': 'count'})
|
||||
df_region_res = df_region[df_region['count'] < 20]
|
||||
def proc():
|
||||
# 区域维度,统计2个月内样本数据量
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + str(
|
||||
last_day) + ' and p_date<' + str(now_day)
|
||||
log.info(f"城市区域维度 sql : {sql}")
|
||||
df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
if df_region_res.empty == False:
|
||||
text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
|
||||
df_region = pd.DataFrame(df.groupby(by=['city', 'region'])['_id'].count()).reset_index()
|
||||
df_region = df_region.rename(columns={'_id': 'count'})
|
||||
df_region_res = df_region[df_region['count'] < 20]
|
||||
|
||||
# 计算2个月的日期,用来计算网站数据量减少百分比
|
||||
# 当前时间
|
||||
now = datetime.datetime.today()
|
||||
# 获取当前月份的第一天
|
||||
first_day_of_month = datetime.datetime(now.year, now.month, 1)
|
||||
# 获取1个月前的第一天
|
||||
last_month = first_day_of_month - datetime.timedelta(days=1)
|
||||
last_day_of_month = last_month.replace(day=1)
|
||||
# 获取2个月前的第一天
|
||||
last2_month = last_day_of_month - datetime.timedelta(days=1)
|
||||
last2_day_of_month = last2_month.replace(day=1)
|
||||
if df_region_res.empty == False:
|
||||
text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
|
||||
|
||||
strat_time = last_day_of_month.strftime('%Y%m%d')
|
||||
end_time = first_day_of_month.strftime('%Y%m%d')
|
||||
# 计算2个月的日期,用来计算网站数据量减少百分比
|
||||
# 当前时间
|
||||
now = datetime.datetime.today()
|
||||
# 获取当前月份的第一天
|
||||
first_day_of_month = datetime.datetime(now.year, now.month, 1)
|
||||
# 获取1个月前的第一天
|
||||
last_month = first_day_of_month - datetime.timedelta(days=1)
|
||||
last_day_of_month = last_month.replace(day=1)
|
||||
# 获取2个月前的第一天
|
||||
last2_month = last_day_of_month - datetime.timedelta(days=1)
|
||||
last2_day_of_month = last2_month.replace(day=1)
|
||||
|
||||
strat_time2 = last2_day_of_month.strftime('%Y%m%d')
|
||||
end_time2 = last_day_of_month.strftime('%Y%m%d')
|
||||
strat_time = last_day_of_month.strftime('%Y%m%d')
|
||||
end_time = first_day_of_month.strftime('%Y%m%d')
|
||||
|
||||
# 房源网站维度,按当月数量,和当月数量相比上月减少百分比
|
||||
sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time + ' and p_date<=' + end_time
|
||||
sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time2 + ' and p_date<=' + end_time2
|
||||
log.info(f"房源网站维度-百分比 当月sql : {sql1}")
|
||||
log.info(f"房源网站维度-百分比 上月sql : {sql2}")
|
||||
df_root1 = es206_client.scan_by_sql_to_data_frame(sql1)
|
||||
df_root2 = es206_client.scan_by_sql_to_data_frame(sql2)
|
||||
strat_time2 = last2_day_of_month.strftime('%Y%m%d')
|
||||
end_time2 = last_day_of_month.strftime('%Y%m%d')
|
||||
|
||||
df_root_group1 = pd.DataFrame(df_root1.groupby(by=['root'])['_id'].count()).reset_index()
|
||||
df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).reset_index()
|
||||
# 房源网站维度,按当月数量,和当月数量相比上月减少百分比
|
||||
sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time + ' and p_date<=' + end_time
|
||||
sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time2 + ' and p_date<=' + end_time2
|
||||
log.info(f"房源网站维度-百分比 当月sql : {sql1}")
|
||||
log.info(f"房源网站维度-百分比 上月sql : {sql2}")
|
||||
df_root1 = es206_client.scan_by_sql_to_data_frame(sql1)
|
||||
df_root2 = es206_client.scan_by_sql_to_data_frame(sql2)
|
||||
|
||||
df_root_group1 = df_root_group1.rename(columns={'_id': 'count'})
|
||||
df_root_group2 = df_root_group2.rename(columns={'_id': 'count'})
|
||||
df_root_group1 = pd.DataFrame(df_root1.groupby(by=['root'])['_id'].count()).reset_index()
|
||||
df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).reset_index()
|
||||
|
||||
# 房源网站,百分比维度,减少30%
|
||||
df_root_merge = pd.merge(df_root_group1, df_root_group2, how='left', on=['root'])
|
||||
df_root_group1 = df_root_group1.rename(columns={'_id': 'count'})
|
||||
df_root_group2 = df_root_group2.rename(columns={'_id': 'count'})
|
||||
|
||||
count_new = 'count_' + strat_time[0:6]
|
||||
count_old = 'count_' + strat_time2[0:6]
|
||||
# 房源网站,百分比维度,减少30%
|
||||
df_root_merge = pd.merge(df_root_group1, df_root_group2, how='left', on=['root'])
|
||||
|
||||
df_root_merge = df_root_merge.rename(columns={'count_x': count_new, 'count_y': count_old})
|
||||
count_new = 'count_' + strat_time[0:6]
|
||||
count_old = 'count_' + strat_time2[0:6]
|
||||
|
||||
df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge[count_new] < df_root_merge[count_old]) & (
|
||||
df_root_merge = df_root_merge.rename(columns={'count_x': count_new, 'count_y': count_old})
|
||||
|
||||
df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge[count_new] < df_root_merge[count_old]) & (
|
||||
(df_root_merge[count_old] - df_root_merge[count_new]) / df_root_merge[count_old] > 0.3))].index)
|
||||
|
||||
if df_root_res.empty == False:
|
||||
if text == '':
|
||||
text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
|
||||
if df_root_res.empty == False:
|
||||
if text == '':
|
||||
text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
|
||||
|
||||
# 房源网站,数量维度,前70个城市当月数量少于100
|
||||
# 房源网站,数量维度,前70个城市当月数量少于100
|
||||
|
||||
cities = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx")
|
||||
topCities = cities.head(70)
|
||||
city_list = list(topCities['cityname'])
|
||||
log.info(f"房源网站维度-数量 city_list : {city_list}")
|
||||
cities = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx")
|
||||
topCities = cities.head(70)
|
||||
city_list = list(topCities['cityname'])
|
||||
log.info(f"房源网站维度-数量 city_list : {city_list}")
|
||||
|
||||
df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root', 'city'])['_id'].count()).reset_index()
|
||||
df_root_city_group1 = df_root_city_group1.rename(columns={'_id': 'count'})
|
||||
df_root_city_res = df_root_city_group1[
|
||||
(df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count'] < 100)].reset_index()
|
||||
df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root', 'city'])['_id'].count()).reset_index()
|
||||
df_root_city_group1 = df_root_city_group1.rename(columns={'_id': 'count'})
|
||||
df_root_city_res = df_root_city_group1[
|
||||
(df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count'] < 100)].reset_index()
|
||||
|
||||
if df_root_city_res.empty == False:
|
||||
if text == '':
|
||||
text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
|
||||
if df_root_city_res.empty == False:
|
||||
if text == '':
|
||||
text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
|
||||
|
||||
path = "/home/airflow/data/jh/price"
|
||||
file_name = f"{now_day}房源样本数量检测.xlsx"
|
||||
air_file_name = path + '/' + file_name
|
||||
path = "/home/airflow/data/jh/price"
|
||||
file_name = f"{now_day}房源样本数量检测.xlsx"
|
||||
air_file_name = path + '/' + file_name
|
||||
return text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new
|
||||
|
||||
|
||||
text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new = proc()
|
||||
|
||||
|
||||
def up_to_nextcloud():
|
||||
@@ -111,12 +117,12 @@ def up_to_nextcloud():
|
||||
|
||||
if text != '':
|
||||
with pd.ExcelWriter(air_file_name) as writer:
|
||||
if df_region_res.empty == False:
|
||||
if not df_region_res.empty:
|
||||
df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city", "region", 'count'], index=False)
|
||||
if df_root_city_res.empty == False:
|
||||
if not df_root_city_res.empty:
|
||||
df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root', 'city', 'count'],
|
||||
index=False)
|
||||
if df_root_res.empty == False:
|
||||
if not df_root_res.empty:
|
||||
df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root', count_old, count_new],
|
||||
index=False)
|
||||
up_to_nextcloud()
|
||||
|
||||
Reference in New Issue
Block a user