250 lines
11 KiB
Python
250 lines
11 KiB
Python
"""
|
|
统计城市 区域 价格涨跌幅
|
|
"""
|
|
import logging
|
|
import pandas as pd
|
|
from sqlalchemy import create_engine
|
|
from goojia_utils import dateutils
|
|
import datetime
|
|
import os
|
|
pathAll = "/home/airflow/data/cityPrice"
|
|
if not os.path.exists(pathAll):
|
|
os.makedirs(pathAll)
|
|
# 创建数据库引擎
|
|
engine235 = create_engine("postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5433/fangjia?client_encoding=utf8",
|
|
echo=False)
|
|
pd.set_option('mode.chained_assignment', None)
|
|
year_month_day = dateutils.format_date(datetime.datetime.now())
|
|
# 执行SQL查询获取城市价格数据
|
|
city_sql = "SELECT DISTINCT urbn_nm FROM price_stat.avm_city_price"
|
|
city_names_set = set(pd.read_sql(city_sql, engine235)['urbn_nm'])
|
|
city_names = list(city_names_set)
|
|
# 假设这是你的pgsql数据
|
|
all_city_data = pd.DataFrame()
|
|
# 执行SQL查询获取区域价格数据
|
|
region_sql = "SELECT DISTINCT urbn_nm,gxda_admndiv_nm FROM price_stat.avm_district_price"
|
|
region_data = pd.read_sql(region_sql, engine235)
|
|
city_data_useinregion = pd.DataFrame()
|
|
# 假设这是你的pgsql数据
|
|
all_region_data = pd.DataFrame()
|
|
|
|
def up_to_nextcloud(aim_file_name,aim_file_path):
|
|
"""
|
|
上传统计信息至nextcloud
|
|
"""
|
|
from nextcloud import NextCloud
|
|
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
|
|
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
|
|
NEXTCLOUD_PASSWORD = "goojia123456"
|
|
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
|
|
nxc.upload_file(aim_file_name, aim_file_path)
|
|
def city_increase(city_names, all_city_data):
|
|
global city_data_useinregion # 声明为全局变量
|
|
# 遍历城市列表,对每个城市进行查询
|
|
for city in city_names:
|
|
query = f"SELECT urbn_nm as 城市, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_city_price WHERE urbn_nm = '{city}'"
|
|
city_data = pd.read_sql(query, engine235)
|
|
|
|
if city_data.empty:
|
|
continue
|
|
|
|
# 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变
|
|
city_data['original_date'] = city_data['月份']
|
|
|
|
# 将日期列转换为日期类型
|
|
city_data['月份'] = pd.to_datetime(city_data['月份'], format='%Y%m')
|
|
|
|
# 对价格数据按照月份排序
|
|
city_data.sort_values(by='月份', inplace=True)
|
|
|
|
# 计算涨跌幅
|
|
city_data['涨跌幅'] = city_data['估家计算均价'].diff() / city_data['估家计算均价'].shift(1) * 100
|
|
|
|
# 计算涨跌幅,并保留两位小数,展示为百分比形式
|
|
price_change = city_data['估家计算均价'].pct_change() * 100
|
|
city_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', '')
|
|
|
|
# 将原始日期列替换为计算涨跌幅后的日期列
|
|
city_data['月份'] = city_data['original_date']
|
|
|
|
# 删除临时列
|
|
city_data.drop(columns=['original_date'], inplace=True)
|
|
|
|
# 将城市数据添加到总的数据中
|
|
all_city_data = pd.concat([all_city_data, city_data], ignore_index=True)
|
|
city_data_useinregion = all_city_data
|
|
return all_city_data
|
|
|
|
|
|
def city_missing():
|
|
df = city_increase(city_names, all_city_data)
|
|
df['月份1'] = df['月份']
|
|
# 将日期字符串转换为日期时间对象
|
|
df["月份"] = pd.to_datetime(df["月份"], format="%Y%m")
|
|
|
|
# 获取当前时间
|
|
current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0)
|
|
|
|
# 计算前一年到上月的时间范围
|
|
one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1)
|
|
last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1)
|
|
|
|
# 创建一个包含所有可能月份的日期范围
|
|
all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS")
|
|
|
|
# 获取每个城市缺失的月份
|
|
missing_months = []
|
|
for city, group in df.groupby("城市"):
|
|
available_months = group["月份"]
|
|
missing = all_months.difference(available_months)
|
|
if len(missing) > 0:
|
|
missing_str = ",".join([m.strftime("%Y%m") for m in missing])
|
|
missing_months.append({"城市": city, "缺失月份": missing_str})
|
|
if len(missing_months) > 0:
|
|
# 创建DataFrame来存储结果
|
|
missing_df = pd.DataFrame(missing_months)
|
|
path = pathAll+'/城市价格数据缺失'+year_month_day+'.xlsx'
|
|
missing_df.to_excel(path, index=False)
|
|
logging.info(f"城市价格缺失数据已写入Excel文件:{path}")
|
|
|
|
cloudPath = '共享/统计/城市区域涨跌幅/' + '城市价格数据缺失'+year_month_day+'.xlsx'
|
|
up_to_nextcloud(path, cloudPath)
|
|
logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘")
|
|
|
|
# print(missing_df)
|
|
df['月份'] = df['月份1']
|
|
# 删除月份1列
|
|
df.drop(columns=['月份1'], inplace=True)
|
|
output_path = pathAll+'/城市价格涨跌幅'+year_month_day+'.xlsx'
|
|
df.to_excel(output_path, index=False)
|
|
logging.info(f"城市涨跌幅数据已写入Excel文件:{output_path}")
|
|
|
|
cloudPath = '共享/统计/城市区域涨跌幅/' + '城市价格涨跌幅'+year_month_day+'.xlsx'
|
|
up_to_nextcloud(output_path, cloudPath)
|
|
logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘")
|
|
|
|
|
|
def region_increase(region_data, all_region_data):
|
|
# 遍历区域列表,对每个区域进行查询
|
|
for index, row in region_data.iterrows():
|
|
city = row['urbn_nm']
|
|
region = row['gxda_admndiv_nm']
|
|
query = f"SELECT urbn_nm as 城市, gxda_admndiv_nm as 区域, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_district_price WHERE urbn_nm = '{city}' AND gxda_admndiv_nm = '{region}'"
|
|
region_data = pd.read_sql(query, engine235)
|
|
|
|
if region_data.empty:
|
|
continue
|
|
|
|
# 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变
|
|
region_data['original_date'] = region_data['月份']
|
|
|
|
# 将日期列转换为日期类型
|
|
region_data['月份'] = pd.to_datetime(region_data['月份'], format='%Y%m')
|
|
|
|
# 对价格数据按照月份排序
|
|
region_data.sort_values(by='月份', inplace=True)
|
|
|
|
# 计算涨跌幅
|
|
region_data['涨跌幅'] = region_data['估家计算均价'].diff() / region_data['估家计算均价'].shift(1) * 100
|
|
|
|
# 计算涨跌幅,并保留两位小数,展示为百分比形式
|
|
price_change = region_data['估家计算均价'].pct_change() * 100
|
|
region_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%',
|
|
'')
|
|
|
|
# 将原始日期列替换为计算涨跌幅后的日期列
|
|
region_data['月份'] = region_data['original_date']
|
|
|
|
# 删除临时列
|
|
region_data.drop(columns=['original_date'], inplace=True)
|
|
|
|
# 将区域数据添加到总的数据中
|
|
all_region_data = pd.concat([all_region_data, region_data], ignore_index=True)
|
|
|
|
return all_region_data
|
|
|
|
|
|
# 城市价个和区域价格比较
|
|
def add_price_comparison_by_month(region_data):
|
|
# 遍历区域数据,比较城市和区域的价格,并添加比值超过3倍的情况到区域数据中
|
|
for index, row in region_data.iterrows():
|
|
city = row['城市']
|
|
region = row['区域']
|
|
region_month = row['月份']
|
|
region_price = row['估家计算均价']
|
|
city_data = city_data_useinregion
|
|
# 获取城市对应月份的价格
|
|
city_price_row = city_data[(city_data['城市'] == city) & (city_data['月份'] == region_month)]
|
|
if city_price_row.empty:
|
|
continue
|
|
city_price = city_price_row['估家城市均价'].iloc[0]
|
|
|
|
# 比较价格并计算比值
|
|
price_ratio = max(city_price, region_price) / min(city_price, region_price)
|
|
|
|
# 如果比值超过3倍,将结果添加到区域数据中
|
|
if price_ratio > 3:
|
|
region_data.loc[index, '城市区域价格比值超3倍'] = price_ratio
|
|
|
|
return region_data
|
|
|
|
|
|
def region_missing():
|
|
df = region_increase(region_data, all_region_data)
|
|
df['月份1'] = df['月份']
|
|
# 将日期字符串转换为日期时间对象
|
|
df["月份"] = pd.to_datetime(df["月份"], format="%Y%m")
|
|
|
|
# 获取当前时间
|
|
current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0)
|
|
|
|
# 计算前一年到上月的时间范围
|
|
one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1)
|
|
last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1)
|
|
|
|
# 创建一个包含所有可能月份的日期范围
|
|
all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS")
|
|
|
|
# 获取每个城市-区域组合缺失的月份
|
|
missing_months = []
|
|
for (city, region), group in df.groupby(["城市", "区域"]):
|
|
available_months = group["月份"]
|
|
missing = all_months.difference(available_months)
|
|
if len(missing) > 0:
|
|
missing_str = ",".join([m.strftime("%Y%m") for m in missing])
|
|
missing_months.append({"城市": city, "区域": region, "缺失月份": missing_str})
|
|
|
|
if missing_months:
|
|
# 创建DataFrame来存储结果
|
|
missing_df = pd.DataFrame(missing_months)
|
|
path = pathAll+'/区域价格数据缺失'+year_month_day+'.xlsx'
|
|
missing_df.to_excel(path, index=False)
|
|
logging.info(f"区域价格缺失数据已写入Excel文件:{path}")
|
|
cloudPath = '共享/统计/城市区域涨跌幅/' + '区域价格数据缺失' + year_month_day + '.xlsx'
|
|
up_to_nextcloud(path, cloudPath)
|
|
logging.info(f"区域价格缺失数据Excel文件已经上传到云盘")
|
|
|
|
# 如果区域在 missing_df 中,则将 df 中的该区域数据删除
|
|
for city, region in zip(missing_df['城市'], missing_df['区域']):
|
|
df = df[(df['城市'] != city) | (df['区域'] != region)]
|
|
|
|
df['月份'] = df['月份1']
|
|
# 删除月份1列
|
|
df.drop(columns=['月份1'], inplace=True)
|
|
|
|
# 添加价格比值超过3倍的列
|
|
# print(city_data_useinregion)
|
|
df = add_price_comparison_by_month(df)
|
|
|
|
output_path = pathAll+'/区域价格涨跌幅'+year_month_day+'.xlsx'
|
|
df.to_excel(output_path, index=False)
|
|
logging.info(f"区域涨跌幅数据已写入Excel文件:{output_path}")
|
|
|
|
cloudPath = '共享/统计/城市区域涨跌幅/' + '区域价格涨跌幅'+year_month_day+'.xlsx'
|
|
up_to_nextcloud(output_path, cloudPath)
|
|
logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘")
|
|
|
|
|
|
city_missing()
|
|
region_missing()
|