From 668eb29a8dd00898d0fcf8782290f3096b99c7c2 Mon Sep 17 00:00:00 2001 From: mjt <13607600062@163.com> Date: Thu, 30 May 2024 17:19:13 +0800 Subject: [PATCH] =?UTF-8?q?=E5=9F=8E=E5=B8=82=E5=8C=BA=E5=9F=9F=E4=BB=B7?= =?UTF-8?q?=E6=A0=BC=E6=B6=A8=E8=B7=8C=E5=B9=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/stat/cityPrice/__init__.py | 0 .../increase_decrease_city_region.py | 249 ++++++++++++++++++ dags/stat/cityPrice/test.py | 97 +++++++ .../stat/dag_increase_decrease_city_region.py | 17 ++ 4 files changed, 363 insertions(+) create mode 100644 dags/stat/cityPrice/__init__.py create mode 100644 dags/stat/cityPrice/increase_decrease_city_region.py create mode 100644 dags/stat/cityPrice/test.py create mode 100644 dags/stat/dag_increase_decrease_city_region.py diff --git a/dags/stat/cityPrice/__init__.py b/dags/stat/cityPrice/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/stat/cityPrice/increase_decrease_city_region.py b/dags/stat/cityPrice/increase_decrease_city_region.py new file mode 100644 index 0000000..36581a0 --- /dev/null +++ b/dags/stat/cityPrice/increase_decrease_city_region.py @@ -0,0 +1,249 @@ +""" +统计城市 区域 价格涨跌幅 +""" +import logging +import pandas as pd +from sqlalchemy import create_engine +from goojia_utils import dateutils +import datetime +import os +pathAll = "/home/airflow/data/cityPrice" +if not os.path.exists(pathAll): + os.makedirs(pathAll) +# 创建数据库引擎 +engine235 = create_engine("postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5432/fangjia?client_encoding=utf8", + echo=False) +pd.set_option('mode.chained_assignment', None) +year_month_day = dateutils.format_date(datetime.datetime.now()) +# 执行SQL查询获取城市价格数据 +city_sql = "SELECT DISTINCT urbn_nm FROM price_stat.avm_city_price" +city_names_set = set(pd.read_sql(city_sql, engine235)['urbn_nm']) +city_names = list(city_names_set) +# 假设这是你的pgsql数据 +all_city_data = pd.DataFrame() +# 执行SQL查询获取区域价格数据 +region_sql = "SELECT DISTINCT urbn_nm,gxda_admndiv_nm FROM price_stat.avm_district_price" +region_data = pd.read_sql(region_sql, engine235) +city_data_useinregion = pd.DataFrame() +# 假设这是你的pgsql数据 +all_region_data = pd.DataFrame() + +def up_to_nextcloud(aim_file_name,aim_file_path): + """ + 上传统计信息至nextcloud + """ + from nextcloud import NextCloud + NEXTCLOUD_URL = "https://nextcloud.fangjia.com" + NEXTCLOUD_USERNAME = "shenjian@fangjia.com" + NEXTCLOUD_PASSWORD = "goojia123456" + nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True) + nxc.upload_file(aim_file_name, aim_file_path) +def city_increase(city_names, all_city_data): + global city_data_useinregion # 声明为全局变量 + # 遍历城市列表,对每个城市进行查询 + for city in city_names: + query = f"SELECT urbn_nm as 城市, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_city_price WHERE urbn_nm = '{city}'" + city_data = pd.read_sql(query, engine235) + + if city_data.empty: + continue + + # 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变 + city_data['original_date'] = city_data['月份'] + + # 将日期列转换为日期类型 + city_data['月份'] = pd.to_datetime(city_data['月份'], format='%Y%m') + + # 对价格数据按照月份排序 + city_data.sort_values(by='月份', inplace=True) + + # 计算涨跌幅 + city_data['涨跌幅'] = city_data['估家计算均价'].diff() / city_data['估家计算均价'].shift(1) * 100 + + # 计算涨跌幅,并保留两位小数,展示为百分比形式 + price_change = city_data['估家计算均价'].pct_change() * 100 + city_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', '') + + # 将原始日期列替换为计算涨跌幅后的日期列 + city_data['月份'] = city_data['original_date'] + + # 删除临时列 + city_data.drop(columns=['original_date'], inplace=True) + + # 将城市数据添加到总的数据中 + all_city_data = pd.concat([all_city_data, city_data], ignore_index=True) + city_data_useinregion = all_city_data + return all_city_data + + +def city_missing(): + df = city_increase(city_names, all_city_data) + df['月份1'] = df['月份'] + # 将日期字符串转换为日期时间对象 + df["月份"] = pd.to_datetime(df["月份"], format="%Y%m") + + # 获取当前时间 + current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0) + + # 计算前一年到上月的时间范围 + one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1) + last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1) + + # 创建一个包含所有可能月份的日期范围 + all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS") + + # 获取每个城市缺失的月份 + missing_months = [] + for city, group in df.groupby("城市"): + available_months = group["月份"] + missing = all_months.difference(available_months) + if len(missing) > 0: + missing_str = ",".join([m.strftime("%Y%m") for m in missing]) + missing_months.append({"城市": city, "缺失月份": missing_str}) + if len(missing_months) > 0: + # 创建DataFrame来存储结果 + missing_df = pd.DataFrame(missing_months) + path = pathAll+'/城市价格数据缺失'+year_month_day+'.xlsx' + missing_df.to_excel(path, index=False) + logging.info(f"城市价格缺失数据已写入Excel文件:{path}") + + cloudPath = '共享/统计/城市区域涨跌幅/' + '城市价格数据缺失'+year_month_day+'.xlsx' + up_to_nextcloud(path, cloudPath) + logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘") + + # print(missing_df) + df['月份'] = df['月份1'] + # 删除月份1列 + df.drop(columns=['月份1'], inplace=True) + output_path = pathAll+'/城市价格涨跌幅'+year_month_day+'.xlsx' + df.to_excel(output_path, index=False) + logging.info(f"城市涨跌幅数据已写入Excel文件:{output_path}") + + cloudPath = '共享/统计/城市区域涨跌幅/' + '城市价格涨跌幅'+year_month_day+'.xlsx' + up_to_nextcloud(output_path, cloudPath) + logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘") + + +def region_increase(region_data, all_region_data): + # 遍历区域列表,对每个区域进行查询 + for index, row in region_data.iterrows(): + city = row['urbn_nm'] + region = row['gxda_admndiv_nm'] + query = f"SELECT urbn_nm as 城市, gxda_admndiv_nm as 区域, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_district_price WHERE urbn_nm = '{city}' AND gxda_admndiv_nm = '{region}'" + region_data = pd.read_sql(query, engine235) + + if region_data.empty: + continue + + # 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变 + region_data['original_date'] = region_data['月份'] + + # 将日期列转换为日期类型 + region_data['月份'] = pd.to_datetime(region_data['月份'], format='%Y%m') + + # 对价格数据按照月份排序 + region_data.sort_values(by='月份', inplace=True) + + # 计算涨跌幅 + region_data['涨跌幅'] = region_data['估家计算均价'].diff() / region_data['估家计算均价'].shift(1) * 100 + + # 计算涨跌幅,并保留两位小数,展示为百分比形式 + price_change = region_data['估家计算均价'].pct_change() * 100 + region_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', + '') + + # 将原始日期列替换为计算涨跌幅后的日期列 + region_data['月份'] = region_data['original_date'] + + # 删除临时列 + region_data.drop(columns=['original_date'], inplace=True) + + # 将区域数据添加到总的数据中 + all_region_data = pd.concat([all_region_data, region_data], ignore_index=True) + + return all_region_data + + +# 城市价个和区域价格比较 +def add_price_comparison_by_month(region_data): + # 遍历区域数据,比较城市和区域的价格,并添加比值超过3倍的情况到区域数据中 + for index, row in region_data.iterrows(): + city = row['城市'] + region = row['区域'] + region_month = row['月份'] + region_price = row['估家计算均价'] + city_data = city_data_useinregion + # 获取城市对应月份的价格 + city_price_row = city_data[(city_data['城市'] == city) & (city_data['月份'] == region_month)] + if city_price_row.empty: + continue + city_price = city_price_row['估家城市均价'].iloc[0] + + # 比较价格并计算比值 + price_ratio = max(city_price, region_price) / min(city_price, region_price) + + # 如果比值超过3倍,将结果添加到区域数据中 + if price_ratio > 3: + region_data.loc[index, '城市区域价格比值超3倍'] = price_ratio + + return region_data + + +def region_missing(): + df = region_increase(region_data, all_region_data) + df['月份1'] = df['月份'] + # 将日期字符串转换为日期时间对象 + df["月份"] = pd.to_datetime(df["月份"], format="%Y%m") + + # 获取当前时间 + current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0) + + # 计算前一年到上月的时间范围 + one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1) + last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1) + + # 创建一个包含所有可能月份的日期范围 + all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS") + + # 获取每个城市-区域组合缺失的月份 + missing_months = [] + for (city, region), group in df.groupby(["城市", "区域"]): + available_months = group["月份"] + missing = all_months.difference(available_months) + if len(missing) > 0: + missing_str = ",".join([m.strftime("%Y%m") for m in missing]) + missing_months.append({"城市": city, "区域": region, "缺失月份": missing_str}) + + if missing_months: + # 创建DataFrame来存储结果 + missing_df = pd.DataFrame(missing_months) + path = pathAll+'/区域价格数据缺失'+year_month_day+'.xlsx' + missing_df.to_excel(path, index=False) + logging.info(f"区域价格缺失数据已写入Excel文件:{path}") + cloudPath = '共享/统计/城市区域涨跌幅/' + '区域价格数据缺失' + year_month_day + '.xlsx' + up_to_nextcloud(path, cloudPath) + logging.info(f"区域价格缺失数据Excel文件已经上传到云盘") + + # 如果区域在 missing_df 中,则将 df 中的该区域数据删除 + for city, region in zip(missing_df['城市'], missing_df['区域']): + df = df[(df['城市'] != city) | (df['区域'] != region)] + + df['月份'] = df['月份1'] + # 删除月份1列 + df.drop(columns=['月份1'], inplace=True) + + # 添加价格比值超过3倍的列 + # print(city_data_useinregion) + df = add_price_comparison_by_month(df) + + output_path = pathAll+'/区域价格涨跌幅'+year_month_day+'.xlsx' + df.to_excel(output_path, index=False) + logging.info(f"区域涨跌幅数据已写入Excel文件:{output_path}") + + cloudPath = '共享/统计/城市区域涨跌幅/' + '区域价格涨跌幅'+year_month_day+'.xlsx' + up_to_nextcloud(output_path, cloudPath) + logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘") + + +city_missing() +region_missing() diff --git a/dags/stat/cityPrice/test.py b/dags/stat/cityPrice/test.py new file mode 100644 index 0000000..4e08b92 --- /dev/null +++ b/dags/stat/cityPrice/test.py @@ -0,0 +1,97 @@ +import logging +import pandas as pd +from sqlalchemy import create_engine + +# 创建数据库引擎 +engine235 = create_engine("postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5432/fangjia?client_encoding=utf8", + echo=False) + +# 执行SQL查询获取区域价格数据 +region_sql = "SELECT DISTINCT urbn_nm,gxda_admndiv_nm FROM price_stat.avm_district_price_copy1" +region_data = pd.read_sql(region_sql, engine235) + +# 假设这是你的pgsql数据 +all_region_data = pd.DataFrame() +pd.set_option('mode.chained_assignment', None) + +def region_increase(region_data, all_region_data): + # 遍历区域列表,对每个区域进行查询 + for index, row in region_data.iterrows(): + city = row['urbn_nm'] + region = row['gxda_admndiv_nm'] + query = f"SELECT urbn_nm as 城市, gxda_admndiv_nm as 区域, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_district_price_copy1 WHERE urbn_nm = '{city}' AND gxda_admndiv_nm = '{region}'" + region_data = pd.read_sql(query, engine235) + + if region_data.empty: + continue + + # 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变 + region_data['original_date'] = region_data['月份'] + + # 将日期列转换为日期类型 + region_data['月份'] = pd.to_datetime(region_data['月份'], format='%Y%m') + + # 对价格数据按照月份排序 + region_data.sort_values(by='月份', inplace=True) + + # 计算涨跌幅 + region_data['涨跌幅'] = region_data['估家计算均价'].diff() / region_data['估家计算均价'].shift(1) * 100 + + # 计算涨跌幅,并保留两位小数,展示为百分比形式 + price_change = region_data['估家计算均价'].pct_change() * 100 + region_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', '') + + # 将原始日期列替换为计算涨跌幅后的日期列 + region_data['月份'] = region_data['original_date'] + + # 删除临时列 + region_data.drop(columns=['original_date'], inplace=True) + + # 将区域数据添加到总的数据中 + all_region_data = pd.concat([all_region_data, region_data], ignore_index=True) + + return all_region_data + +def region_missing(): + df = region_increase(region_data, all_region_data) + df['月份1'] = df['月份'] + # 将日期字符串转换为日期时间对象 + df["月份"] = pd.to_datetime(df["月份"], format="%Y%m") + + # 获取当前时间 + current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0) + + # 计算前一年到上月的时间范围 + one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1) + last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1) + + # 创建一个包含所有可能月份的日期范围 + all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS") + + # 获取每个城市-区域组合缺失的月份 + missing_months = [] + for (city, region), group in df.groupby(["城市", "区域"]): + available_months = group["月份"] + missing = all_months.difference(available_months) + if len(missing) > 0: + missing_str = ",".join([m.strftime("%Y%m") for m in missing]) + missing_months.append({"城市": city, "区域": region, "缺失月份": missing_str}) + + if missing_months: + # 创建DataFrame来存储结果 + missing_df = pd.DataFrame(missing_months) + path = 'E:\\goojia\\file\\excel文件\\airflow\\missing_region.xlsx' + missing_df.to_excel(path, index=False) + + # 如果区域在 missing_df 中,则将 df 中的该区域数据删除 + for city, region in zip(missing_df['城市'], missing_df['区域']): + df = df[(df['城市'] != city) | (df['区域'] != region)] + + df['月份'] = df['月份1'] + # 删除月份1列 + df.drop(columns=['月份1'], inplace=True) + output_path = 'E:\\goojia\\file\\excel文件\\airflow\\all_region_price_with_change.xlsx' + df.to_excel(output_path, index=False) + logging.info(f"区域涨跌幅数据已写入Excel文件:{output_path}") + +region_missing() diff --git a/dags/stat/dag_increase_decrease_city_region.py b/dags/stat/dag_increase_decrease_city_region.py new file mode 100644 index 0000000..e048931 --- /dev/null +++ b/dags/stat/dag_increase_decrease_city_region.py @@ -0,0 +1,17 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + dag_id="increase_decrease_city_region_dag", + # 每月的最后一天运行 + schedule_interval="0 10 L * *", + start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"), + description="常用字段覆盖率统计", + catchup=True +) as dag: + t1 = BashOperator( + task_id='increase_decrease_city_region', + bash_command='python /opt/airflow/dags/repo/dags/stat/cityPrice/increase_decrease_city_region.py', + ) + t1