版本修改
This commit is contained in:
@@ -10,34 +10,8 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
city_num = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
if city_num is not None:
|
||||
log.info("输出参数" + city_num)
|
||||
|
||||
df = pd.DataFrame()
|
||||
if city_num is not None and int(city_num) == 92:
|
||||
df = pd.read_excel('/opt/airflow/dags/repo/dags/stat/evaluate/data/估值92城市数据统计.xlsx')
|
||||
else:
|
||||
df = pd.read_excel('/opt/airflow/dags/repo/dags/stat/evaluate/data/估值数据统计.xlsx')
|
||||
year_month_day = dateutils.format_date(datetime.datetime.now())
|
||||
path = "/home/airflow/data/evaluate"
|
||||
aim_file_name = path + '/估值数据统计结果' + year_month_day + '.xlsx'
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat') # 成交
|
||||
average_statistics_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'average_statistics') # 第三方均价
|
||||
deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price')
|
||||
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'mapping_alias': 1})
|
||||
.batch_size(10000)))
|
||||
seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'})
|
||||
|
||||
cities = df[['城市']].drop_duplicates(keep='first')['城市'].values.tolist()
|
||||
|
||||
# ================== 核心统计函数(在全局定义,供 main 调用) ==================
|
||||
|
||||
def count_stat():
|
||||
"""
|
||||
@@ -128,9 +102,50 @@ def deal_stat():
|
||||
new_df = pd.merge(new_df, seaweed_half_year_rename_df, how='left', on=['城市'])
|
||||
new_df.to_excel(aim_file_name, index=False)
|
||||
|
||||
# ================== 🛡️ 安全运行入口 ==================
|
||||
|
||||
log.info("开始统计小区数量、近半年第三方均价、近半年成交信息")
|
||||
count_stat()
|
||||
# third_avg_stat()
|
||||
deal_stat()
|
||||
log.info("完成统计小区数量、近半年第三方均价、近半年成交信息")
|
||||
if __name__ == "__main__":
|
||||
# 1. 只有以脚本形式运行时,才解析命令行参数
|
||||
city_num = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
if city_num is not None:
|
||||
log.info("输出参数" + city_num)
|
||||
|
||||
# 2. 读取 Excel 配置
|
||||
df = pd.DataFrame()
|
||||
# 增加额外的 try-except 容错,防止意外传入非数字字符串(比如在别处不小心执行时)
|
||||
try:
|
||||
if city_num is not None and int(city_num) == 92:
|
||||
df = pd.read_excel('/opt/airflow/dags/repo/dags/stat/evaluate/data/估值92城市数据统计.xlsx')
|
||||
else:
|
||||
df = pd.read_excel('/opt/airflow/dags/repo/dags/stat/evaluate/data/估值数据统计.xlsx')
|
||||
except ValueError:
|
||||
log.warning(f"无法将参数 '{city_num}' 转换为整数,将使用默认数据源进行统计。")
|
||||
df = pd.read_excel('/opt/airflow/dags/repo/dags/stat/evaluate/data/估值数据统计.xlsx')
|
||||
|
||||
year_month_day = dateutils.format_date(datetime.datetime.now())
|
||||
path = "/home/airflow/data/evaluate"
|
||||
aim_file_name = path + '/估值数据统计结果' + year_month_day + '.xlsx'
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
|
||||
# 3. 延迟初始化数据库连接(避免在 Airflow 解析阶段浪费连接资源)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat') # 成交
|
||||
average_statistics_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'average_statistics') # 第三方均价
|
||||
deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price')
|
||||
|
||||
# 4. 加载 MongoDB 数据
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'mapping_alias': 1})
|
||||
.batch_size(10000)))
|
||||
seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'})
|
||||
|
||||
cities = df[['城市']].drop_duplicates(keep='first')['城市'].values.tolist()
|
||||
|
||||
# 5. 执行主统计逻辑
|
||||
log.info("开始统计小区数量、近半年第三方均价、近半年成交信息")
|
||||
count_stat()
|
||||
# third_avg_stat()
|
||||
deal_stat()
|
||||
log.info("完成统计小区数量、近半年第三方均价、近半年成交信息")
|
||||
@@ -8,22 +8,9 @@ import datetime
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
sell_num = sys.argv[1] if len(sys.argv) > 1 else "2"
|
||||
sell_num = int(sell_num)
|
||||
log.info("输出参数" + str(sell_num))
|
||||
|
||||
year_month_day = dateutils.format_date(datetime.datetime.now())
|
||||
aim_file_name = '/home/airflow/data/evaluate/估值数据统计结果' + year_month_day + '.xlsx'
|
||||
start_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-93))
|
||||
one_year_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-366))
|
||||
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'mapping_alias': 1})
|
||||
.batch_size(10000)))
|
||||
seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'})
|
||||
|
||||
# ================== 核心函数定义(供 main 调用) ==================
|
||||
|
||||
def get_sell(df, daysBefore=365):
|
||||
cities = df[['城市']].drop_duplicates(keep='first')['城市'].values.tolist()
|
||||
@@ -115,6 +102,7 @@ def merge_aim():
|
||||
df = pd.merge(df, need_df, how='left', on=['城市'])
|
||||
df.to_excel(aim_file_name, index=False)
|
||||
|
||||
|
||||
def up_to_nextcloud():
|
||||
"""
|
||||
上传统计信息至nextcloud
|
||||
@@ -127,11 +115,40 @@ def up_to_nextcloud():
|
||||
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
|
||||
nxc.upload_file(aim_file_name, aim_file_path)
|
||||
|
||||
log.info("开始统计统计近三个月、近一年样本量及小区属性")
|
||||
sell_stat()
|
||||
property_stat()
|
||||
merge_aim()
|
||||
log.info("完成统计统计近三个月、近一年样本量及小区属性")
|
||||
log.info("开始上传至云盘")
|
||||
up_to_nextcloud()
|
||||
log.info("云盘上传完成")
|
||||
|
||||
# ================== 🛡️ 安全运行入口 ==================
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 1. 只有脚本运行时,才解析命令行参数并转为整数
|
||||
sell_num_arg = sys.argv[1] if len(sys.argv) > 1 else "2"
|
||||
try:
|
||||
sell_num = int(sell_num_arg)
|
||||
except ValueError:
|
||||
log.warning(f"无法将参数 '{sell_num_arg}' 转换为整数,将使用默认值 2。")
|
||||
sell_num = 2
|
||||
log.info("输出参数" + str(sell_num))
|
||||
|
||||
# 2. 统计时间范围及存储文件名定义
|
||||
year_month_day = dateutils.format_date(datetime.datetime.now())
|
||||
aim_file_name = '/home/airflow/data/evaluate/估值数据统计结果' + year_month_day + '.xlsx'
|
||||
start_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-93))
|
||||
one_year_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-366))
|
||||
|
||||
# 3. 延迟数据库初始化(避开 Airflow 解析器扫描)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
# 4. 加载基础数据
|
||||
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'mapping_alias': 1})
|
||||
.batch_size(10000)))
|
||||
seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'})
|
||||
|
||||
# 5. 开始核心统计与上传逻辑
|
||||
log.info("开始统计统计近三个月、近一年样本量及小区属性")
|
||||
sell_stat()
|
||||
property_stat()
|
||||
merge_aim()
|
||||
log.info("完成统计统计近三个月、近一年样本量及小区属性")
|
||||
log.info("开始上传至云盘")
|
||||
up_to_nextcloud()
|
||||
log.info("云盘上传完成")
|
||||
Reference in New Issue
Block a user