版本修改

This commit is contained in:
王文艳
2026-07-22 16:06:26 +08:00
parent eddb99892a
commit a11383312e
@@ -9,6 +9,27 @@ import logging
log = logging.getLogger(__name__)
# ================== 辅助上传函数(全局定义,供 main 调用) ==================
def up_to_nextcloud():
"""
上传统计信息至nextcloud
"""
from nextcloud import NextCloud
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
NEXTCLOUD_PASSWORD = "goojia123456"
# 使用运行时生成的 output_date 和 aim_file_name 全局变量
aim_file_path = '共享/统计/常用覆盖率统计' + output_date + '.xlsx'
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
nxc.upload_file(aim_file_name, aim_file_path)
# ================== 🛡️ 安全运行入口 ==================
if __name__ == "__main__":
# 1. 延迟初始化数据库连接(避开 Airflow 进程的加载期)
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat')
@@ -18,12 +39,14 @@ deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price')
es206_client = ElasticSearchClient(host='192.168.0.206')
# 2. 读取城市列表与清洗
city_df = pd.DataFrame(list(server_area_conn.find({}, {'city_grade': 1, 'display': 1})))
city_df = city_df.rename(columns={'city_grade': 'city_level', 'display': 'city'})
city_df = city_df[city_df['city_level'] != 0]
# 昆玉无小区
city_df = city_df[city_df['city'] != '昆玉'].reset_index(drop=True)
# 3. 巨量 MongoDB 查询与路径准备
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'cat': 'district', 'status': 0},
{'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'addr_alias': 1})
@@ -37,6 +60,7 @@ if not os.path.exists(aim_path):
ret_df = pd.read_excel(path + '/财力评级分布情况20221215.xlsx')
# 4. 双层大循环处理小区与别名
for index, item in city_df.iterrows():
city = item['city']
region_df = seaweed_df[(seaweed_df['city'] == city)].copy()
@@ -66,26 +90,9 @@ for index, item in city_df.iterrows():
# 有别名小区量【存在楼盘别名或地址别名】
city_df.loc[index, 'exist_alias_community_count'] = seaweed_count
# 【已过期】
# #### 1. 小区数量
# result_df = pd.DataFrame(list(seaweed_conn.aggregate([
# {'$match': {'cat': 'district', 'status': 0}},
# {'$group': {'_id': "$city", 'community_count': {'$sum': 1}}}])))
#
#
# result_df = result_df.rename(columns={'_id': 'city'})
# df = pd.merge(result_df, city_df, how='right', on='city')
# df.fillna(0, inplace=True)
# 有别名小区量 该方式统计不准,可能存在[]情况也算了
# alias_df = pd.DataFrame(list(seaweed_conn.aggregate([
# {'$match': {'cat': 'district', 'status': 0, 'mapping_alias': {'$ne': None}}},
# {'$group': {'_id': "$city", 'alias_count': {'$sum': 1}}}])))
# alias_df = alias_df.rename(columns={'_id': 'city'})
# df = pd.merge(df, alias_df, how='left', on=['city'])
df = city_df.copy()
# 5. 主力属性的多项重度 MongoDB Aggregation 统计
# 主力户型
main_type_count = pd.DataFrame(list(seaweed_conn.aggregate([
{'$match': {'cat': 'district', 'status': 0, 'main_type': {'$ne': None}}},
@@ -93,6 +100,7 @@ main_type_count = pd.DataFrame(list(seaweed_conn.aggregate([
main_type_count = main_type_count.rename(columns={'_id': 'city'})
df = pd.merge(df, main_type_count, how='left', on=['city'])
df['main_type_coverage'] = df['main_type_count'] * 100 / df['community_count']
# 主力面积
main_type_area_count = pd.DataFrame(list(seaweed_conn.aggregate([
{'$match': {'cat': 'district', 'status': 0, 'main_type_area': {'$ne': None}}},
@@ -100,6 +108,7 @@ main_type_area_count = pd.DataFrame(list(seaweed_conn.aggregate([
main_type_area_count = main_type_area_count.rename(columns={'_id': 'city'})
df = pd.merge(df, main_type_area_count, how='left', on=['city'])
df['main_type_area_coverage'] = df['main_type_area_count'] * 100 / df['community_count']
# 物业类型
estate_type_count = pd.DataFrame(list(seaweed_conn.aggregate([
{'$match': {'cat': 'district', 'status': 0, 'estate_type2': {'$ne': None}}},
@@ -107,6 +116,7 @@ estate_type_count = pd.DataFrame(list(seaweed_conn.aggregate([
estate_type_count = estate_type_count.rename(columns={'_id': 'city'})
df = pd.merge(df, estate_type_count, how='left', on=['city'])
df['estate_type_coverage'] = df['estate_type_count'] * 100 / df['community_count']
# 建成年代
finish_building_date_count = pd.DataFrame(list(seaweed_conn.aggregate([
{'$match': {'cat': 'district', 'status': 0, 'finish_building_date': {'$ne': None}}},
@@ -114,6 +124,7 @@ finish_building_date_count = pd.DataFrame(list(seaweed_conn.aggregate([
finish_building_date_count = finish_building_date_count.rename(columns={'_id': 'city'})
df = pd.merge(df, finish_building_date_count, how='left', on=['city'])
df['finish_building_date_coverage'] = df['finish_building_date_count'] * 100 / df['community_count']
# 占地面积
all_area_count = pd.DataFrame(list(seaweed_conn.aggregate([
{'$match': {'cat': 'district', 'status': 0, 'all_area': {'$ne': None}}},
@@ -121,6 +132,7 @@ all_area_count = pd.DataFrame(list(seaweed_conn.aggregate([
all_area_count = all_area_count.rename(columns={'_id': 'city'})
df = pd.merge(df, all_area_count, how='left', on=['city'])
df['all_area_coverage'] = df['all_area_count'] * 100 / df['community_count']
# 建筑面积
build_area_count = pd.DataFrame(list(seaweed_conn.aggregate([
{'$match': {'cat': 'district', 'status': 0, 'build_area': {'$ne': None}}},
@@ -136,6 +148,7 @@ greening_rate_count = pd.DataFrame(list(seaweed_conn.aggregate([
greening_rate_count = greening_rate_count.rename(columns={'_id': 'city'})
df = pd.merge(df, greening_rate_count, how='left', on=['city'])
df['greening_rate_coverage'] = df['greening_rate_count'] * 100 / df['community_count']
# 容积率
volume_rate_count = pd.DataFrame(list(seaweed_conn.aggregate([
{'$match': {'cat': 'district', 'status': 0, 'cubage_percent': {'$ne': None}}},
@@ -144,13 +157,10 @@ volume_rate_count = volume_rate_count.rename(columns={'_id': 'city'})
df = pd.merge(df, volume_rate_count, how='left', on=['city'])
df['volume_rate_coverage'] = df['volume_rate_count'] * 100 / df['community_count']
# 户均总价
# 6. 人均/财力等 Elasticsearch 批量循环查询
df['total_price_count'] = None
# 物业费
df['estate_charge_count'] = None
# 人均消费
df['avg_cost_count'] = None
# 财力评级
df['consumption_rating_count'] = None
for index, item in df.iterrows():
curr_city = item['city']
@@ -175,9 +185,7 @@ df['consumption_rating_coverage'] = df['consumption_rating_count'] * 100 / df['c
df['estate_charge_coverage'] = df['estate_charge_count'] * 100 / df['community_count']
df['avg_cost_coverage'] = df['avg_cost_count'] * 100 / df['community_count']
##### 2. 近一年成交量
# 小区别名升级为单独df
# 7. 近一年成交量 MongoDB 加载
actions = list()
for index, item in seaweed_df.iterrows():
if type(item['alias']) == list:
@@ -197,8 +205,7 @@ deal_price_seaweed_notnull_df.loc[:, 'one_year_deal'] = deal_price_seaweed_notnu
deal_price_half_year_df = deal_price_seaweed_notnull_df[['city', 'one_year_deal']].drop_duplicates(keep='first')
df = pd.merge(df, deal_price_half_year_df, how='left', on=['city'])
# #### 2. 均价数量
# 8. 均价租金统计
avg_price_df = commutils.get_avg_price()
rent_price_df = commutils.get_rent_price()
@@ -214,7 +221,6 @@ df = pd.merge(df, rent_price_df, how='left', on=['city'])
df['avg_price_coverage'] = df['avg_price_count'] * 100 / df['community_count']
df['sqmetre_rent_coverage'] = df['sqmetre_rent_count'] * 100 / df['community_count']
df.fillna(0, inplace=True)
df['community_count'] = df['community_count'].astype(int)
@@ -231,13 +237,13 @@ df = df[['city', 'community_count', 'city_level', 'avg_price_coverage', 'sqmetre
'exist_alias_community_count', 'one_year_deal', 'main_type_coverage', 'main_type_area_coverage', 'total_price_coverage', 'estate_type_coverage',
'finish_building_date_coverage', 'all_area_coverage', 'build_area_coverage', 'estate_charge_coverage', 'avg_cost_coverage', 'consumption_rating_coverage', 'greening_rate_coverage', 'volume_rate_coverage']]
# 9. ClickHouse 写入与去重操作
df['weekend'] = dateutils.datetime.datetime.now()
# 删除本日生成的重复数据,防止手工点击生成
clickHouseClient.execute(
"ALTER TABLE fangjia_stat.community_coverage_stat DELETE WHERE weekend = toDate('" + dateutils.format_date(dateutils.datetime.datetime.now(), '%Y-%m-%d') + "')")
clickHouseClient.to_sql(df, table='community_coverage_stat', if_exists='insert', order_by='weekend')
# 上传至nextcloud
# 10. 文件输出与 NextCloud 上传
ret_df = pd.merge(ret_df, df, how='left', on=['city'])
del ret_df['是否合理']
ret_df = ret_df.rename(columns={'city': '城市', 'city_level': '城市级别', 'community_count': "楼盘数量", "alias_count": "别名数量", "addr_alias_count": "地址别名数量", "exist_alias_community_count": "有别名覆盖小区数量",
@@ -246,18 +252,4 @@ output_date = dateutils.format_date(dateutils.datetime.datetime.now())
aim_file_name = aim_path + "/" + output_date + '覆盖率统计.xlsx'
ret_df.to_excel(aim_file_name, index=False)
# https://nextcloud.fangjia.com/s/3aw663fy9BbiBeLji'bie
def up_to_nextcloud():
"""
上传统计信息至nextcloud
"""
from nextcloud import NextCloud
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
NEXTCLOUD_PASSWORD = "goojia123456"
aim_file_path = '共享/统计/常用覆盖率统计' + output_date + '.xlsx'
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
nxc.upload_file(aim_file_name, aim_file_path)
up_to_nextcloud()