diff --git a/dags/stats/community/community_count_price_rent.py b/dags/stats/community/community_count_price_rent.py index 08ba153..3a561c6 100644 --- a/dags/stats/community/community_count_price_rent.py +++ b/dags/stats/community/community_count_price_rent.py @@ -9,244 +9,8 @@ import logging log = logging.getLogger(__name__) -server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') -seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat') -sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent') -clickHouseClient = ClickHouseClient(host="192.168.0.216", database='fangjia_stat') -deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price') -es206_client = ElasticSearchClient(host='192.168.0.206') - -city_df = pd.DataFrame(list(server_area_conn.find({}, {'city_grade': 1, 'display': 1}))) -city_df = city_df.rename(columns={'city_grade': 'city_level', 'display': 'city'}) -city_df = city_df[city_df['city_level'] != 0] -# 昆玉无小区 -city_df = city_df[city_df['city'] != '昆玉'].reset_index(drop=True) - -seaweed_df = pd.DataFrame( - list(seaweed_conn.find({'cat': 'district', 'status': 0}, - {'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'addr_alias': 1}) - .batch_size(100000))) - -path = "/opt/airflow/dags/repo/dags/business/business/community/data" - -aim_path = "/home/airflow/data/stat" -if not os.path.exists(aim_path): - os.makedirs(aim_path) - -ret_df = pd.read_excel(path + '/财力评级分布情况20221215.xlsx') - -for index, item in city_df.iterrows(): - city = item['city'] - region_df = seaweed_df[(seaweed_df['city'] == city)].copy() - alias_count = 0 - addr_count = 0 - seaweed_count = 0 - # 楼盘数量 - city_df.loc[index, 'community_count'] = len(region_df) - if len(region_df) == 0: - log.error(f"该城市{city}小区数量为0") - for index1, item1 in region_df.iterrows(): - flag = True - if type(item1['alias']) == list: - if len(item1['alias']) > 0: - alias_count += len(item1['alias']) - seaweed_count += 1 - flag = False - if type(item1['addr_alias']) == list: - if len(item1['addr_alias']) > 0: - addr_count += len(item1['addr_alias']) - if len(item1['addr_alias']) > 0 and flag is True: - seaweed_count += 1 - # 楼盘别名总数 - city_df.loc[index, 'alias_count'] = alias_count - # 地址别名数量 - city_df.loc[index, 'addr_alias_count'] = addr_count - # 有别名小区量【存在楼盘别名或地址别名】 - city_df.loc[index, 'exist_alias_community_count'] = seaweed_count - -# 【已过期】 -# #### 1. 小区数量 -# result_df = pd.DataFrame(list(seaweed_conn.aggregate([ -# {'$match': {'cat': 'district', 'status': 0}}, -# {'$group': {'_id': "$city", 'community_count': {'$sum': 1}}}]))) -# -# -# result_df = result_df.rename(columns={'_id': 'city'}) -# df = pd.merge(result_df, city_df, how='right', on='city') -# df.fillna(0, inplace=True) - -# 有别名小区量 该方式统计不准,可能存在[]情况也算了 -# alias_df = pd.DataFrame(list(seaweed_conn.aggregate([ -# {'$match': {'cat': 'district', 'status': 0, 'mapping_alias': {'$ne': None}}}, -# {'$group': {'_id': "$city", 'alias_count': {'$sum': 1}}}]))) -# alias_df = alias_df.rename(columns={'_id': 'city'}) -# df = pd.merge(df, alias_df, how='left', on=['city']) - -df = city_df.copy() - -# 主力户型 -main_type_count = pd.DataFrame(list(seaweed_conn.aggregate([ - {'$match': {'cat': 'district', 'status': 0, 'main_type': {'$ne': None}}}, - {'$group': {'_id': "$city", 'main_type_count': {'$sum': 1}}}]))) -main_type_count = main_type_count.rename(columns={'_id': 'city'}) -df = pd.merge(df, main_type_count, how='left', on=['city']) -df['main_type_coverage'] = df['main_type_count'] * 100 / df['community_count'] -# 主力面积 -main_type_area_count = pd.DataFrame(list(seaweed_conn.aggregate([ - {'$match': {'cat': 'district', 'status': 0, 'main_type_area': {'$ne': None}}}, - {'$group': {'_id': "$city", 'main_type_area_count': {'$sum': 1}}}]))) -main_type_area_count = main_type_area_count.rename(columns={'_id': 'city'}) -df = pd.merge(df, main_type_area_count, how='left', on=['city']) -df['main_type_area_coverage'] = df['main_type_area_count'] * 100 / df['community_count'] -# 物业类型 -estate_type_count = pd.DataFrame(list(seaweed_conn.aggregate([ - {'$match': {'cat': 'district', 'status': 0, 'estate_type2': {'$ne': None}}}, - {'$group': {'_id': "$city", 'estate_type_count': {'$sum': 1}}}]))) -estate_type_count = estate_type_count.rename(columns={'_id': 'city'}) -df = pd.merge(df, estate_type_count, how='left', on=['city']) -df['estate_type_coverage'] = df['estate_type_count'] * 100 / df['community_count'] -# 建成年代 -finish_building_date_count = pd.DataFrame(list(seaweed_conn.aggregate([ - {'$match': {'cat': 'district', 'status': 0, 'finish_building_date': {'$ne': None}}}, - {'$group': {'_id': "$city", 'finish_building_date_count': {'$sum': 1}}}]))) -finish_building_date_count = finish_building_date_count.rename(columns={'_id': 'city'}) -df = pd.merge(df, finish_building_date_count, how='left', on=['city']) -df['finish_building_date_coverage'] = df['finish_building_date_count'] * 100 / df['community_count'] -# 占地面积 -all_area_count = pd.DataFrame(list(seaweed_conn.aggregate([ - {'$match': {'cat': 'district', 'status': 0, 'all_area': {'$ne': None}}}, - {'$group': {'_id': "$city", 'all_area_count': {'$sum': 1}}}]))) -all_area_count = all_area_count.rename(columns={'_id': 'city'}) -df = pd.merge(df, all_area_count, how='left', on=['city']) -df['all_area_coverage'] = df['all_area_count'] * 100 / df['community_count'] -# 建筑面积 -build_area_count = pd.DataFrame(list(seaweed_conn.aggregate([ - {'$match': {'cat': 'district', 'status': 0, 'build_area': {'$ne': None}}}, - {'$group': {'_id': "$city", 'build_area_count': {'$sum': 1}}}]))) -build_area_count = build_area_count.rename(columns={'_id': 'city'}) -df = pd.merge(df, build_area_count, how='left', on=['city']) -df['build_area_coverage'] = df['build_area_count'] * 100 / df['community_count'] - -# 绿化率 -greening_rate_count = pd.DataFrame(list(seaweed_conn.aggregate([ - {'$match': {'cat': 'district', 'status': 0, 'virescence_percent': {'$ne': None}}}, - {'$group': {'_id': "$city", 'greening_rate_count': {'$sum': 1}}}]))) -greening_rate_count = greening_rate_count.rename(columns={'_id': 'city'}) -df = pd.merge(df, greening_rate_count, how='left', on=['city']) -df['greening_rate_coverage'] = df['greening_rate_count'] * 100 / df['community_count'] -# 容积率 -volume_rate_count = pd.DataFrame(list(seaweed_conn.aggregate([ - {'$match': {'cat': 'district', 'status': 0, 'cubage_percent': {'$ne': None}}}, - {'$group': {'_id': "$city", 'volume_rate_count': {'$sum': 1}}}]))) -volume_rate_count = volume_rate_count.rename(columns={'_id': 'city'}) -df = pd.merge(df, volume_rate_count, how='left', on=['city']) -df['volume_rate_coverage'] = df['volume_rate_count'] * 100 / df['community_count'] - -# 户均总价 -df['total_price_count'] = None -# 物业费 -df['estate_charge_count'] = None -# 人均消费 -df['avg_cost_count'] = None -# 财力评级 -df['consumption_rating_count'] = None -for index, item in df.iterrows(): - curr_city = item['city'] - sql = f"select count(*) from community_summary where city='{curr_city}' and total_price>0 " - num = es206_client.search_by_sql(sql)['hits']['total']['value'] - df.loc[index, 'total_price_count'] = num - - sql = f"select count(*) from community_summary where city='{curr_city}' and estate_charge>0 " - estate_charge_num = es206_client.search_by_sql(sql)['hits']['total']['value'] - df.loc[index, 'estate_charge_count'] = estate_charge_num - - sql = f"select count(*) from community_summary where city='{curr_city}' and avg_cost>0 " - avg_cost_count = es206_client.search_by_sql(sql)['hits']['total']['value'] - df.loc[index, 'avg_cost_count'] = avg_cost_count - - sql = f"select count(*) from community_summary where city='{curr_city}' and consumption_rating>0 " - consumption_rating_num = es206_client.search_by_sql(sql)['hits']['total']['value'] - df.loc[index, 'consumption_rating_count'] = consumption_rating_num - -df['total_price_coverage'] = df['total_price_count'] * 100 / df['community_count'] -df['consumption_rating_coverage'] = df['consumption_rating_count'] * 100 / df['community_count'] -df['estate_charge_coverage'] = df['estate_charge_count'] * 100 / df['community_count'] -df['avg_cost_coverage'] = df['avg_cost_count'] * 100 / df['community_count'] - -##### 2. 近一年成交量 - -# 小区别名升级为单独df -actions = list() -for index, item in seaweed_df.iterrows(): - if type(item['alias']) == list: - for name in item['alias']: - actions.append({'_id': item['_id'], 'city': item['city'], 'name': name, 'region': item['region']}) -alias_df = pd.DataFrame(actions) -seaweed_alias_df = pd.concat([seaweed_df[['_id', 'city', 'region', 'name']], alias_df]) -deal_price_df = pd.DataFrame(list(deal_price_conn.find( - {'trade_date': {'$gte': dateutils.add_months(dateutils.datetime.datetime.now(), days=-365)}}, - {'city': 1, 'region': 1, 'district_name': 1, '_id': 0}).batch_size(10000))) -deal_price_df = deal_price_df.rename(columns={'district_name': 'name'}) -deal_price_seaweed_df = pd.merge(deal_price_df, seaweed_alias_df, how='left', on=['city', 'region', 'name']) -# 剔除没有配到的小区 -deal_price_seaweed_notnull_df = deal_price_seaweed_df.dropna(subset=['_id']) -deal_price_seaweed_notnull_df.loc[:, 'one_year_deal'] = deal_price_seaweed_notnull_df.groupby(['city'])[ - 'city'].transform('size') -deal_price_half_year_df = deal_price_seaweed_notnull_df[['city', 'one_year_deal']].drop_duplicates(keep='first') -df = pd.merge(df, deal_price_half_year_df, how='left', on=['city']) - -# #### 2. 均价数量 - -avg_price_df = commutils.get_avg_price() -rent_price_df = commutils.get_rent_price() - -avg_price_df['avg_price_count'] = avg_price_df.groupby(by=['city'])['_id'].transform('count') -rent_price_df['sqmetre_rent_count'] = rent_price_df.groupby(by=['city'])['_id'].transform('count') - -avg_price_df = avg_price_df[['city', 'avg_price_count']].drop_duplicates(keep='first') -rent_price_df = rent_price_df[['city', 'sqmetre_rent_count']].drop_duplicates(keep='first') - -df = pd.merge(df, avg_price_df, how='left', on=['city']) -df = pd.merge(df, rent_price_df, how='left', on=['city']) - -df['avg_price_coverage'] = df['avg_price_count'] * 100 / df['community_count'] -df['sqmetre_rent_coverage'] = df['sqmetre_rent_count'] * 100 / df['community_count'] - - -df.fillna(0, inplace=True) - -df['community_count'] = df['community_count'].astype(int) -df['avg_price_count'] = df['avg_price_count'].astype(int) -df['sqmetre_rent_count'] = df['sqmetre_rent_count'].astype(int) -df['one_year_deal'] = df['one_year_deal'].astype(int) -df['alias_count'] = df['alias_count'].astype(int) -df['addr_alias_count'] = df['addr_alias_count'].astype(int) -df['exist_alias_community_count'] = df['exist_alias_community_count'].astype(int) - -df = df.round({'avg_price_coverage': 2, 'sqmetre_rent_coverage': 2, 'main_type_coverage': 2, 'main_type_area_coverage': 2, 'total_price_coverage': 2, 'estate_type_coverage': 2, - 'finish_building_date_coverage': 2, 'all_area_coverage': 2, 'build_area_coverage': 2, 'estate_charge_coverage': 2, 'avg_cost_coverage': 2, 'consumption_rating_coverage': 2, 'greening_rate_coverage': 2, 'volume_rate_coverage': 2}) -df = df[['city', 'community_count', 'city_level', 'avg_price_coverage', 'sqmetre_rent_coverage', 'alias_count', 'addr_alias_count', - 'exist_alias_community_count', 'one_year_deal', 'main_type_coverage', 'main_type_area_coverage', 'total_price_coverage', 'estate_type_coverage', - 'finish_building_date_coverage', 'all_area_coverage', 'build_area_coverage', 'estate_charge_coverage', 'avg_cost_coverage', 'consumption_rating_coverage', 'greening_rate_coverage', 'volume_rate_coverage']] - -df['weekend'] = dateutils.datetime.datetime.now() -# 删除本日生成的重复数据,防止手工点击生成 -clickHouseClient.execute( - "ALTER TABLE fangjia_stat.community_coverage_stat DELETE WHERE weekend = toDate('" + dateutils.format_date(dateutils.datetime.datetime.now(), '%Y-%m-%d') + "')") -clickHouseClient.to_sql(df, table='community_coverage_stat', if_exists='insert', order_by='weekend') - -# 上传至nextcloud -ret_df = pd.merge(ret_df, df, how='left', on=['city']) -del ret_df['是否合理'] -ret_df = ret_df.rename(columns={'city': '城市', 'city_level': '城市级别', 'community_count': "楼盘数量", "alias_count": "别名数量", "addr_alias_count": "地址别名数量", "exist_alias_community_count": "有别名覆盖小区数量", - 'estate_type_coverage': '物业类型覆盖率', 'avg_price_coverage': '均价覆盖率', 'sqmetre_rent_coverage': '平米租金覆盖率', 'total_price_coverage': '户均总价覆盖率', 'estate_charge_coverage': '物业费覆盖率', 'avg_cost_coverage': '人均消费覆盖率', 'main_type_coverage': '主力户型覆盖率', 'main_type_area_coverage': '主力面积覆盖率', 'finish_building_date_coverage': '建成年代覆盖率', 'all_area_coverage': '占地面积覆盖率', 'build_area_coverage': '建筑面积覆盖率', 'consumption_rating_coverage': '财力评级覆盖率', 'greening_rate_coverage': '绿化率覆盖率', 'volume_rate_coverage': '容积率覆盖率'}) -output_date = dateutils.format_date(dateutils.datetime.datetime.now()) -aim_file_name = aim_path + "/" + output_date + '覆盖率统计.xlsx' -ret_df.to_excel(aim_file_name, index=False) - -# https://nextcloud.fangjia.com/s/3aw663fy9BbiBeLji'bie +# ================== 辅助上传函数(全局定义,供 main 调用) ================== def up_to_nextcloud(): """ @@ -256,8 +20,236 @@ def up_to_nextcloud(): NEXTCLOUD_URL = "https://nextcloud.fangjia.com" NEXTCLOUD_USERNAME = "shenjian@fangjia.com" NEXTCLOUD_PASSWORD = "goojia123456" + # 使用运行时生成的 output_date 和 aim_file_name 全局变量 aim_file_path = '共享/统计/常用覆盖率统计' + output_date + '.xlsx' nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True) nxc.upload_file(aim_file_name, aim_file_path) -up_to_nextcloud() + +# ================== 🛡️ 安全运行入口 ================== + +if __name__ == "__main__": + # 1. 延迟初始化数据库连接(避开 Airflow 进程的加载期) + server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') + seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') + district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat') + sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent') + clickHouseClient = ClickHouseClient(host="192.168.0.216", database='fangjia_stat') + deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price') + + es206_client = ElasticSearchClient(host='192.168.0.206') + + # 2. 读取城市列表与清洗 + city_df = pd.DataFrame(list(server_area_conn.find({}, {'city_grade': 1, 'display': 1}))) + city_df = city_df.rename(columns={'city_grade': 'city_level', 'display': 'city'}) + city_df = city_df[city_df['city_level'] != 0] + # 昆玉无小区 + city_df = city_df[city_df['city'] != '昆玉'].reset_index(drop=True) + + # 3. 巨量 MongoDB 查询与路径准备 + seaweed_df = pd.DataFrame( + list(seaweed_conn.find({'cat': 'district', 'status': 0}, + {'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'addr_alias': 1}) + .batch_size(100000))) + + path = "/opt/airflow/dags/repo/dags/business/business/community/data" + + aim_path = "/home/airflow/data/stat" + if not os.path.exists(aim_path): + os.makedirs(aim_path) + + ret_df = pd.read_excel(path + '/财力评级分布情况20221215.xlsx') + + # 4. 双层大循环处理小区与别名 + for index, item in city_df.iterrows(): + city = item['city'] + region_df = seaweed_df[(seaweed_df['city'] == city)].copy() + alias_count = 0 + addr_count = 0 + seaweed_count = 0 + # 楼盘数量 + city_df.loc[index, 'community_count'] = len(region_df) + if len(region_df) == 0: + log.error(f"该城市{city}小区数量为0") + for index1, item1 in region_df.iterrows(): + flag = True + if type(item1['alias']) == list: + if len(item1['alias']) > 0: + alias_count += len(item1['alias']) + seaweed_count += 1 + flag = False + if type(item1['addr_alias']) == list: + if len(item1['addr_alias']) > 0: + addr_count += len(item1['addr_alias']) + if len(item1['addr_alias']) > 0 and flag is True: + seaweed_count += 1 + # 楼盘别名总数 + city_df.loc[index, 'alias_count'] = alias_count + # 地址别名数量 + city_df.loc[index, 'addr_alias_count'] = addr_count + # 有别名小区量【存在楼盘别名或地址别名】 + city_df.loc[index, 'exist_alias_community_count'] = seaweed_count + + df = city_df.copy() + + # 5. 主力属性的多项重度 MongoDB Aggregation 统计 + # 主力户型 + main_type_count = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0, 'main_type': {'$ne': None}}}, + {'$group': {'_id': "$city", 'main_type_count': {'$sum': 1}}}]))) + main_type_count = main_type_count.rename(columns={'_id': 'city'}) + df = pd.merge(df, main_type_count, how='left', on=['city']) + df['main_type_coverage'] = df['main_type_count'] * 100 / df['community_count'] + + # 主力面积 + main_type_area_count = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0, 'main_type_area': {'$ne': None}}}, + {'$group': {'_id': "$city", 'main_type_area_count': {'$sum': 1}}}]))) + main_type_area_count = main_type_area_count.rename(columns={'_id': 'city'}) + df = pd.merge(df, main_type_area_count, how='left', on=['city']) + df['main_type_area_coverage'] = df['main_type_area_count'] * 100 / df['community_count'] + + # 物业类型 + estate_type_count = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0, 'estate_type2': {'$ne': None}}}, + {'$group': {'_id': "$city", 'estate_type_count': {'$sum': 1}}}]))) + estate_type_count = estate_type_count.rename(columns={'_id': 'city'}) + df = pd.merge(df, estate_type_count, how='left', on=['city']) + df['estate_type_coverage'] = df['estate_type_count'] * 100 / df['community_count'] + + # 建成年代 + finish_building_date_count = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0, 'finish_building_date': {'$ne': None}}}, + {'$group': {'_id': "$city", 'finish_building_date_count': {'$sum': 1}}}]))) + finish_building_date_count = finish_building_date_count.rename(columns={'_id': 'city'}) + df = pd.merge(df, finish_building_date_count, how='left', on=['city']) + df['finish_building_date_coverage'] = df['finish_building_date_count'] * 100 / df['community_count'] + + # 占地面积 + all_area_count = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0, 'all_area': {'$ne': None}}}, + {'$group': {'_id': "$city", 'all_area_count': {'$sum': 1}}}]))) + all_area_count = all_area_count.rename(columns={'_id': 'city'}) + df = pd.merge(df, all_area_count, how='left', on=['city']) + df['all_area_coverage'] = df['all_area_count'] * 100 / df['community_count'] + + # 建筑面积 + build_area_count = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0, 'build_area': {'$ne': None}}}, + {'$group': {'_id': "$city", 'build_area_count': {'$sum': 1}}}]))) + build_area_count = build_area_count.rename(columns={'_id': 'city'}) + df = pd.merge(df, build_area_count, how='left', on=['city']) + df['build_area_coverage'] = df['build_area_count'] * 100 / df['community_count'] + + # 绿化率 + greening_rate_count = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0, 'virescence_percent': {'$ne': None}}}, + {'$group': {'_id': "$city", 'greening_rate_count': {'$sum': 1}}}]))) + greening_rate_count = greening_rate_count.rename(columns={'_id': 'city'}) + df = pd.merge(df, greening_rate_count, how='left', on=['city']) + df['greening_rate_coverage'] = df['greening_rate_count'] * 100 / df['community_count'] + + # 容积率 + volume_rate_count = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0, 'cubage_percent': {'$ne': None}}}, + {'$group': {'_id': "$city", 'volume_rate_count': {'$sum': 1}}}]))) + volume_rate_count = volume_rate_count.rename(columns={'_id': 'city'}) + df = pd.merge(df, volume_rate_count, how='left', on=['city']) + df['volume_rate_coverage'] = df['volume_rate_count'] * 100 / df['community_count'] + + # 6. 人均/财力等 Elasticsearch 批量循环查询 + df['total_price_count'] = None + df['estate_charge_count'] = None + df['avg_cost_count'] = None + df['consumption_rating_count'] = None + for index, item in df.iterrows(): + curr_city = item['city'] + sql = f"select count(*) from community_summary where city='{curr_city}' and total_price>0 " + num = es206_client.search_by_sql(sql)['hits']['total']['value'] + df.loc[index, 'total_price_count'] = num + + sql = f"select count(*) from community_summary where city='{curr_city}' and estate_charge>0 " + estate_charge_num = es206_client.search_by_sql(sql)['hits']['total']['value'] + df.loc[index, 'estate_charge_count'] = estate_charge_num + + sql = f"select count(*) from community_summary where city='{curr_city}' and avg_cost>0 " + avg_cost_count = es206_client.search_by_sql(sql)['hits']['total']['value'] + df.loc[index, 'avg_cost_count'] = avg_cost_count + + sql = f"select count(*) from community_summary where city='{curr_city}' and consumption_rating>0 " + consumption_rating_num = es206_client.search_by_sql(sql)['hits']['total']['value'] + df.loc[index, 'consumption_rating_count'] = consumption_rating_num + + df['total_price_coverage'] = df['total_price_count'] * 100 / df['community_count'] + df['consumption_rating_coverage'] = df['consumption_rating_count'] * 100 / df['community_count'] + df['estate_charge_coverage'] = df['estate_charge_count'] * 100 / df['community_count'] + df['avg_cost_coverage'] = df['avg_cost_count'] * 100 / df['community_count'] + + # 7. 近一年成交量 MongoDB 加载 + actions = list() + for index, item in seaweed_df.iterrows(): + if type(item['alias']) == list: + for name in item['alias']: + actions.append({'_id': item['_id'], 'city': item['city'], 'name': name, 'region': item['region']}) + alias_df = pd.DataFrame(actions) + seaweed_alias_df = pd.concat([seaweed_df[['_id', 'city', 'region', 'name']], alias_df]) + deal_price_df = pd.DataFrame(list(deal_price_conn.find( + {'trade_date': {'$gte': dateutils.add_months(dateutils.datetime.datetime.now(), days=-365)}}, + {'city': 1, 'region': 1, 'district_name': 1, '_id': 0}).batch_size(10000))) + deal_price_df = deal_price_df.rename(columns={'district_name': 'name'}) + deal_price_seaweed_df = pd.merge(deal_price_df, seaweed_alias_df, how='left', on=['city', 'region', 'name']) + # 剔除没有配到的小区 + deal_price_seaweed_notnull_df = deal_price_seaweed_df.dropna(subset=['_id']) + deal_price_seaweed_notnull_df.loc[:, 'one_year_deal'] = deal_price_seaweed_notnull_df.groupby(['city'])[ + 'city'].transform('size') + deal_price_half_year_df = deal_price_seaweed_notnull_df[['city', 'one_year_deal']].drop_duplicates(keep='first') + df = pd.merge(df, deal_price_half_year_df, how='left', on=['city']) + + # 8. 均价租金统计 + avg_price_df = commutils.get_avg_price() + rent_price_df = commutils.get_rent_price() + + avg_price_df['avg_price_count'] = avg_price_df.groupby(by=['city'])['_id'].transform('count') + rent_price_df['sqmetre_rent_count'] = rent_price_df.groupby(by=['city'])['_id'].transform('count') + + avg_price_df = avg_price_df[['city', 'avg_price_count']].drop_duplicates(keep='first') + rent_price_df = rent_price_df[['city', 'sqmetre_rent_count']].drop_duplicates(keep='first') + + df = pd.merge(df, avg_price_df, how='left', on=['city']) + df = pd.merge(df, rent_price_df, how='left', on=['city']) + + df['avg_price_coverage'] = df['avg_price_count'] * 100 / df['community_count'] + df['sqmetre_rent_coverage'] = df['sqmetre_rent_count'] * 100 / df['community_count'] + + df.fillna(0, inplace=True) + + df['community_count'] = df['community_count'].astype(int) + df['avg_price_count'] = df['avg_price_count'].astype(int) + df['sqmetre_rent_count'] = df['sqmetre_rent_count'].astype(int) + df['one_year_deal'] = df['one_year_deal'].astype(int) + df['alias_count'] = df['alias_count'].astype(int) + df['addr_alias_count'] = df['addr_alias_count'].astype(int) + df['exist_alias_community_count'] = df['exist_alias_community_count'].astype(int) + + df = df.round({'avg_price_coverage': 2, 'sqmetre_rent_coverage': 2, 'main_type_coverage': 2, 'main_type_area_coverage': 2, 'total_price_coverage': 2, 'estate_type_coverage': 2, + 'finish_building_date_coverage': 2, 'all_area_coverage': 2, 'build_area_coverage': 2, 'estate_charge_coverage': 2, 'avg_cost_coverage': 2, 'consumption_rating_coverage': 2, 'greening_rate_coverage': 2, 'volume_rate_coverage': 2}) + df = df[['city', 'community_count', 'city_level', 'avg_price_coverage', 'sqmetre_rent_coverage', 'alias_count', 'addr_alias_count', + 'exist_alias_community_count', 'one_year_deal', 'main_type_coverage', 'main_type_area_coverage', 'total_price_coverage', 'estate_type_coverage', + 'finish_building_date_coverage', 'all_area_coverage', 'build_area_coverage', 'estate_charge_coverage', 'avg_cost_coverage', 'consumption_rating_coverage', 'greening_rate_coverage', 'volume_rate_coverage']] + + # 9. ClickHouse 写入与去重操作 + df['weekend'] = dateutils.datetime.datetime.now() + clickHouseClient.execute( + "ALTER TABLE fangjia_stat.community_coverage_stat DELETE WHERE weekend = toDate('" + dateutils.format_date(dateutils.datetime.datetime.now(), '%Y-%m-%d') + "')") + clickHouseClient.to_sql(df, table='community_coverage_stat', if_exists='insert', order_by='weekend') + + # 10. 文件输出与 NextCloud 上传 + ret_df = pd.merge(ret_df, df, how='left', on=['city']) + del ret_df['是否合理'] + ret_df = ret_df.rename(columns={'city': '城市', 'city_level': '城市级别', 'community_count': "楼盘数量", "alias_count": "别名数量", "addr_alias_count": "地址别名数量", "exist_alias_community_count": "有别名覆盖小区数量", + 'estate_type_coverage': '物业类型覆盖率', 'avg_price_coverage': '均价覆盖率', 'sqmetre_rent_coverage': '平米租金覆盖率', 'total_price_coverage': '户均总价覆盖率', 'estate_charge_coverage': '物业费覆盖率', 'avg_cost_coverage': '人均消费覆盖率', 'main_type_coverage': '主力户型覆盖率', 'main_type_area_coverage': '主力面积覆盖率', 'finish_building_date_coverage': '建成年代覆盖率', 'all_area_coverage': '占地面积覆盖率', 'build_area_coverage': '建筑面积覆盖率', 'consumption_rating_coverage': '财力评级覆盖率', 'greening_rate_coverage': '绿化率覆盖率', 'volume_rate_coverage': '容积率覆盖率'}) + output_date = dateutils.format_date(dateutils.datetime.datetime.now()) + aim_file_name = aim_path + "/" + output_date + '覆盖率统计.xlsx' + ret_df.to_excel(aim_file_name, index=False) + + up_to_nextcloud() \ No newline at end of file