264 lines
14 KiB
Python
264 lines
14 KiB
Python
"""
|
|
小区数量 租金 均价 及 覆盖率统计
|
|
"""
|
|
|
|
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ClickHouseClient, ElasticSearchClient, commutils
|
|
import pandas as pd
|
|
import os
|
|
import logging
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
|
|
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
|
district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat')
|
|
sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent')
|
|
clickHouseClient = ClickHouseClient(host="192.168.0.216", database='fangjia_stat')
|
|
deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price')
|
|
|
|
es206_client = ElasticSearchClient(host='192.168.0.206')
|
|
|
|
city_df = pd.DataFrame(list(server_area_conn.find({}, {'city_grade': 1, 'display': 1})))
|
|
city_df = city_df.rename(columns={'city_grade': 'city_level', 'display': 'city'})
|
|
city_df = city_df[city_df['city_level'] != 0]
|
|
# 昆玉无小区
|
|
city_df = city_df[city_df['city'] != '昆玉'].reset_index(drop=True)
|
|
|
|
seaweed_df = pd.DataFrame(
|
|
list(seaweed_conn.find({'cat': 'district', 'status': 0},
|
|
{'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'addr_alias': 1})
|
|
.batch_size(100000)))
|
|
|
|
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
|
|
|
aim_path = "/home/airflow/data/stat"
|
|
if not os.path.exists(aim_path):
|
|
os.makedirs(aim_path)
|
|
|
|
ret_df = pd.read_excel(path + '/财力评级分布情况20221215.xlsx')
|
|
|
|
for index, item in city_df.iterrows():
|
|
city = item['city']
|
|
region_df = seaweed_df[(seaweed_df['city'] == city)].copy()
|
|
alias_count = 0
|
|
addr_count = 0
|
|
seaweed_count = 0
|
|
# 楼盘数量
|
|
city_df.loc[index, 'community_count'] = len(region_df)
|
|
if len(region_df) == 0:
|
|
log.error(f"该城市{city}小区数量为0")
|
|
for index1, item1 in region_df.iterrows():
|
|
flag = True
|
|
if type(item1['alias']) == list:
|
|
if len(item1['alias']) > 0:
|
|
alias_count += len(item1['alias'])
|
|
seaweed_count += 1
|
|
flag = False
|
|
if type(item1['addr_alias']) == list:
|
|
if len(item1['addr_alias']) > 0:
|
|
addr_count += len(item1['addr_alias'])
|
|
if len(item1['addr_alias']) > 0 and flag is True:
|
|
seaweed_count += 1
|
|
# 楼盘别名总数
|
|
city_df.loc[index, 'alias_count'] = alias_count
|
|
# 地址别名数量
|
|
city_df.loc[index, 'addr_alias_count'] = addr_count
|
|
# 有别名小区量【存在楼盘别名或地址别名】
|
|
city_df.loc[index, 'exist_alias_community_count'] = seaweed_count
|
|
|
|
# 【已过期】
|
|
# #### 1. 小区数量
|
|
# result_df = pd.DataFrame(list(seaweed_conn.aggregate([
|
|
# {'$match': {'cat': 'district', 'status': 0}},
|
|
# {'$group': {'_id': "$city", 'community_count': {'$sum': 1}}}])))
|
|
#
|
|
#
|
|
# result_df = result_df.rename(columns={'_id': 'city'})
|
|
# df = pd.merge(result_df, city_df, how='right', on='city')
|
|
# df.fillna(0, inplace=True)
|
|
|
|
# 有别名小区量 该方式统计不准,可能存在[]情况也算了
|
|
# alias_df = pd.DataFrame(list(seaweed_conn.aggregate([
|
|
# {'$match': {'cat': 'district', 'status': 0, 'mapping_alias': {'$ne': None}}},
|
|
# {'$group': {'_id': "$city", 'alias_count': {'$sum': 1}}}])))
|
|
# alias_df = alias_df.rename(columns={'_id': 'city'})
|
|
# df = pd.merge(df, alias_df, how='left', on=['city'])
|
|
|
|
df = city_df.copy()
|
|
|
|
# 主力户型
|
|
main_type_count = pd.DataFrame(list(seaweed_conn.aggregate([
|
|
{'$match': {'cat': 'district', 'status': 0, 'main_type': {'$ne': None}}},
|
|
{'$group': {'_id': "$city", 'main_type_count': {'$sum': 1}}}])))
|
|
main_type_count = main_type_count.rename(columns={'_id': 'city'})
|
|
df = pd.merge(df, main_type_count, how='left', on=['city'])
|
|
df['main_type_coverage'] = df['main_type_count'] * 100 / df['community_count']
|
|
# 主力面积
|
|
main_type_area_count = pd.DataFrame(list(seaweed_conn.aggregate([
|
|
{'$match': {'cat': 'district', 'status': 0, 'main_type_area': {'$ne': None}}},
|
|
{'$group': {'_id': "$city", 'main_type_area_count': {'$sum': 1}}}])))
|
|
main_type_area_count = main_type_area_count.rename(columns={'_id': 'city'})
|
|
df = pd.merge(df, main_type_area_count, how='left', on=['city'])
|
|
df['main_type_area_coverage'] = df['main_type_area_count'] * 100 / df['community_count']
|
|
# 物业类型
|
|
estate_type_count = pd.DataFrame(list(seaweed_conn.aggregate([
|
|
{'$match': {'cat': 'district', 'status': 0, 'estate_type2': {'$ne': None}}},
|
|
{'$group': {'_id': "$city", 'estate_type_count': {'$sum': 1}}}])))
|
|
estate_type_count = estate_type_count.rename(columns={'_id': 'city'})
|
|
df = pd.merge(df, estate_type_count, how='left', on=['city'])
|
|
df['estate_type_coverage'] = df['estate_type_count'] * 100 / df['community_count']
|
|
# 建成年代
|
|
finish_building_date_count = pd.DataFrame(list(seaweed_conn.aggregate([
|
|
{'$match': {'cat': 'district', 'status': 0, 'finish_building_date': {'$ne': None}}},
|
|
{'$group': {'_id': "$city", 'finish_building_date_count': {'$sum': 1}}}])))
|
|
finish_building_date_count = finish_building_date_count.rename(columns={'_id': 'city'})
|
|
df = pd.merge(df, finish_building_date_count, how='left', on=['city'])
|
|
df['finish_building_date_coverage'] = df['finish_building_date_count'] * 100 / df['community_count']
|
|
# 占地面积
|
|
all_area_count = pd.DataFrame(list(seaweed_conn.aggregate([
|
|
{'$match': {'cat': 'district', 'status': 0, 'all_area': {'$ne': None}}},
|
|
{'$group': {'_id': "$city", 'all_area_count': {'$sum': 1}}}])))
|
|
all_area_count = all_area_count.rename(columns={'_id': 'city'})
|
|
df = pd.merge(df, all_area_count, how='left', on=['city'])
|
|
df['all_area_coverage'] = df['all_area_count'] * 100 / df['community_count']
|
|
# 建筑面积
|
|
build_area_count = pd.DataFrame(list(seaweed_conn.aggregate([
|
|
{'$match': {'cat': 'district', 'status': 0, 'build_area': {'$ne': None}}},
|
|
{'$group': {'_id': "$city", 'build_area_count': {'$sum': 1}}}])))
|
|
build_area_count = build_area_count.rename(columns={'_id': 'city'})
|
|
df = pd.merge(df, build_area_count, how='left', on=['city'])
|
|
df['build_area_coverage'] = df['build_area_count'] * 100 / df['community_count']
|
|
|
|
# 绿化率
|
|
greening_rate_count = pd.DataFrame(list(seaweed_conn.aggregate([
|
|
{'$match': {'cat': 'district', 'status': 0, 'virescence_percent': {'$ne': None}}},
|
|
{'$group': {'_id': "$city", 'greening_rate_count': {'$sum': 1}}}])))
|
|
greening_rate_count = greening_rate_count.rename(columns={'_id': 'city'})
|
|
df = pd.merge(df, greening_rate_count, how='left', on=['city'])
|
|
df['greening_rate_coverage'] = df['greening_rate_count'] * 100 / df['community_count']
|
|
# 容积率
|
|
volume_rate_count = pd.DataFrame(list(seaweed_conn.aggregate([
|
|
{'$match': {'cat': 'district', 'status': 0, 'cubage_percent': {'$ne': None}}},
|
|
{'$group': {'_id': "$city", 'volume_rate_count': {'$sum': 1}}}])))
|
|
volume_rate_count = volume_rate_count.rename(columns={'_id': 'city'})
|
|
df = pd.merge(df, volume_rate_count, how='left', on=['city'])
|
|
df['volume_rate_coverage'] = df['volume_rate_count'] * 100 / df['community_count']
|
|
|
|
# 户均总价
|
|
df['total_price_count'] = None
|
|
# 物业费
|
|
df['estate_charge_count'] = None
|
|
# 人均消费
|
|
df['avg_cost_count'] = None
|
|
# 财力评级
|
|
df['consumption_rating_count'] = None
|
|
for index, item in df.iterrows():
|
|
curr_city = item['city']
|
|
sql = f"select count(*) from community_summary where city='{curr_city}' and total_price>0 "
|
|
num = es206_client.search_by_sql(sql)['hits']['total']['value']
|
|
df.loc[index, 'total_price_count'] = num
|
|
|
|
sql = f"select count(*) from community_summary where city='{curr_city}' and estate_charge>0 "
|
|
estate_charge_num = es206_client.search_by_sql(sql)['hits']['total']['value']
|
|
df.loc[index, 'estate_charge_count'] = estate_charge_num
|
|
|
|
sql = f"select count(*) from community_summary where city='{curr_city}' and avg_cost>0 "
|
|
avg_cost_count = es206_client.search_by_sql(sql)['hits']['total']['value']
|
|
df.loc[index, 'avg_cost_count'] = avg_cost_count
|
|
|
|
sql = f"select count(*) from community_summary where city='{curr_city}' and consumption_rating>0 "
|
|
consumption_rating_num = es206_client.search_by_sql(sql)['hits']['total']['value']
|
|
df.loc[index, 'consumption_rating_count'] = consumption_rating_num
|
|
|
|
df['total_price_coverage'] = df['total_price_count'] * 100 / df['community_count']
|
|
df['consumption_rating_coverage'] = df['consumption_rating_count'] * 100 / df['community_count']
|
|
df['estate_charge_coverage'] = df['estate_charge_count'] * 100 / df['community_count']
|
|
df['avg_cost_coverage'] = df['avg_cost_count'] * 100 / df['community_count']
|
|
|
|
##### 2. 近一年成交量
|
|
|
|
# 小区别名升级为单独df
|
|
actions = list()
|
|
for index, item in seaweed_df.iterrows():
|
|
if type(item['alias']) == list:
|
|
for name in item['alias']:
|
|
actions.append({'_id': item['_id'], 'city': item['city'], 'name': name, 'region': item['region']})
|
|
alias_df = pd.DataFrame(actions)
|
|
seaweed_alias_df = pd.concat([seaweed_df[['_id', 'city', 'region', 'name']], alias_df])
|
|
deal_price_df = pd.DataFrame(list(deal_price_conn.find(
|
|
{'trade_date': {'$gte': dateutils.add_months(dateutils.datetime.datetime.now(), days=-365)}},
|
|
{'city': 1, 'region': 1, 'district_name': 1, '_id': 0}).batch_size(10000)))
|
|
deal_price_df = deal_price_df.rename(columns={'district_name': 'name'})
|
|
deal_price_seaweed_df = pd.merge(deal_price_df, seaweed_alias_df, how='left', on=['city', 'region', 'name'])
|
|
# 剔除没有配到的小区
|
|
deal_price_seaweed_notnull_df = deal_price_seaweed_df.dropna(subset=['_id'])
|
|
deal_price_seaweed_notnull_df.loc[:, 'one_year_deal'] = deal_price_seaweed_notnull_df.groupby(['city'])[
|
|
'city'].transform('size')
|
|
deal_price_half_year_df = deal_price_seaweed_notnull_df[['city', 'one_year_deal']].drop_duplicates(keep='first')
|
|
df = pd.merge(df, deal_price_half_year_df, how='left', on=['city'])
|
|
|
|
# #### 2. 均价数量
|
|
|
|
avg_price_df = commutils.get_avg_price()
|
|
rent_price_df = commutils.get_rent_price()
|
|
|
|
avg_price_df['avg_price_count'] = avg_price_df.groupby(by=['city'])['_id'].transform('count')
|
|
rent_price_df['sqmetre_rent_count'] = rent_price_df.groupby(by=['city'])['_id'].transform('count')
|
|
|
|
avg_price_df = avg_price_df[['city', 'avg_price_count']].drop_duplicates(keep='first')
|
|
rent_price_df = rent_price_df[['city', 'sqmetre_rent_count']].drop_duplicates(keep='first')
|
|
|
|
df = pd.merge(df, avg_price_df, how='left', on=['city'])
|
|
df = pd.merge(df, rent_price_df, how='left', on=['city'])
|
|
|
|
df['avg_price_coverage'] = df['avg_price_count'] * 100 / df['community_count']
|
|
df['sqmetre_rent_coverage'] = df['sqmetre_rent_count'] * 100 / df['community_count']
|
|
|
|
|
|
df.fillna(0, inplace=True)
|
|
|
|
df['community_count'] = df['community_count'].astype(int)
|
|
df['avg_price_count'] = df['avg_price_count'].astype(int)
|
|
df['sqmetre_rent_count'] = df['sqmetre_rent_count'].astype(int)
|
|
df['one_year_deal'] = df['one_year_deal'].astype(int)
|
|
df['alias_count'] = df['alias_count'].astype(int)
|
|
df['addr_alias_count'] = df['addr_alias_count'].astype(int)
|
|
df['exist_alias_community_count'] = df['exist_alias_community_count'].astype(int)
|
|
|
|
df = df.round({'avg_price_coverage': 2, 'sqmetre_rent_coverage': 2, 'main_type_coverage': 2, 'main_type_area_coverage': 2, 'total_price_coverage': 2, 'estate_type_coverage': 2,
|
|
'finish_building_date_coverage': 2, 'all_area_coverage': 2, 'build_area_coverage': 2, 'estate_charge_coverage': 2, 'avg_cost_coverage': 2, 'consumption_rating_coverage': 2, 'greening_rate_coverage': 2, 'volume_rate_coverage': 2})
|
|
df = df[['city', 'community_count', 'city_level', 'avg_price_coverage', 'sqmetre_rent_coverage', 'alias_count', 'addr_alias_count',
|
|
'exist_alias_community_count', 'one_year_deal', 'main_type_coverage', 'main_type_area_coverage', 'total_price_coverage', 'estate_type_coverage',
|
|
'finish_building_date_coverage', 'all_area_coverage', 'build_area_coverage', 'estate_charge_coverage', 'avg_cost_coverage', 'consumption_rating_coverage', 'greening_rate_coverage', 'volume_rate_coverage']]
|
|
|
|
df['weekend'] = dateutils.datetime.datetime.now()
|
|
# 删除本日生成的重复数据,防止手工点击生成
|
|
clickHouseClient.execute(
|
|
"ALTER TABLE fangjia_stat.community_coverage_stat DELETE WHERE weekend = toDate('" + dateutils.format_date(dateutils.datetime.datetime.now(), '%Y-%m-%d') + "')")
|
|
clickHouseClient.to_sql(df, table='community_coverage_stat', if_exists='insert', order_by='weekend')
|
|
|
|
# 上传至nextcloud
|
|
ret_df = pd.merge(ret_df, df, how='left', on=['city'])
|
|
del ret_df['是否合理']
|
|
ret_df = ret_df.rename(columns={'city': '城市', 'city_level': '城市级别', 'community_count': "楼盘数量", "alias_count": "别名数量", "addr_alias_count": "地址别名数量", "exist_alias_community_count": "有别名覆盖小区数量",
|
|
'estate_type_coverage': '物业类型覆盖率', 'avg_price_coverage': '均价覆盖率', 'sqmetre_rent_coverage': '平米租金覆盖率', 'total_price_coverage': '户均总价覆盖率', 'estate_charge_coverage': '物业费覆盖率', 'avg_cost_coverage': '人均消费覆盖率', 'main_type_coverage': '主力户型覆盖率', 'main_type_area_coverage': '主力面积覆盖率', 'finish_building_date_coverage': '建成年代覆盖率', 'all_area_coverage': '占地面积覆盖率', 'build_area_coverage': '建筑面积覆盖率', 'consumption_rating_coverage': '财力评级覆盖率', 'greening_rate_coverage': '绿化率覆盖率', 'volume_rate_coverage': '容积率覆盖率'})
|
|
output_date = dateutils.format_date(dateutils.datetime.datetime.now())
|
|
aim_file_name = aim_path + "/" + output_date + '覆盖率统计.xlsx'
|
|
ret_df.to_excel(aim_file_name, index=False)
|
|
|
|
# https://nextcloud.fangjia.com/s/3aw663fy9BbiBeLji'bie
|
|
|
|
def up_to_nextcloud():
|
|
"""
|
|
上传统计信息至nextcloud
|
|
"""
|
|
from nextcloud import NextCloud
|
|
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
|
|
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
|
|
NEXTCLOUD_PASSWORD = "goojia123456"
|
|
aim_file_path = '共享/统计/常用覆盖率统计' + output_date + '.xlsx'
|
|
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
|
|
nxc.upload_file(aim_file_name, aim_file_path)
|
|
|
|
up_to_nextcloud()
|