新增字段统计

This commit is contained in:
shenjian@fangjia.com
2023-03-29 15:55:48 +08:00
parent 3ab1d31852
commit 2eecb42410
@@ -5,6 +5,9 @@
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ClickHouseClient, ElasticSearchClient
import pandas as pd
import os
import logging
log = logging.getLogger(__name__)
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -18,10 +21,12 @@ es206_client = ElasticSearchClient(host='192.168.0.206')
city_df = pd.DataFrame(list(server_area_conn.find({}, {'city_grade': 1, 'display': 1})))
city_df = city_df.rename(columns={'city_grade': 'city_level', 'display': 'city'})
city_df = city_df[city_df['city_level'] != 0]
# 昆玉无小区
city_df = city_df[city_df['city'] != '昆玉'].reset_index(drop=True)
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'addr_alias': 1})
.batch_size(10000)))
.batch_size(100000)))
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
@@ -39,6 +44,8 @@ for index, item in city_df.iterrows():
seaweed_count = 0
# 楼盘数量
city_df.loc[index, 'community_count'] = len(region_df)
if len(region_df) == 0:
log.error(f"该城市{city}小区数量为0")
for index1, item1 in region_df.iterrows():
flag = True
if type(item1['alias']) == list: