Merge remote-tracking branch 'origin/master'
This commit is contained in:
@@ -14,6 +14,8 @@ from sklearn.mixture import GaussianMixture
|
||||
from sklearn.neighbors import BallTree
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
import multiprocessing
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed")
|
||||
@@ -474,75 +476,74 @@ def load_data_by_city(city):
|
||||
data['volume_price_ratio'] = data['volume_price_ratio'].fillna(data['volume_price_ratio'].mean())
|
||||
return data
|
||||
|
||||
import multiprocessing
|
||||
|
||||
def process_city(city):
|
||||
try:
|
||||
logger.info(f"开始计算楼盘评级:{city}")
|
||||
data = load_data_by_city(city)
|
||||
data = comms_rating(data)
|
||||
data["comm_id"] = data['_id']
|
||||
|
||||
db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy",
|
||||
"rent_ro_price_ratio", "avg_price", "total_score", "community_quality_score",
|
||||
"community_price_score", "location_advantage_score", "flowability_score", "living_facilities_score",
|
||||
"education_score", "hospital_score", "rating", "community_quality_rating", "community_price_rating",
|
||||
"location_advantage_rating", "flowability_rating", "living_facilities_rating", "education_rating",
|
||||
"hospital_rating", "supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num",
|
||||
"key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num",
|
||||
"ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "build_year", "age",
|
||||
"cubage_percent", "virescence_percent", "estate_charge", 'estate_type']
|
||||
|
||||
df_simple = data[db_fields].copy()
|
||||
if "block" not in df_simple.columns:
|
||||
df_simple['block'] = None
|
||||
|
||||
for factor_score_field in ['total_score', 'community_quality_score', 'community_price_score',
|
||||
'location_advantage_score', 'flowability_score', 'living_facilities_score',
|
||||
'education_score', 'hospital_score']:
|
||||
key = "" if factor_score_field == "total_score" else f"_{factor_score_field.replace('_score', '')}"
|
||||
|
||||
df_simple[f'city{key}_rank'] = df_simple.groupby('city')[factor_score_field].rank(
|
||||
method='dense', ascending=False).astype(int)
|
||||
df_simple[f'city{key}_total_rank'] = df_simple[f'city{key}_rank'].max()
|
||||
|
||||
df_simple[f'region{key}_rank'] = df_simple.groupby(['city', 'region'])[factor_score_field].rank(
|
||||
method='dense', ascending=False).astype(int)
|
||||
df_simple[f'region{key}_total_rank'] = df_simple[f'region{key}_rank'].max()
|
||||
|
||||
df_simple[f'block{key}_rank'] = None
|
||||
df_simple[f'block{key}_total_rank'] = None
|
||||
if df_simple['block'].notna().any():
|
||||
subset = df_simple[df_simple['block'].notna()]
|
||||
df_simple.loc[subset.index, f'block{key}_rank'] = subset.groupby(
|
||||
['city', 'region', 'block'])[factor_score_field].rank(method='dense', ascending=False).astype(int)
|
||||
df_simple.loc[subset.index, f'block{key}_total_rank'] = df_simple.loc[subset.index][f'block{key}_rank'].max()
|
||||
|
||||
for score_factor in ['total_score', 'community_quality_score', 'community_price_score',
|
||||
'location_advantage_score', 'flowability_score', 'living_facilities_score',
|
||||
'education_score', 'hospital_score']:
|
||||
df_simple[score_factor] = df_simple[score_factor].apply(lambda x: round(x, 2))
|
||||
|
||||
df_simple['rent_ro_price_ratio'] = df_simple['rent_ro_price_ratio'].apply(lambda x: round(x, 2))
|
||||
df_simple['avg_price'] = df_simple['avg_price'].astype(int)
|
||||
|
||||
updater = MongoDataFrameUpdater(community_rate_client)
|
||||
updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id'])
|
||||
logger.info(f"楼盘评级计算结束:{city}")
|
||||
except Exception as e:
|
||||
logging.error(f"{city}楼盘评级计算失败;{e}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--city', type=str, default='all')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.city == 'all':
|
||||
cities = areautils.get_all_cities()
|
||||
else:
|
||||
cities = [city for city in args.city.split(',')]
|
||||
cities = [city.strip() for city in args.city.split(',') if city.strip()]
|
||||
|
||||
db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy",
|
||||
"rent_ro_price_ratio", "avg_price", "total_score", "community_quality_score",
|
||||
"community_price_score",
|
||||
"location_advantage_score", "flowability_score", "living_facilities_score", "education_score",
|
||||
"hospital_score", "rating", "community_quality_rating", "community_price_rating",
|
||||
"location_advantage_rating", "flowability_rating", "living_facilities_rating",
|
||||
"education_rating", "hospital_rating", "supermarket_num", "park_num", "catering_num", "mall_num",
|
||||
"kindergarten_num",
|
||||
"key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num",
|
||||
"ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "build_year", "age",
|
||||
"cubage_percent", "virescence_percent", "estate_charge", 'estate_type']
|
||||
for city in cities:
|
||||
try:
|
||||
logger.info(f"开始计算楼盘评级:{city}")
|
||||
data = load_data_by_city(city)
|
||||
data = comms_rating(data)
|
||||
data["comm_id"] = data['_id']
|
||||
db_df = data[db_fields]
|
||||
df_simple = db_df.copy()
|
||||
if "block" not in df_simple.columns:
|
||||
df_simple['block'] = None
|
||||
for factor_score_field in ['total_score', 'community_quality_score', 'community_price_score',
|
||||
'location_advantage_score', 'flowability_score', 'living_facilities_score',
|
||||
'education_score', 'hospital_score']:
|
||||
key = "_"
|
||||
if factor_score_field != "total_score":
|
||||
key = f"_{factor_score_field.replace('_score', '')}"
|
||||
else:
|
||||
key = ""
|
||||
|
||||
df_simple[f'city{key}_rank'] = df_simple.groupby('city')[factor_score_field].rank(method='dense',
|
||||
ascending=False).astype(
|
||||
int)
|
||||
df_simple[f'city{key}_total_rank'] = df_simple[f'city{key}_rank'].max()
|
||||
df_simple[f'region{key}_rank'] = df_simple.groupby(['city', 'region'])[factor_score_field].rank(
|
||||
method='dense', ascending=False).astype(int)
|
||||
df_simple[f'region{key}_total_rank'] = df_simple[f'region{key}_rank'].max()
|
||||
# 处理bis为None的情况 - 只对非空bis值计算排名
|
||||
df_simple[f'block{key}_total_rank'] = None
|
||||
df_simple[f'block{key}_rank'] = None
|
||||
# 对bis不为None的记录计算商圈排名
|
||||
bis_not_null = df_simple['block'].notna()
|
||||
if bis_not_null.any():
|
||||
df_simple.loc[bis_not_null, f'block{key}_rank'] = \
|
||||
df_simple.loc[bis_not_null].groupby(['city', 'region', 'block'])[factor_score_field].rank(
|
||||
method='dense', ascending=False).astype(int)
|
||||
df_simple.loc[bis_not_null, f'block{key}_total_rank'] = df_simple.loc[bis_not_null][
|
||||
f'block{key}_rank'].max()
|
||||
for score_factor in ['total_score', 'community_quality_score', 'community_price_score',
|
||||
'location_advantage_score', 'flowability_score', 'living_facilities_score',
|
||||
'education_score', 'hospital_score']:
|
||||
df_simple[f'{score_factor}'] = df_simple[f'{score_factor}'].apply(lambda x: round(x, 2))
|
||||
|
||||
df_simple['rent_ro_price_ratio'] = df_simple['rent_ro_price_ratio'].apply(lambda x: round(x, 2))
|
||||
df_simple['avg_price'] = df_simple['avg_price'].astype(int)
|
||||
updater = MongoDataFrameUpdater(community_rate_client)
|
||||
updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id'])
|
||||
logger.info(f"楼盘评级计算结束:{city}")
|
||||
except Exception as e:
|
||||
print(e)
|
||||
logging.error(f"{city}楼盘评级计算失败;{e}")
|
||||
raise e
|
||||
with multiprocessing.Pool(processes=min(8, len(cities))) as pool:
|
||||
pool.map(process_city, cities)
|
||||
Reference in New Issue
Block a user