Merge branch 'master' of http://gitlab.fangjia.com/dev0/airflow
This commit is contained in:
@@ -208,7 +208,7 @@ def load_comms(city):
|
||||
seaweed['_id'] = seaweed['_id'].apply(lambda x: str(x))
|
||||
seaweed['lng'] = seaweed['lng'] / 10000000000
|
||||
seaweed['lat'] = seaweed['lat'] / 10000000000
|
||||
seaweed = seaweed.rename(columns={"create_year": "build_year", "estate_type2": "estate_type", "business_quarter": "block"})
|
||||
seaweed = seaweed.rename(columns={"create_year": "build_year", "estate_type2": "estate_type", "business_quarter": "block", "_id": "comm_id"})
|
||||
year = datetime.now().year
|
||||
seaweed['build_year'] = seaweed['build_year'].apply(lambda x: extract_years(x))
|
||||
seaweed['age'] = seaweed['build_year'].apply(lambda x: (year - x) if pd.notna(x) else None)
|
||||
@@ -259,8 +259,7 @@ def load_comm_stat(city):
|
||||
|
||||
firstDayOfThisMonth = dateutils.parse_date(dateutils.format_date(datetime.utcnow(), "%Y-%m"), "%Y-%m")
|
||||
lastDayOfPreMonth = dateutils.add_months(firstDayOfThisMonth, days=-1)
|
||||
sqmetre_rent = pd.DataFrame(list(
|
||||
sqmetre_rent_client.find({"weekend": lastDayOfPreMonth, "cat": "district", "city": city},
|
||||
sqmetre_rent = pd.DataFrame(list(sqmetre_rent_client.find({"weekend": lastDayOfPreMonth, "cat": "district", "city": city},
|
||||
{"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0})))
|
||||
sqmetre_rent = sqmetre_rent.rename(columns={"district_name": "name", "avg_price": "sqmetre_rent"})
|
||||
stat = stat.merge(sqmetre_rent, on=['city', 'region', 'name'], how='left')
|
||||
@@ -275,13 +274,12 @@ def load_comms_summary(city):
|
||||
all_community_summary = []
|
||||
for doc in elastic_client.scan_by_sql(f"select * from community_summary where city = '{city}'"):
|
||||
source = doc['_source']
|
||||
community_summary = {"city": source['city'], "region": source['region'], "name": source['name']}
|
||||
community_summary = {"comm_id": source["id"]}
|
||||
for key in source.keys():
|
||||
if key.endswith("num") or key.endswith("_distance"):
|
||||
community_summary[key] = source[key]
|
||||
all_community_summary.append(community_summary)
|
||||
all_community_summary = pd.DataFrame(all_community_summary)
|
||||
all_community_summary = all_community_summary.drop_duplicates(subset=['city', 'region', 'name'])
|
||||
return all_community_summary
|
||||
|
||||
|
||||
@@ -455,7 +453,7 @@ def load_data_by_city(city):
|
||||
stat = load_comm_stat(city=city)
|
||||
comms_summary = load_comms_summary(city=city)
|
||||
data = comms.merge(stat, how="left", on=['city', 'region', 'name'])
|
||||
data = data.merge(comms_summary, how="left", on=['city', 'region', 'name'])
|
||||
data = data.merge(comms_summary, how="left", on=['comm_id'])
|
||||
num_fields = ["supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num",
|
||||
"key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num",
|
||||
"ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num",
|
||||
@@ -482,8 +480,6 @@ def process_city(city):
|
||||
logger.info(f"开始计算楼盘评级:{city}")
|
||||
data = load_data_by_city(city)
|
||||
data = comms_rating(data)
|
||||
data["comm_id"] = data['_id']
|
||||
|
||||
db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy",
|
||||
"rent_to_price_ratio", "avg_price", "total_score", "community_quality_score",
|
||||
"community_price_score", "location_advantage_score", "flowability_score", "living_facilities_score",
|
||||
@@ -492,7 +488,7 @@ def process_city(city):
|
||||
"hospital_rating", "supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num",
|
||||
"key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num",
|
||||
"ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "build_year", "age",
|
||||
"cubage_percent", "virescence_percent", "estate_charge", 'estate_type', 'resistance_coef']
|
||||
"cubage_percent", "virescence_percent", "estate_charge", 'estate_type', 'resistance_coef', "region_rating"]
|
||||
|
||||
df_simple = data[db_fields].copy()
|
||||
if "block" not in df_simple.columns:
|
||||
@@ -504,12 +500,10 @@ def process_city(city):
|
||||
'education_score', 'hospital_score']:
|
||||
key = "" if factor_score_field == "total_score" else f"_{factor_score_field.replace('_score', '')}"
|
||||
|
||||
df_simple[f'city{key}_rank'] = df_simple.groupby('city')[factor_score_field].rank(
|
||||
method='dense', ascending=False).astype(int)
|
||||
df_simple[f'city{key}_rank'] = df_simple[factor_score_field].rank(method='min', ascending=False).astype(int)
|
||||
df_simple[f'city{key}_total_rank'] = df_simple[f'city{key}_rank'].max()
|
||||
|
||||
df_simple[f'region{key}_rank'] = df_simple.groupby(['city', 'region'])[factor_score_field].rank(
|
||||
method='dense', ascending=False).astype(int)
|
||||
df_simple[f'region{key}_rank'] = df_simple.groupby(['city', 'region'])[factor_score_field].rank(method='min', ascending=False).astype(int)
|
||||
df_simple[f'region{key}_total_rank'] = df_simple[f'region{key}_rank'].max()
|
||||
|
||||
df_simple[f'block{key}_rank'] = None
|
||||
@@ -523,7 +517,7 @@ def process_city(city):
|
||||
data_not_null['region_block'] = data_not_null['region'] + "_" + data_not_null['block']
|
||||
for item in data_not_null['region_block'].unique():
|
||||
group = data_not_null[(data_not_null['region'] == str(item).split("_")[0])&(data_not_null['block'] == str(item).split("_")[1])].copy()
|
||||
group[f'block{key}_rank'] = group.groupby(['city', 'region', 'block'])[factor_score_field].rank(method='min', ascending=False).astype(int)
|
||||
group[f'block{key}_rank'] = group[factor_score_field].rank(method='min', ascending=False).astype(int)
|
||||
group[f'block{key}_total_rank'] = group[f'block{key}_rank'].max()
|
||||
buffer.append(group)
|
||||
df_simple = pd.concat(buffer)
|
||||
@@ -532,9 +526,16 @@ def process_city(city):
|
||||
for score_factor in ['total_score', 'community_quality_score', 'community_price_score','location_advantage_score', 'flowability_score', 'living_facilities_score','education_score', 'hospital_score']:
|
||||
df_simple[score_factor] = df_simple[score_factor].apply(lambda x: round(x, 2))
|
||||
df_simple['rent_to_price_ratio'] = df_simple['rent_to_price_ratio'].apply(lambda x: round(x, 2))
|
||||
df_simple['yoy'] = df_simple['yoy'].apply(lambda x: round(x, 2))
|
||||
updater = MongoDataFrameUpdater(community_rate_client)
|
||||
updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id'], dtype={"avg_price": int,
|
||||
"build_year": int, 'resistance_coef': int})
|
||||
dtype = {"avg_price": int,"build_year": int, 'resistance_coef': int}
|
||||
for field in db_fields:
|
||||
if "_rank" in field or "_num" in field:
|
||||
dtype[field] = int
|
||||
for field in df_simple.columns:
|
||||
if "_rank" in field or "_num" in field:
|
||||
dtype[field] = int
|
||||
updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id'], dtype=dtype)
|
||||
logger.info(f"楼盘评级计算结束:{city}")
|
||||
except Exception as e:
|
||||
logging.error(f"{city}楼盘评级计算失败;{e}")
|
||||
|
||||
Reference in New Issue
Block a user