优化周边配套
This commit is contained in:
@@ -5,6 +5,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import os
|
||||
import re
|
||||
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
|
||||
@@ -37,6 +38,7 @@ def format_seaweed(x):
|
||||
"""
|
||||
try:
|
||||
if str(x.mapping_alias) != 'nan' and x.mapping_alias != None:
|
||||
# 过滤映射别名,防止含有主名
|
||||
if x.name in x.mapping_alias:
|
||||
x.mapping_alias.remove(x.name)
|
||||
x.mapping_alias = ','.join(x.mapping_alias)
|
||||
@@ -50,6 +52,25 @@ def format_seaweed(x):
|
||||
x.lng = commutils.format_lat_lng(x.lng)
|
||||
x.lat = commutils.format_lat_lng(x.lat)
|
||||
x.cubage_percent = commutils.format_plot_ratio(x.cubage_percent)
|
||||
# 过滤房型,大于5房过滤,剔除厅卫
|
||||
if 'house_type' in x and str(x['house_type']) != 'nan':
|
||||
if re.match('[1-6]室', x['house_type']) is None:
|
||||
x.house_type = None
|
||||
x.house_area = None
|
||||
else:
|
||||
x.house_type = re.sub('(\d+厅)?(\d+厨)?(\d+卫)?', '', x.house_type)
|
||||
if '1室' in x.house_type:
|
||||
if x.house_area > 125 or x.house_area < 20:
|
||||
x.house_area = None
|
||||
if '2室' in x.house_type:
|
||||
if x.house_area > 200 or x.house_area < 30:
|
||||
x.house_area = None
|
||||
if '3室' in x.house_type:
|
||||
if x.house_area > 260 or x.house_area < 60:
|
||||
x.house_area = None
|
||||
if '4室' in x.house_type:
|
||||
if x.house_area > 260 or x.house_area < 90:
|
||||
x.house_area = None
|
||||
if type(x.alias) == str and '@' in x.alias:
|
||||
x.alias = None
|
||||
|
||||
|
||||
+48
-36
@@ -28,53 +28,55 @@ curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
|
||||
path = "/home/airflow/data/mt"
|
||||
|
||||
district_facilitys_df = pd.DataFrame()
|
||||
for city in cities:
|
||||
temp_df = pd.DataFrame(
|
||||
list(district_facilitys_conn.find({'city': city}, {'city': 1, 'region': 1, 'district_name': 1, 'school': 1,
|
||||
'supermarket': 1, 'subway': 1, 'bank': 1, 'hospital': 1}).batch_size(10000)))
|
||||
district_facilitys_df = pd.concat([district_facilitys_df, temp_df])
|
||||
district_facilitys_school_df = pd.DataFrame(list(district_facilitys_conn.aggregate([
|
||||
{'$match': {'school': {'$ne': None}}},
|
||||
{'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name',
|
||||
'school': {'$size': '$school'}}}
|
||||
]).batch_size(10000)))
|
||||
district_facilitys_school_df = district_facilitys_school_df.drop(columns=['_id'])
|
||||
|
||||
district_facilitys_supermarket_df = pd.DataFrame(list(district_facilitys_conn.aggregate([
|
||||
{'$match': {'supermarket': {'$ne': None}}},
|
||||
{'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name',
|
||||
'supermarket': {'$size': '$supermarket'}}}
|
||||
]).batch_size(10000)))
|
||||
district_facilitys_supermarket_df = district_facilitys_supermarket_df.drop(columns=['_id'])
|
||||
|
||||
def count_facilitys(x):
|
||||
ret = x
|
||||
try:
|
||||
if 'school' in x and x['school'] is not None and str(x['school']) != 'nan':
|
||||
for item in x['school']:
|
||||
if item is not None:
|
||||
ret.school = len(item)
|
||||
if 'supermarket' in x and x['supermarket'] is not None and str(x['supermarket']) != 'nan':
|
||||
for item in x['supermarket']:
|
||||
if item is not None:
|
||||
ret.supermarket = len(item)
|
||||
if 'subway' in x and x['subway'] is not None and str(x['subway']) != 'nan':
|
||||
for item in x['subway']:
|
||||
if item is not None:
|
||||
ret.subway = len(item)
|
||||
if 'bank' in x and x['bank'] is not None and str(x['bank']) != 'nan':
|
||||
for item in x['bank']:
|
||||
if item is not None:
|
||||
ret.bank = len(item)
|
||||
if 'hospital' in x and x['hospital'] is not None and str(x['hospital']) != 'nan':
|
||||
for item in x['hospital']:
|
||||
if item is not None:
|
||||
ret.hospital = len(item)
|
||||
return ret
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(x)
|
||||
return None
|
||||
district_facilitys_subway_df = pd.DataFrame(list(district_facilitys_conn.aggregate([
|
||||
{'$match': {'subway': {'$ne': None}}},
|
||||
{'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name',
|
||||
'subway': {'$size': '$subway'}}}
|
||||
]).batch_size(10000)))
|
||||
district_facilitys_subway_df = district_facilitys_subway_df.drop(columns=['_id'])
|
||||
|
||||
district_facilitys_bank_df = pd.DataFrame(list(district_facilitys_conn.aggregate([
|
||||
{'$match': {'bank': {'$ne': None}}},
|
||||
{'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name',
|
||||
'bank': {'$size': '$bank'}}}
|
||||
]).batch_size(10000)))
|
||||
district_facilitys_bank_df = district_facilitys_bank_df.drop(columns=['_id'])
|
||||
|
||||
district_facilitys_new_df = district_facilitys_df.apply(lambda x: count_facilitys(x), axis=1)
|
||||
district_facilitys_hospital_df = pd.DataFrame(list(district_facilitys_conn.aggregate([
|
||||
{'$match': {'hospital': {'$ne': None}}},
|
||||
{'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name',
|
||||
'hospital': {'$size': '$hospital'}}}
|
||||
]).batch_size(10000)))
|
||||
district_facilitys_hospital_df = district_facilitys_hospital_df.drop(columns=['_id'])
|
||||
|
||||
district_facilitys_new_df = district_facilitys_new_df[['city', 'region', 'district_name', 'school', 'bank', 'hospital', 'supermarket', 'subway']]
|
||||
district_facilitys_df = pd.merge(district_facilitys_school_df,district_facilitys_supermarket_df, how='left', on=['city', 'region', 'district_name'])
|
||||
district_facilitys_df = pd.merge(district_facilitys_df,district_facilitys_subway_df, how='left', on=['city', 'region', 'district_name'])
|
||||
district_facilitys_df = pd.merge(district_facilitys_df,district_facilitys_bank_df, how='left', on=['city', 'region', 'district_name'])
|
||||
district_facilitys_df = pd.merge(district_facilitys_df,district_facilitys_hospital_df, how='left', on=['city', 'region', 'district_name'])
|
||||
|
||||
district_facilitys_new_df = district_facilitys_df[['city', 'region', 'district_name', 'school', 'bank', 'hospital', 'supermarket', 'subway']]
|
||||
district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_name': 'name', 'supermarket': 'market'})
|
||||
district_facilitys_new_df = district_facilitys_new_df.drop_duplicates(subsets=['city', 'region', 'name'], keep='first')
|
||||
|
||||
result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8')
|
||||
result_df = pd.merge(result_df, district_facilitys_new_df, how='left', on=['city', 'region', 'name'])
|
||||
result_df.loc[:, 'district_facilitys'] = '学校、商场、地铁、银行、医院'
|
||||
|
||||
|
||||
# 剔除特别大的
|
||||
def format_facilitys(x):
|
||||
if type(x['school']) is not int:
|
||||
x.school = None
|
||||
@@ -86,6 +88,16 @@ def format_facilitys(x):
|
||||
x.market = None
|
||||
if type(x['subway']) is not int:
|
||||
x.subway = None
|
||||
if x['school'] > 10:
|
||||
x.school = None
|
||||
if x['market'] > 15:
|
||||
x.market = None
|
||||
if x['subway'] > 5:
|
||||
x.subway = None
|
||||
if x['bank'] > 30:
|
||||
x.bank = None
|
||||
if x['hospital'] > 10:
|
||||
x.hospital = None
|
||||
return x
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user