新增美团数据

This commit is contained in:
Jian Shen
2022-06-28 09:35:52 +08:00
parent 669e2c6acb
commit 27765c33a2
10 changed files with 374 additions and 1 deletions
+2 -1
View File
@@ -1,2 +1,3 @@
poi/business/.*
stat/evaluate/*
stat/evaluate/*
mt/supply/*
View File
+28
View File
@@ -0,0 +1,28 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
dag_id="mt_dag",
# 每月5日运行
schedule_interval="0 5 5 * *",
start_date=pendulum.datetime(2022, 6, 28, tz="Asia/Shanghai"),
catchup=True,
) as dag:
t1 = BashOperator(
task_id='mt_comm',
bash_command='python /opt/airflow/dags/repo/dags/mt/supply/comm.py',
)
t2 = BashOperator(
task_id='mt_rent_avg_price',
bash_command='python /opt/airflow/dags/repo/dags/mt/supply/rent_avg_price.py',
)
t3 = BashOperator(
task_id='mt_facilitys',
bash_command='python /opt/airflow/dags/repo/dags/mt/supply/facilitys.py',
)
t4 = BashOperator(
task_id='mt_deal',
bash_command='python /opt/airflow/dags/repo/dags/mt/supply/deal.py',
)
t1 >> t2 >> t3 >> t4
View File
+68
View File
@@ -0,0 +1,68 @@
"""
小区属性导出
"""
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
import pandas as pd
import datetime
import os
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
path = "/home/airflow/data/mt"
if not os.path.exists(path):
os.makedirs(path)
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx')
cities = city_df['城市列表'].values.tolist()
seaweed_df = pd.DataFrame()
for city in cities:
temp_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0, 'city': city}).batch_size(10000)))
seaweed_df = pd.concat([seaweed_df, temp_df])
seaweed_result_df = seaweed_df[
['_id', 'city', 'region', 'name', 'alias', 'address', 'addr_alias', 'lng', 'lat', 'block', 'loop_line_info',
'business_quarter',
'household_count', 'estate_charge', 'car_site_count', 'property_life', 'house_property', 'all_area', 'build_area',
'cubage_percent', 'virescence_percent', 'estate_type2', 'build_type', 'build_structure', 'finish_building_date',
'developer',
'main_type', 'main_type_area']]
def format_seaweed(x):
"""
格式化输出小区属性
:param x:
:return:
"""
names = [x.name]
try:
if x.alias is not None and str(x.alias) != 'nan':
names.extend(x.alias)
x.alias = ','.join(x.alias)
if x.addr_alias is not None and str(x.addr_alias) != 'nan':
x.addr_alias = ','.join(x.addr_alias)
x.landscaping_ratio = commutils.format_greening_rate(x.landscaping_ratio)
x.all_area = commutils.format_all_area(x.all_area)
x.build_area = commutils.format_build_area(x.build_area)
x.car_site_count = commutils.format_car_site_count(x.car_site_count)
x.households = commutils.format_household_count(x.households)
x.lng = commutils.format_lat_lng(x.lng)
x.lat = commutils.format_lat_lng(x.lat)
x.plot_ratio = commutils.format_plot_ratio(x.plot_ratio)
if type(x.alias) == str and '@' in x.alias:
x.alias = None
return x
except Exception as e:
print(e)
print(x)
pass
seaweed_new_result_df = seaweed_result_df.apply(lambda x: format_seaweed(x), axis=1)
seaweed_new_result_df = seaweed_new_result_df.rename(columns={'_id': 'id', 'household_count': 'households', 'estate_charge': 'property_fee',
'cubage_percent' : 'plot_ratio', 'virescence_percent': 'landscaping_ratio', 'estate_type2': 'property_type',
'main_type': 'house_type', 'main_type_area': 'house_area'})
seaweed_new_result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv',
sep='@', encoding='utf-8', index=False)
Binary file not shown.
+90
View File
@@ -0,0 +1,90 @@
"""
每个小区近一年成交统计
"""
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ElasticSearchClient
import pandas as pd
import datetime
import logging
es45_client = ElasticSearchClient(host='192.168.0.45')
deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price')
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx')
cities = city_df['城市列表'].values.tolist()
# 1. 获取「今天」
today = datetime.date.today()
# 2. 获取当前月的第一天
first = today.replace(day=1)
# 3. 减一天,得到上个月的最后一天
last_month = first - datetime.timedelta(days=1)
month_start = last_month.strftime("%Y%m") + '01'
curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01'
logging.info(month_start + ' ' + curr_month_start)
month_start_date = dateutils.parse_date(month_start, '%Y%m%d')
curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
path = "/home/airflow/data/mt"
result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8')
"""
MONGO城市成交数据
"""
deal_num_df = pd.DataFrame(list(deal_price_conn.aggregate([
{'$match': {'match_id': {'$exists': True}, 'area': {'$gte': 20}, 'avg_price': {'$gt': 0},
'trade_date': {'$gte': dateutils.add_months(curr_month_start_date, months=-12),
'$lt': curr_month_start_date}}},
{'$group': {'_id': '$match_id', 'deal_num': {'$sum': 1}}}
])))
deal_num_df = deal_num_df.rename(columns={'_id': 'id'})
result_df = pd.merge(result_df, deal_num_df, how='left', on='id')
"""
ES上海成交数据
"""
deal_es_num_df = es45_client.scan_to_data_frame('esf_deal_price_lansi', {
"_source": ["fjCity", "fjRegion", "fjName"],
"query": {
"bool": {
"must": [
{
"exists": {"field": "fjName"}
},
{
"range": {
"contractTime": {
"gte": dateutils.add_months(curr_month_start_date, months=-12),
"lt": curr_month_start_date
}
}
}
]
}
}
})
deal_es_num_df = deal_es_num_df.rename(columns={'fjCity': 'city', 'fjRegion': 'region', 'fjName': 'name'})
deal_es_num_df = deal_es_num_df.drop(columns={'_id'})
deal_es_num_df['es_num'] = 0
deal_es_num_df['es_num'] = deal_es_num_df.groupby(by=['city', 'region', 'name'])['es_num'].transform('count')
deal_es_num_df = deal_es_num_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first')
result_df['deal_num'].fillna(0, inplace=True)
result_df = pd.merge(result_df, deal_es_num_df, how='left', on=['city', 'region', 'name'])
result_df['deal_num'] += result_df['es_num']
result_df = result_df.drop(columns={'es_num'})
columns = ['id', 'city', 'region', 'name', 'alias', 'address', 'addr_alias', 'avg_price', 'rent_price', 'lng', 'lat',
'block', 'loop_line_info', 'business_quarter', 'households', 'property_fee', 'car_site_count',
'property_life', 'house_property',
'all_area', 'build_area', 'plot_ratio', 'landscaping_ratio', 'property_type', 'build_type',
'build_structure', 'finish_building_date',
'developer', 'house_type', 'house_area', 'deal_num', 'district_facilitys', 'school', 'market', 'subway',
'bank', 'hospital']
result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@',
encoding='utf-8', index=False, columns=columns)
+92
View File
@@ -0,0 +1,92 @@
"""
周边配套数量统计
"""
from goojia_utils import MAIN_MONGO_CLIENT, dateutils
import pandas as pd
import datetime
import logging
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx')
cities = city_df['城市列表'].values.tolist()
# 1. 获取「今天」
today = datetime.date.today()
# 2. 获取当前月的第一天
first = today.replace(day=1)
# 3. 减一天,得到上个月的最后一天
last_month = first - datetime.timedelta(days=1)
month_start = last_month.strftime("%Y%m") + '01'
curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01'
logging.info(month_start + ' ' + curr_month_start)
month_start_date = dateutils.parse_date(month_start, '%Y%m%d')
curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
path = "/home/airflow/data/mt"
district_facilitys_df = pd.DataFrame()
for city in cities:
temp_df = pd.DataFrame(
list(district_facilitys_conn.find({'city': city}, {'city': 1, 'region': 1, 'district_name': 1, 'school': 1,
'supermarket': 1, 'subway': 1, 'bank': 1, 'hospital': 1}).batch_size(10000)))
district_facilitys_df = pd.concat([district_facilitys_df, temp_df])
def count_facilitys(x):
ret = x
try:
if 'school' in x and x['school'] is not None and str(x['school']) != 'nan':
for item in x['school']:
if item is not None:
ret.school = len(item)
if 'supermarket' in x and x['supermarket'] is not None and str(x['supermarket']) != 'nan':
for item in x['supermarket']:
if item is not None:
ret.supermarket = len(item)
if 'subway' in x and x['subway'] is not None and str(x['subway']) != 'nan':
for item in x['subway']:
if item is not None:
ret.subway = len(item)
if 'bank' in x and x['bank'] is not None and str(x['bank']) != 'nan':
for item in x['bank']:
if item is not None:
ret.bank = len(item)
if 'hospital' in x and x['hospital'] is not None and str(x['hospital']) != 'nan':
for item in x['hospital']:
if item is not None:
ret.hospital = len(item)
return ret
except Exception as e:
print(e)
print(x)
return None
district_facilitys_new_df = district_facilitys_df.apply(lambda x: count_facilitys(x), axis=1)
district_facilitys_new_df = district_facilitys_new_df[['city', 'region', 'district_name', 'school', 'bank', 'hospital', 'supermarket', 'subway']]
district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_name': 'name', 'supermarket': 'market'})
result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8')
result_df = pd.merge(result_df, district_facilitys_new_df, how='left', on=['city', 'region', 'name'])
result_df.loc[:, 'district_facilitys'] = '学校、商场、地铁、银行、医院'
def format_facilitys(x):
if type(x['school']) is not int:
x.school = None
if type(x['bank']) is not int:
x.bank = None
if type(x['hospital']) is not int:
x.hospital = None
if type(x['market']) is not int:
x.market = None
if type(x['subway']) is not int:
x.subway = None
return x
result_df = result_df.apply(lambda x: format_facilitys(x), axis=1)
+94
View File
@@ -0,0 +1,94 @@
"""
租金、均价导出
"""
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
import pandas as pd
import datetime
import logging
district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat')
sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent')
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx')
cities = city_df['城市列表'].values.tolist()
path = "/home/airflow/data/mt"
# 1. 获取「今天」
today = datetime.date.today()
# 2. 获取当前月的第一天
first = today.replace(day=1)
# 3. 减一天,得到上个月的最后一天
last_month = first - datetime.timedelta(days=1)
month_start = last_month.strftime("%Y%m") + '01'
curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01'
logging.info("日期信息:" + month_start + ' ' + curr_month_start)
month_start_date = dateutils.parse_date(month_start, '%Y%m%d')
curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
threshold_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/租金和房价设置阀值(19个城市).xlsx')
min_rent = dict()
max_rent = dict()
min_price = dict()
max_price = dict()
for index, item in threshold_df.iterrows():
if item['租金最小值'] != '缺租金':
min_rent[item['城市']] = item['租金最小值']
if item['租金最大值'] != '缺租金':
max_rent[item['城市']] = item['租金最大值']
min_price[item['城市']] = item['均价最小值']
avg_df = pd.DataFrame()
for city in cities:
# 求当月均价均值
temp_df = pd.DataFrame(list(district_stat_conn.find(
{'weekend': {'$gt': month_start_date, '$lte': curr_month_start_date}, 'city': city},
{'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1, 'weekend': 1}).batch_size(10000)))
temp_df['avg_price_mean'] = temp_df.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform('mean').astype(int)
temp_df = temp_df[['city', 'region', 'district_name', 'avg_price_mean']]
temp_df = temp_df.drop_duplicates(subset=['city', 'region', 'district_name'], keep='first')
# 过滤阈值
if city in min_price:
temp_df = temp_df[(temp_df['avg_price_mean'] >= min_price[city]) & (temp_df['avg_price_mean'] <= max_price[city])]
avg_df = pd.concat([avg_df, temp_df])
avg_df = avg_df.rename(columns={'district_name': 'name', 'avg_price_mean': 'avg_price'})
rent_df = pd.DataFrame()
for city in cities:
# 求租金
temp_df = pd.DataFrame(list(sqmetre_rent_conn.find(
{'weekend': {'$gt': month_start_date, '$lte': curr_month_start_date}, 'city': city},
{'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1}).batch_size(10000)))
# 根据阈值过滤
if city in min_rent:
temp_df = temp_df[(temp_df['avg_price'] >= min_rent[city]) & (temp_df['avg_price'] <= max_rent[city])]
rent_df = pd.concat([rent_df, temp_df])
rent_df = rent_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_price'})
avg_df = avg_df[['city', 'region', 'name', 'avg_price']]
rent_df = rent_df[['city', 'region', 'name', 'rent_price']]
seaweed_new_result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8')
result_df = pd.merge(seaweed_new_result_df, avg_df, how='left', on=['city', 'region', 'name'])
result_df = pd.merge(result_df, rent_df, how='left', on=['city', 'region', 'name'])
def format_rent(x):
"""
过滤租售比,大于2000租金为空
:param x:
:return:
"""
x.rent_price = commutils.format_rent_price(x.avg_price, x.rent_price)
return x
result_df = result_df.apply(lambda x: format_rent(x), axis=1)
result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8', index=False)