新增美团数据
This commit is contained in:
+2
-1
@@ -1,2 +1,3 @@
|
||||
poi/business/.*
|
||||
stat/evaluate/*
|
||||
stat/evaluate/*
|
||||
mt/supply/*
|
||||
@@ -0,0 +1,28 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="mt_dag",
|
||||
# 每月5日运行
|
||||
schedule_interval="0 5 5 * *",
|
||||
start_date=pendulum.datetime(2022, 6, 28, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='mt_comm',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/mt/supply/comm.py',
|
||||
)
|
||||
t2 = BashOperator(
|
||||
task_id='mt_rent_avg_price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/mt/supply/rent_avg_price.py',
|
||||
)
|
||||
t3 = BashOperator(
|
||||
task_id='mt_facilitys',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/mt/supply/facilitys.py',
|
||||
)
|
||||
t4 = BashOperator(
|
||||
task_id='mt_deal',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/mt/supply/deal.py',
|
||||
)
|
||||
t1 >> t2 >> t3 >> t4
|
||||
@@ -0,0 +1,68 @@
|
||||
"""
|
||||
小区属性导出
|
||||
"""
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import os
|
||||
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
|
||||
path = "/home/airflow/data/mt"
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
|
||||
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx')
|
||||
cities = city_df['城市列表'].values.tolist()
|
||||
|
||||
seaweed_df = pd.DataFrame()
|
||||
for city in cities:
|
||||
temp_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0, 'city': city}).batch_size(10000)))
|
||||
seaweed_df = pd.concat([seaweed_df, temp_df])
|
||||
|
||||
seaweed_result_df = seaweed_df[
|
||||
['_id', 'city', 'region', 'name', 'alias', 'address', 'addr_alias', 'lng', 'lat', 'block', 'loop_line_info',
|
||||
'business_quarter',
|
||||
'household_count', 'estate_charge', 'car_site_count', 'property_life', 'house_property', 'all_area', 'build_area',
|
||||
'cubage_percent', 'virescence_percent', 'estate_type2', 'build_type', 'build_structure', 'finish_building_date',
|
||||
'developer',
|
||||
'main_type', 'main_type_area']]
|
||||
|
||||
|
||||
def format_seaweed(x):
|
||||
"""
|
||||
格式化输出小区属性
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
names = [x.name]
|
||||
try:
|
||||
if x.alias is not None and str(x.alias) != 'nan':
|
||||
names.extend(x.alias)
|
||||
x.alias = ','.join(x.alias)
|
||||
if x.addr_alias is not None and str(x.addr_alias) != 'nan':
|
||||
x.addr_alias = ','.join(x.addr_alias)
|
||||
x.landscaping_ratio = commutils.format_greening_rate(x.landscaping_ratio)
|
||||
x.all_area = commutils.format_all_area(x.all_area)
|
||||
x.build_area = commutils.format_build_area(x.build_area)
|
||||
x.car_site_count = commutils.format_car_site_count(x.car_site_count)
|
||||
x.households = commutils.format_household_count(x.households)
|
||||
x.lng = commutils.format_lat_lng(x.lng)
|
||||
x.lat = commutils.format_lat_lng(x.lat)
|
||||
x.plot_ratio = commutils.format_plot_ratio(x.plot_ratio)
|
||||
if type(x.alias) == str and '@' in x.alias:
|
||||
x.alias = None
|
||||
|
||||
return x
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(x)
|
||||
pass
|
||||
|
||||
|
||||
seaweed_new_result_df = seaweed_result_df.apply(lambda x: format_seaweed(x), axis=1)
|
||||
seaweed_new_result_df = seaweed_new_result_df.rename(columns={'_id': 'id', 'household_count': 'households', 'estate_charge': 'property_fee',
|
||||
'cubage_percent' : 'plot_ratio', 'virescence_percent': 'landscaping_ratio', 'estate_type2': 'property_type',
|
||||
'main_type': 'house_type', 'main_type_area': 'house_area'})
|
||||
seaweed_new_result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv',
|
||||
sep='@', encoding='utf-8', index=False)
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,90 @@
|
||||
"""
|
||||
每个小区近一年成交统计
|
||||
"""
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ElasticSearchClient
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import logging
|
||||
|
||||
|
||||
es45_client = ElasticSearchClient(host='192.168.0.45')
|
||||
deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price')
|
||||
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
|
||||
|
||||
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx')
|
||||
cities = city_df['城市列表'].values.tolist()
|
||||
|
||||
# 1. 获取「今天」
|
||||
today = datetime.date.today()
|
||||
# 2. 获取当前月的第一天
|
||||
first = today.replace(day=1)
|
||||
# 3. 减一天,得到上个月的最后一天
|
||||
last_month = first - datetime.timedelta(days=1)
|
||||
|
||||
month_start = last_month.strftime("%Y%m") + '01'
|
||||
curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01'
|
||||
logging.info(month_start + ' ' + curr_month_start)
|
||||
|
||||
month_start_date = dateutils.parse_date(month_start, '%Y%m%d')
|
||||
curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
|
||||
path = "/home/airflow/data/mt"
|
||||
|
||||
result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8')
|
||||
|
||||
"""
|
||||
MONGO城市成交数据
|
||||
"""
|
||||
deal_num_df = pd.DataFrame(list(deal_price_conn.aggregate([
|
||||
{'$match': {'match_id': {'$exists': True}, 'area': {'$gte': 20}, 'avg_price': {'$gt': 0},
|
||||
'trade_date': {'$gte': dateutils.add_months(curr_month_start_date, months=-12),
|
||||
'$lt': curr_month_start_date}}},
|
||||
{'$group': {'_id': '$match_id', 'deal_num': {'$sum': 1}}}
|
||||
])))
|
||||
|
||||
deal_num_df = deal_num_df.rename(columns={'_id': 'id'})
|
||||
result_df = pd.merge(result_df, deal_num_df, how='left', on='id')
|
||||
|
||||
"""
|
||||
ES上海成交数据
|
||||
"""
|
||||
deal_es_num_df = es45_client.scan_to_data_frame('esf_deal_price_lansi', {
|
||||
"_source": ["fjCity", "fjRegion", "fjName"],
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"exists": {"field": "fjName"}
|
||||
},
|
||||
{
|
||||
"range": {
|
||||
"contractTime": {
|
||||
"gte": dateutils.add_months(curr_month_start_date, months=-12),
|
||||
"lt": curr_month_start_date
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
})
|
||||
deal_es_num_df = deal_es_num_df.rename(columns={'fjCity': 'city', 'fjRegion': 'region', 'fjName': 'name'})
|
||||
deal_es_num_df = deal_es_num_df.drop(columns={'_id'})
|
||||
deal_es_num_df['es_num'] = 0
|
||||
deal_es_num_df['es_num'] = deal_es_num_df.groupby(by=['city', 'region', 'name'])['es_num'].transform('count')
|
||||
deal_es_num_df = deal_es_num_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first')
|
||||
|
||||
result_df['deal_num'].fillna(0, inplace=True)
|
||||
result_df = pd.merge(result_df, deal_es_num_df, how='left', on=['city', 'region', 'name'])
|
||||
result_df['deal_num'] += result_df['es_num']
|
||||
result_df = result_df.drop(columns={'es_num'})
|
||||
|
||||
|
||||
columns = ['id', 'city', 'region', 'name', 'alias', 'address', 'addr_alias', 'avg_price', 'rent_price', 'lng', 'lat',
|
||||
'block', 'loop_line_info', 'business_quarter', 'households', 'property_fee', 'car_site_count',
|
||||
'property_life', 'house_property',
|
||||
'all_area', 'build_area', 'plot_ratio', 'landscaping_ratio', 'property_type', 'build_type',
|
||||
'build_structure', 'finish_building_date',
|
||||
'developer', 'house_type', 'house_area', 'deal_num', 'district_facilitys', 'school', 'market', 'subway',
|
||||
'bank', 'hospital']
|
||||
result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@',
|
||||
encoding='utf-8', index=False, columns=columns)
|
||||
@@ -0,0 +1,92 @@
|
||||
"""
|
||||
周边配套数量统计
|
||||
"""
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import logging
|
||||
|
||||
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
|
||||
|
||||
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx')
|
||||
cities = city_df['城市列表'].values.tolist()
|
||||
|
||||
# 1. 获取「今天」
|
||||
today = datetime.date.today()
|
||||
# 2. 获取当前月的第一天
|
||||
first = today.replace(day=1)
|
||||
# 3. 减一天,得到上个月的最后一天
|
||||
last_month = first - datetime.timedelta(days=1)
|
||||
|
||||
month_start = last_month.strftime("%Y%m") + '01'
|
||||
curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01'
|
||||
logging.info(month_start + ' ' + curr_month_start)
|
||||
|
||||
month_start_date = dateutils.parse_date(month_start, '%Y%m%d')
|
||||
curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
|
||||
|
||||
path = "/home/airflow/data/mt"
|
||||
|
||||
district_facilitys_df = pd.DataFrame()
|
||||
for city in cities:
|
||||
temp_df = pd.DataFrame(
|
||||
list(district_facilitys_conn.find({'city': city}, {'city': 1, 'region': 1, 'district_name': 1, 'school': 1,
|
||||
'supermarket': 1, 'subway': 1, 'bank': 1, 'hospital': 1}).batch_size(10000)))
|
||||
district_facilitys_df = pd.concat([district_facilitys_df, temp_df])
|
||||
|
||||
|
||||
def count_facilitys(x):
|
||||
ret = x
|
||||
try:
|
||||
if 'school' in x and x['school'] is not None and str(x['school']) != 'nan':
|
||||
for item in x['school']:
|
||||
if item is not None:
|
||||
ret.school = len(item)
|
||||
if 'supermarket' in x and x['supermarket'] is not None and str(x['supermarket']) != 'nan':
|
||||
for item in x['supermarket']:
|
||||
if item is not None:
|
||||
ret.supermarket = len(item)
|
||||
if 'subway' in x and x['subway'] is not None and str(x['subway']) != 'nan':
|
||||
for item in x['subway']:
|
||||
if item is not None:
|
||||
ret.subway = len(item)
|
||||
if 'bank' in x and x['bank'] is not None and str(x['bank']) != 'nan':
|
||||
for item in x['bank']:
|
||||
if item is not None:
|
||||
ret.bank = len(item)
|
||||
if 'hospital' in x and x['hospital'] is not None and str(x['hospital']) != 'nan':
|
||||
for item in x['hospital']:
|
||||
if item is not None:
|
||||
ret.hospital = len(item)
|
||||
return ret
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(x)
|
||||
return None
|
||||
|
||||
|
||||
district_facilitys_new_df = district_facilitys_df.apply(lambda x: count_facilitys(x), axis=1)
|
||||
|
||||
district_facilitys_new_df = district_facilitys_new_df[['city', 'region', 'district_name', 'school', 'bank', 'hospital', 'supermarket', 'subway']]
|
||||
district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_name': 'name', 'supermarket': 'market'})
|
||||
|
||||
result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8')
|
||||
result_df = pd.merge(result_df, district_facilitys_new_df, how='left', on=['city', 'region', 'name'])
|
||||
result_df.loc[:, 'district_facilitys'] = '学校、商场、地铁、银行、医院'
|
||||
|
||||
|
||||
def format_facilitys(x):
|
||||
if type(x['school']) is not int:
|
||||
x.school = None
|
||||
if type(x['bank']) is not int:
|
||||
x.bank = None
|
||||
if type(x['hospital']) is not int:
|
||||
x.hospital = None
|
||||
if type(x['market']) is not int:
|
||||
x.market = None
|
||||
if type(x['subway']) is not int:
|
||||
x.subway = None
|
||||
return x
|
||||
|
||||
|
||||
result_df = result_df.apply(lambda x: format_facilitys(x), axis=1)
|
||||
@@ -0,0 +1,94 @@
|
||||
"""
|
||||
租金、均价导出
|
||||
"""
|
||||
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import logging
|
||||
|
||||
district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat')
|
||||
sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent')
|
||||
|
||||
|
||||
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx')
|
||||
cities = city_df['城市列表'].values.tolist()
|
||||
|
||||
path = "/home/airflow/data/mt"
|
||||
|
||||
# 1. 获取「今天」
|
||||
today = datetime.date.today()
|
||||
# 2. 获取当前月的第一天
|
||||
first = today.replace(day=1)
|
||||
# 3. 减一天,得到上个月的最后一天
|
||||
last_month = first - datetime.timedelta(days=1)
|
||||
|
||||
month_start = last_month.strftime("%Y%m") + '01'
|
||||
curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01'
|
||||
logging.info("日期信息:" + month_start + ' ' + curr_month_start)
|
||||
|
||||
month_start_date = dateutils.parse_date(month_start, '%Y%m%d')
|
||||
curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
|
||||
|
||||
threshold_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/租金和房价设置阀值(19个城市).xlsx')
|
||||
min_rent = dict()
|
||||
max_rent = dict()
|
||||
min_price = dict()
|
||||
max_price = dict()
|
||||
for index, item in threshold_df.iterrows():
|
||||
if item['租金最小值'] != '缺租金':
|
||||
min_rent[item['城市']] = item['租金最小值']
|
||||
if item['租金最大值'] != '缺租金':
|
||||
max_rent[item['城市']] = item['租金最大值']
|
||||
min_price[item['城市']] = item['均价最小值']
|
||||
|
||||
avg_df = pd.DataFrame()
|
||||
for city in cities:
|
||||
# 求当月均价均值
|
||||
temp_df = pd.DataFrame(list(district_stat_conn.find(
|
||||
{'weekend': {'$gt': month_start_date, '$lte': curr_month_start_date}, 'city': city},
|
||||
{'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1, 'weekend': 1}).batch_size(10000)))
|
||||
temp_df['avg_price_mean'] = temp_df.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform('mean').astype(int)
|
||||
temp_df = temp_df[['city', 'region', 'district_name', 'avg_price_mean']]
|
||||
temp_df = temp_df.drop_duplicates(subset=['city', 'region', 'district_name'], keep='first')
|
||||
# 过滤阈值
|
||||
if city in min_price:
|
||||
temp_df = temp_df[(temp_df['avg_price_mean'] >= min_price[city]) & (temp_df['avg_price_mean'] <= max_price[city])]
|
||||
avg_df = pd.concat([avg_df, temp_df])
|
||||
|
||||
avg_df = avg_df.rename(columns={'district_name': 'name', 'avg_price_mean': 'avg_price'})
|
||||
|
||||
rent_df = pd.DataFrame()
|
||||
for city in cities:
|
||||
# 求租金
|
||||
temp_df = pd.DataFrame(list(sqmetre_rent_conn.find(
|
||||
{'weekend': {'$gt': month_start_date, '$lte': curr_month_start_date}, 'city': city},
|
||||
{'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1}).batch_size(10000)))
|
||||
# 根据阈值过滤
|
||||
if city in min_rent:
|
||||
temp_df = temp_df[(temp_df['avg_price'] >= min_rent[city]) & (temp_df['avg_price'] <= max_rent[city])]
|
||||
rent_df = pd.concat([rent_df, temp_df])
|
||||
|
||||
rent_df = rent_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_price'})
|
||||
|
||||
avg_df = avg_df[['city', 'region', 'name', 'avg_price']]
|
||||
rent_df = rent_df[['city', 'region', 'name', 'rent_price']]
|
||||
|
||||
|
||||
seaweed_new_result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8')
|
||||
result_df = pd.merge(seaweed_new_result_df, avg_df, how='left', on=['city', 'region', 'name'])
|
||||
result_df = pd.merge(result_df, rent_df, how='left', on=['city', 'region', 'name'])
|
||||
|
||||
|
||||
def format_rent(x):
|
||||
"""
|
||||
过滤租售比,大于2000租金为空
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
x.rent_price = commutils.format_rent_price(x.avg_price, x.rent_price)
|
||||
return x
|
||||
|
||||
|
||||
result_df = result_df.apply(lambda x: format_rent(x), axis=1)
|
||||
result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8', index=False)
|
||||
Reference in New Issue
Block a user