96 lines
3.9 KiB
Python
96 lines
3.9 KiB
Python
"""
|
|
每个小区近一年成交统计
|
|
"""
|
|
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ElasticSearchClient
|
|
import pandas as pd
|
|
import datetime
|
|
import logging
|
|
import sys
|
|
|
|
|
|
es45_client = ElasticSearchClient(host='192.168.0.45')
|
|
deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price')
|
|
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
|
|
|
|
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx')
|
|
cities = city_df['城市列表'].values.tolist()
|
|
|
|
path = "/home/airflow/data/mt"
|
|
|
|
output_date = sys.argv[1] if sys.argv[1] else dateutils.format_date(datetime.datetime.now(), '%Y%m')
|
|
|
|
month_start = dateutils.format_date(dateutils.add_months(dateutils.parse_date(output_date, '%Y%m'), months=-1), '%Y%m') + '01'
|
|
curr_month_start = output_date + '01'
|
|
logging.info("日期信息:" + month_start + ' ' + curr_month_start)
|
|
|
|
month_start_date = dateutils.parse_date(month_start, '%Y%m%d')
|
|
curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
|
|
|
|
result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8')
|
|
|
|
if 'deal_num' in result_df.columns:
|
|
result_df = result_df.drop(columns=['deal_num'])
|
|
"""
|
|
MONGO城市成交数据
|
|
"""
|
|
deal_num_df = pd.DataFrame(list(deal_price_conn.aggregate([
|
|
{'$match': {'match_id': {'$exists': True}, 'area': {'$gte': 20}, 'avg_price': {'$gt': 0},
|
|
'trade_date': {'$gte': dateutils.add_months(curr_month_start_date, months=-12),
|
|
'$lt': curr_month_start_date}}},
|
|
{'$group': {'_id': '$match_id', 'deal_num': {'$sum': 1}}}
|
|
])))
|
|
|
|
deal_num_df = deal_num_df.rename(columns={'_id': 'id'})
|
|
result_df = pd.merge(result_df, deal_num_df, how='left', on='id')
|
|
|
|
"""
|
|
ES上海成交数据
|
|
"""
|
|
deal_es_num_df = es45_client.scan_to_data_frame('esf_deal_price_lansi', {
|
|
"_source": ["fjCity", "fjRegion", "fjName"],
|
|
"query": {
|
|
"bool": {
|
|
"must": [
|
|
{
|
|
"exists": {"field": "fjName"}
|
|
},
|
|
{
|
|
"range": {
|
|
"contractTime": {
|
|
"gte": dateutils.add_months(curr_month_start_date, months=-12),
|
|
"lt": curr_month_start_date
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
})
|
|
deal_es_num_df = deal_es_num_df.rename(columns={'fjCity': 'city', 'fjRegion': 'region', 'fjName': 'name'})
|
|
deal_es_num_df = deal_es_num_df.drop(columns={'_id'})
|
|
deal_es_num_df['es_num'] = 0
|
|
deal_es_num_df['es_num'] = deal_es_num_df.groupby(by=['city', 'region', 'name'])['es_num'].transform('count')
|
|
deal_es_num_df = deal_es_num_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first')
|
|
|
|
result_df = pd.merge(result_df, deal_es_num_df, how='left', on=['city', 'region', 'name'])
|
|
result_df['deal_num'].fillna(0, inplace=True)
|
|
result_df['es_num'].fillna(0, inplace=True)
|
|
result_df['deal_num'] += result_df['es_num']
|
|
result_df = result_df.drop(columns={'es_num'})
|
|
|
|
# 暂时标识所有小区均未删除
|
|
result_df['status'] = 0
|
|
|
|
# 剔除id为na的
|
|
result_df = result_df[result_df['id'].notna()]
|
|
# 在导出时最后一次去重,防止小区重复
|
|
result_df = result_df.drop_duplicates(subset=['id'], keep='first')
|
|
|
|
columns = ['id', 'city', 'region', 'name', 'alias', 'address', 'addr_alias', 'avg_price', 'rent_price', 'lng', 'lat',
|
|
'block', 'loop_line_info', 'business_quarter', 'households', 'property_fee', 'car_site_count',
|
|
'property_life', 'house_property',
|
|
'all_area', 'build_area', 'plot_ratio', 'landscaping_ratio', 'property_type', 'build_type',
|
|
'build_structure', 'finish_building_date',
|
|
'developer', 'house_type', 'house_area', 'deal_num', 'district_facilitys', 'school', 'market', 'subway',
|
|
'bank', 'hospital', 'status']
|
|
result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False, columns=columns) |