Files
airflow/dags/jh/price/price_third_gxd.py
T
2024-07-23 10:58:51 +08:00

75 lines
4.5 KiB
Python

import logging
import os
import sys
import pandas as pd
import requests
from goojia_utils import dateutils, nextcloudutils, dingutils
from citymap import city_name_id_map
from pqsql_connect import get_community_base_price,get_evaluation_third_price,get_third_price
from mongodb_connect import get_district_stat_on,get_seaweed,get_gxd_price
yearmonth = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(pd.datetime.utcnow(), "%Y%m")
latest_year_month = dateutils.format_date(dateutils.add_months(dateutils.parse_date(yearmonth, '%Y%m'), months=-1), '%Y%m')
pathAll = "/home/airflow/data/third_gxd"
if not os.path.exists(pathAll):
os.makedirs(pathAll)
df_concat = pd.DataFrame()
for city in city_name_id_map:
print(city)
city_id = city_name_id_map[city]
base_price = get_community_base_price(city_id, yearmonth=yearmonth)
base_price.rename(columns={'price': '当月均价', 'type': '计算方式'}, inplace=True)
base_price_last = get_community_base_price(city_id, yearmonth=latest_year_month)
base_price_last.rename(columns={'price': '上月均价', 'type': '上月计算方式'}, inplace=True)
get_evaluation_third_price_df = get_evaluation_third_price(city_id, yearmonth=yearmonth,type='第三方')
get_evaluation_third_price_df.rename(columns={'third_price': '第三方对标价格'}, inplace=True)
deal_price_df = get_evaluation_third_price(city_id, yearmonth=yearmonth,type='成交')
deal_id = deal_price_df['_id'].tolist()
get_evaluation_third_price_df = get_evaluation_third_price_df[~(get_evaluation_third_price_df['_id'].isin(deal_id))]
third_price = get_third_price(cityid=city_id, yearmonth=latest_year_month)
lianjia_price =third_price[third_price['source']=='链家'].drop_duplicates(subset=["_id"], keep="first")
lianjia_price.rename(columns={'price': '链家价格'}, inplace=True)
anjuke_price =third_price[third_price['source']=='安居客'].drop_duplicates(subset=["_id"], keep="first")
anjuke_price.rename(columns={'price': '安居客价格'}, inplace=True)
base_price_gj = get_district_stat_on(city)
seaweed_df = get_seaweed(city)
if seaweed_df.empty:
continue
seaweed_df['_id'] = seaweed_df['_id'].astype('str')
if base_price_gj.empty:
continue
base_price_gj = base_price_gj.merge(seaweed_df[['_id', 'region', 'name']], on=['region', 'name'], how='left')
base_price_gj.rename(columns={'avg_price': '房价网2C价格', 'avg_price_2b': '房价网2B价格'}, inplace=True)
base_price_gxd = get_gxd_price()
base_price_gxd.rename(columns={'district_name': 'name', 'price': '国信达价格'}, inplace=True)
data_df_temp = seaweed_df[['_id','city', 'region', 'name']].merge(base_price_gxd[['city', 'region', 'name', '国信达价格', 'c_date']], on=['city', 'region', 'name'])
data_df_temp = data_df_temp.sort_values(by="c_date", ascending=False)
data_df_temp = data_df_temp.drop_duplicates(subset=["city", "region", "name"], keep="first")
if base_price.empty or base_price_last.empty or get_evaluation_third_price_df.empty:
continue
price_merge = base_price.merge(base_price_last[['_id', '上月均价', '上月计算方式']], on='_id', how='left')
price_merge = price_merge.merge(get_evaluation_third_price_df[['_id','第三方对标价格']],on='_id')
price_merge = price_merge.merge(lianjia_price[['_id','链家价格']],on='_id',how='left')
price_merge = price_merge.merge(anjuke_price[['_id','安居客价格']],on='_id',how='left')
price_merge = price_merge.merge(base_price_gj[['_id','房价网2C价格','房价网2B价格']], on='_id', how='left')
price_merge = price_merge.merge(data_df_temp[['_id','国信达价格']], on='_id', how='left')
price_merge.rename(columns={'city': '城市', 'district': '区域', '_id': '小区ID', 'name': '小区名','yearmonth':'月份'}, inplace=True)
df_concat = pd.concat([df_concat,price_merge])
# df_concat.to_excel(yearmonth + '小区第三方均价输出.xlsx', encoding='utf-8', index=False)
output_path = pathAll + "/" + yearmonth + '小区第三方均价输出.xlsx'
df_concat.to_excel(output_path, index=False)
cloudPath = '共享/建行/' + yearmonth + '小区第三方均价输出.xlsx'
nextcloudutils.up_to_nextcloud(output_path, cloudPath)
logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘")
infos = "【第三方价格自动检测】"
dingutils.send_text(infos + "数据Excel文件已经上传到云盘,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情", "1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736")