75 lines
4.5 KiB
Python
75 lines
4.5 KiB
Python
import logging
|
|
import os
|
|
import sys
|
|
|
|
import pandas as pd
|
|
import requests
|
|
from goojia_utils import dateutils, nextcloudutils, dingutils
|
|
|
|
from citymap import city_name_id_map
|
|
from pqsql_connect import get_community_base_price,get_evaluation_third_price,get_third_price
|
|
from mongodb_connect import get_district_stat_on,get_seaweed,get_gxd_price
|
|
yearmonth = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(pd.datetime.utcnow(), "%Y%m")
|
|
latest_year_month = dateutils.format_date(dateutils.add_months(dateutils.parse_date(yearmonth, '%Y%m'), months=-1), '%Y%m')
|
|
|
|
pathAll = "/home/airflow/data/third_gxd"
|
|
if not os.path.exists(pathAll):
|
|
os.makedirs(pathAll)
|
|
|
|
df_concat = pd.DataFrame()
|
|
for city in city_name_id_map:
|
|
print(city)
|
|
city_id = city_name_id_map[city]
|
|
base_price = get_community_base_price(city_id, yearmonth=yearmonth)
|
|
base_price.rename(columns={'price': '当月均价', 'type': '计算方式'}, inplace=True)
|
|
base_price_last = get_community_base_price(city_id, yearmonth=latest_year_month)
|
|
base_price_last.rename(columns={'price': '上月均价', 'type': '上月计算方式'}, inplace=True)
|
|
get_evaluation_third_price_df = get_evaluation_third_price(city_id, yearmonth=yearmonth,type='第三方')
|
|
get_evaluation_third_price_df.rename(columns={'third_price': '第三方对标价格'}, inplace=True)
|
|
deal_price_df = get_evaluation_third_price(city_id, yearmonth=yearmonth,type='成交')
|
|
deal_id = deal_price_df['_id'].tolist()
|
|
get_evaluation_third_price_df = get_evaluation_third_price_df[~(get_evaluation_third_price_df['_id'].isin(deal_id))]
|
|
third_price = get_third_price(cityid=city_id, yearmonth=latest_year_month)
|
|
lianjia_price =third_price[third_price['source']=='链家'].drop_duplicates(subset=["_id"], keep="first")
|
|
lianjia_price.rename(columns={'price': '链家价格'}, inplace=True)
|
|
anjuke_price =third_price[third_price['source']=='安居客'].drop_duplicates(subset=["_id"], keep="first")
|
|
anjuke_price.rename(columns={'price': '安居客价格'}, inplace=True)
|
|
base_price_gj = get_district_stat_on(city)
|
|
seaweed_df = get_seaweed(city)
|
|
if seaweed_df.empty:
|
|
continue
|
|
seaweed_df['_id'] = seaweed_df['_id'].astype('str')
|
|
if base_price_gj.empty:
|
|
continue
|
|
base_price_gj = base_price_gj.merge(seaweed_df[['_id', 'region', 'name']], on=['region', 'name'], how='left')
|
|
base_price_gj.rename(columns={'avg_price': '房价网2C价格', 'avg_price_2b': '房价网2B价格'}, inplace=True)
|
|
base_price_gxd = get_gxd_price()
|
|
base_price_gxd.rename(columns={'district_name': 'name', 'price': '国信达价格'}, inplace=True)
|
|
data_df_temp = seaweed_df[['_id','city', 'region', 'name']].merge(base_price_gxd[['city', 'region', 'name', '国信达价格', 'c_date']], on=['city', 'region', 'name'])
|
|
data_df_temp = data_df_temp.sort_values(by="c_date", ascending=False)
|
|
data_df_temp = data_df_temp.drop_duplicates(subset=["city", "region", "name"], keep="first")
|
|
if base_price.empty or base_price_last.empty or get_evaluation_third_price_df.empty:
|
|
continue
|
|
price_merge = base_price.merge(base_price_last[['_id', '上月均价', '上月计算方式']], on='_id', how='left')
|
|
price_merge = price_merge.merge(get_evaluation_third_price_df[['_id','第三方对标价格']],on='_id')
|
|
price_merge = price_merge.merge(lianjia_price[['_id','链家价格']],on='_id',how='left')
|
|
price_merge = price_merge.merge(anjuke_price[['_id','安居客价格']],on='_id',how='left')
|
|
price_merge = price_merge.merge(base_price_gj[['_id','房价网2C价格','房价网2B价格']], on='_id', how='left')
|
|
price_merge = price_merge.merge(data_df_temp[['_id','国信达价格']], on='_id', how='left')
|
|
price_merge.rename(columns={'city': '城市', 'district': '区域', '_id': '小区ID', 'name': '小区名','yearmonth':'月份'}, inplace=True)
|
|
df_concat = pd.concat([df_concat,price_merge])
|
|
# df_concat.to_excel(yearmonth + '小区第三方均价输出.xlsx', encoding='utf-8', index=False)
|
|
|
|
output_path = pathAll + "/" + yearmonth + '小区第三方均价输出.xlsx'
|
|
df_concat.to_excel(output_path, index=False)
|
|
|
|
cloudPath = '共享/建行/' + yearmonth + '小区第三方均价输出.xlsx'
|
|
nextcloudutils.up_to_nextcloud(output_path, cloudPath)
|
|
logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘")
|
|
|
|
|
|
infos = "【第三方价格自动检测】"
|
|
dingutils.send_text(infos + "数据Excel文件已经上传到云盘,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情", "1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736")
|
|
|
|
|