From 9601ede115716c9c4860b3193767a5a9d2298b5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 17 May 2024 16:48:00 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B7=AE=E5=BC=82?= =?UTF-8?q?=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/price/price_exception.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dags/jh/price/price_exception.py b/dags/jh/price/price_exception.py index cfe0817..7fa0ff6 100644 --- a/dags/jh/price/price_exception.py +++ b/dags/jh/price/price_exception.py @@ -111,7 +111,7 @@ for key in six_map: if abs(six_price - five_price) * 100 / five_price > 10: actions.append({'city': info[0], 'region': info[1], 'commid': info[2], 'commname': info[3], 'five_price': five_price,'five_calc_type': five_map[key].split("|")[1], - 'six_price': six_price,'six_calc_type': six_map[key].split("|")[1], + 'six_price': six_price,'six_calc_type': six_map[key].split("|")[1], '差异值': abs(six_price - five_price) * 100 / five_price, '涨跌幅': (six_price - five_price) * 100 / five_price, 'confidence': six_map[key].split("|")[2]}) if info[0] not in city_seaweed_dict: city_seaweed_dict[info[0]] = 1 @@ -147,6 +147,6 @@ city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度') #excel_writer.save() excel_writer.close() -up_to_nextcloud() +#up_to_nextcloud() -send_msg() \ No newline at end of file +#send_msg() \ No newline at end of file From 4cc53dfbf5ba08c2d29a3cf0dcac6096151fdc5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 20 May 2024 13:16:59 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=8B=AC=E5=AE=B6?= =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/price/price_exception.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dags/jh/price/price_exception.py b/dags/jh/price/price_exception.py index 7fa0ff6..191e56b 100644 --- a/dags/jh/price/price_exception.py +++ b/dags/jh/price/price_exception.py @@ -1,13 +1,18 @@ """ 价格异常检测脚本 """ -from goojia_utils import dateutils, pgsql +from goojia_utils import dateutils, pgsql, MAIN_MONGO_CLIENT, data_utils import pandas as pd import logging import os import sys import requests +seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +# 获取独家小区 +exclusive_seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0, 'mark_map.exclusive': '是'}, {'city': 1, 'region': 1, 'name': 1, 'mark_map.exclusive': 1}).batch_size(100000))) +exclusive_seaweed_df = data_utils.flatten_json(exclusive_seaweed_df, 'mark_map') +exclusive_seaweed_df = exclusive_seaweed_df.rename(columns={'name': 'commname', 'exclusive': '独家小区'}) def simple_calc(result_df): """ @@ -132,6 +137,7 @@ excel_writer = pd.ExcelWriter(aim_file_name) if type == 'sample': result_df = simple_calc(result_df) +result_df = pd.merge(result_df, exclusive_seaweed_df, on=['city', 'region', 'commname'], how='left') result_df = result_df.drop_duplicates(subset=['commid'], keep='first') result_df = result_df.rename(columns={'city': '城市', 'region': '区域', 'commid': '小区ID', 'commname': '小区名', 'five_price': '上一月价格', 'five_calc_type': '上一月计算方式', 'six_price': '当月价格', 'six_calc_type': '当月计算方式', 'confidence': '置信度'}) @@ -147,6 +153,6 @@ city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度') #excel_writer.save() excel_writer.close() -#up_to_nextcloud() +up_to_nextcloud() -#send_msg() \ No newline at end of file +send_msg() \ No newline at end of file From 728dfa0e946af9f2b9a8579379fd2ab19dc94f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 29 May 2024 17:00:02 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E5=B1=8F=E8=94=BD=E8=BF=87=E9=AB=98?= =?UTF-8?q?=E7=89=A9=E4=B8=9A=E8=B4=B9=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/comm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index 6387264..09a1e5a 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -155,6 +155,7 @@ def sync(city=None): # 写入csv文件,防止小区变动导致的后续过程更新报错, # seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False) # 写入中间表clickhouse,解决多worker节点问题 + seaweed_df.loc[seaweed_df['estate_charge'] > 1000, 'estate_charge'] = None tmp_ck.to_sql_plus(seaweed_df[['_id', 'city', 'region', 'name', 'lng', 'lat', 'alias', 'estate_type2', 'build_type', 'household_count', 'block', 'estate_charge', 'finish_building_date']], 'community_summary') From c360db45c917ba527b04d420f70da41403bb97e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 29 May 2024 17:58:41 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E5=B1=8F=E8=94=BD=E8=BF=87=E9=AB=98?= =?UTF-8?q?=E7=89=A9=E4=B8=9A=E8=B4=B9=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/comm.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index 09a1e5a..a008cf5 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -47,11 +47,13 @@ def build_doc(item): estate_charge = commutils.format_property_fee(item['estate_charge']) if pd.notna(estate_charge): - doc_dict['estate_charge'] = float(estate_charge) - # 表示该物业费已经从库中补充 - doc_dict['estate_charge_flag'] = 0 - # 先打10分,对于其他的流程在修改分数 - doc_dict['credit_detail'] = {'estate_charge_score': 10} + # 剔除过大物业费导致ES报错 + if float(estate_charge) <= 1000: + doc_dict['estate_charge'] = float(estate_charge) + # 表示该物业费已经从库中补充 + doc_dict['estate_charge_flag'] = 0 + # 先打10分,对于其他的流程在修改分数 + doc_dict['credit_detail'] = {'estate_charge_score': 10} finish_building_date = commutils.format_finish_building_date(item['finish_building_date']) if pd.notna(finish_building_date): @@ -155,7 +157,6 @@ def sync(city=None): # 写入csv文件,防止小区变动导致的后续过程更新报错, # seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False) # 写入中间表clickhouse,解决多worker节点问题 - seaweed_df.loc[seaweed_df['estate_charge'] > 1000, 'estate_charge'] = None tmp_ck.to_sql_plus(seaweed_df[['_id', 'city', 'region', 'name', 'lng', 'lat', 'alias', 'estate_type2', 'build_type', 'household_count', 'block', 'estate_charge', 'finish_building_date']], 'community_summary')