diff --git a/dags/business/business/community/avg_cost.py b/dags/business/business/community/avg_cost.py index f7381e2..73fa72e 100644 --- a/dags/business/business/community/avg_cost.py +++ b/dags/business/business/community/avg_cost.py @@ -11,7 +11,7 @@ from threading import Lock mutex = Lock() log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/business_district.py b/dags/business/business/community/business_district.py index 3c26117..8ccf599 100644 --- a/dags/business/business/community/business_district.py +++ b/dags/business/business/community/business_district.py @@ -10,7 +10,7 @@ from threading import Lock import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database="tmp") index_name = "community_summary" diff --git a/dags/business/business/community/city_level_and_price_rating.py b/dags/business/business/community/city_level_and_price_rating.py index a5c0175..1ac246f 100644 --- a/dags/business/business/community/city_level_and_price_rating.py +++ b/dags/business/business/community/city_level_and_price_rating.py @@ -11,7 +11,7 @@ import sys def sync(): - es206_client = ElasticSearchClient(host='192.168.0.206') + es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') log = logging.getLogger(__name__) path = "/opt/airflow/dags/repo/dags/business/business/community/data" diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index a008cf5..892248f 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -10,7 +10,7 @@ import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/consumption.py b/dags/business/business/community/consumption.py index 9a6d5c8..284a905 100644 --- a/dags/business/business/community/consumption.py +++ b/dags/business/business/community/consumption.py @@ -15,7 +15,7 @@ from sklearn.preprocessing import QuantileTransformer log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/opt/airflow/dags/repo/dags/business/business/community/data" save_path_prefix = "/home/airflow/data/community" diff --git a/dags/business/business/community/credit.py b/dags/business/business/community/credit.py index 2fe336e..649427a 100644 --- a/dags/business/business/community/credit.py +++ b/dags/business/business/community/credit.py @@ -9,7 +9,7 @@ import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/home/airflow/data/community" diff --git a/dags/business/business/community/del_comm.py b/dags/business/business/community/del_comm.py index 658cffc..b6d46e4 100644 --- a/dags/business/business/community/del_comm.py +++ b/dags/business/business/community/del_comm.py @@ -10,7 +10,7 @@ import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/education.py b/dags/business/business/community/education.py index 916c9c1..b9135b4 100644 --- a/dags/business/business/community/education.py +++ b/dags/business/business/community/education.py @@ -10,7 +10,7 @@ from threading import Lock import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database="tmp") index_name = "community_summary" diff --git a/dags/business/business/community/education2.py b/dags/business/business/community/education2.py index 7d83350..b49168b 100644 --- a/dags/business/business/community/education2.py +++ b/dags/business/business/community/education2.py @@ -10,7 +10,7 @@ from threading import Lock import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database="tmp") index_name = "community_summary" diff --git a/dags/business/business/community/estate_charge_complete.py b/dags/business/business/community/estate_charge_complete.py index 9f2ad7c..2033e5f 100644 --- a/dags/business/business/community/estate_charge_complete.py +++ b/dags/business/business/community/estate_charge_complete.py @@ -11,7 +11,7 @@ from threading import Lock log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" file_path = "/opt/airflow/dags/repo/dags/business/business/community/data" mutex = Lock() diff --git a/dags/business/business/community/estate_charge_ranking.py b/dags/business/business/community/estate_charge_ranking.py index a5319af..5e09847 100644 --- a/dags/business/business/community/estate_charge_ranking.py +++ b/dags/business/business/community/estate_charge_ranking.py @@ -14,7 +14,7 @@ from sklearn.preprocessing import QuantileTransformer log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/opt/airflow/dags/repo/dags/business/business/community/data" diff --git a/dags/business/business/community/extra.py b/dags/business/business/community/extra.py index 7561eac..baa6bf1 100644 --- a/dags/business/business/community/extra.py +++ b/dags/business/business/community/extra.py @@ -17,7 +17,7 @@ from threading import Lock import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/home/airflow/data/community" diff --git a/dags/business/business/community/full_name.py b/dags/business/business/community/full_name.py index 867262d..f65988e 100644 --- a/dags/business/business/community/full_name.py +++ b/dags/business/business/community/full_name.py @@ -7,7 +7,6 @@ import pandas as pd from elasticsearch import helpers import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') es89_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" @@ -41,7 +40,7 @@ def deal(actions, x): if len(doc_info) > 0: actions.append(action_doc) if len(actions) == 5000: - helpers.bulk(es206_client.elastic_client, actions) + helpers.bulk(es89_client.elastic_client, actions) count += len(actions) log.info(f"已更新{count}") actions.clear() @@ -53,13 +52,13 @@ def sync(city=None): sql = "select city, region from community_summary" if pd.notna(city): sql += " where city='" + city + "'" - community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + community_summary_df = es89_client.scan_by_sql_to_data_frame(sql) actions = list() community_summary_df.apply(lambda x: deal(actions, x), axis=1) if len(actions) > 0: - helpers.bulk(es206_client.elastic_client, actions) + helpers.bulk(es89_client.elastic_client, actions) actions.clear() log.info(f"完成转换省市区全拼至206 {index_name}") diff --git a/dags/business/business/community/life.py b/dags/business/business/community/life.py index 7d0ba49..55ee981 100644 --- a/dags/business/business/community/life.py +++ b/dags/business/business/community/life.py @@ -10,7 +10,7 @@ from threading import Lock import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/medical.py b/dags/business/business/community/medical.py index 0de3227..ab9b194 100644 --- a/dags/business/business/community/medical.py +++ b/dags/business/business/community/medical.py @@ -11,7 +11,7 @@ import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/other.py b/dags/business/business/community/other.py index 6a67cff..3e42cb2 100644 --- a/dags/business/business/community/other.py +++ b/dags/business/business/community/other.py @@ -7,7 +7,7 @@ import pandas as pd from elasticsearch import helpers import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/home/airflow/data/community" diff --git a/dags/business/business/community/percentile.py b/dags/business/business/community/percentile.py index 9f466e8..6e120c9 100644 --- a/dags/business/business/community/percentile.py +++ b/dags/business/business/community/percentile.py @@ -9,7 +9,7 @@ import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" @@ -17,39 +17,34 @@ def calc_percentile(city=None): """ 计算消费等级 """ - sql = "select id, city, region, name, avg_price, total_price from community_summary" + sql = "select id, city, region, name, avg_price, total_price, avg_cost, rent_price, relative_price_ranking, finish_building_date, estate_charge, estate_charge_fix, household_count from community_summary" if pd.notna(city): sql += " where city='" + city + "'" community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) - - consumption_aim_df = community_summary_df[['id', 'city', 'avg_price', 'total_price']] - + community_summary_df['estate_charge_fix'] = community_summary_df['estate_charge'].combine_first(community_summary_df['estate_charge_fix']) + community_summary_df['relative_price_ranking'] = community_summary_df['relative_price_ranking'] * 100 + consumption_aim_df = community_summary_df[['id', 'city', 'avg_price', 'total_price', 'avg_cost', 'rent_price', 'relative_price_ranking', 'finish_building_date', + 'estate_charge_fix', 'household_count']] number = [] for i in range(1, 101): number.append(i / 100) percents_actions = list() + # 计算百分位的字段 + columns = ['avg_price', 'total_price', 'avg_cost', 'rent_price', 'relative_price_ranking', 'finish_building_date', + 'estate_charge_fix', 'household_count'] for city_now, item in consumption_aim_df.groupby(by=['city']).describe(percentiles=number).iterrows(): - for i in range(1, 101): - if str(i) + '%' in item['total_price']: - if pd.notna((item['total_price'])[str(i) + '%']): - action = {'city': city_now, 'type': 'total_price', - 'price': int((item['total_price'])[str(i) + '%']), 'percent': i} - else: - if pd.notna((item['total_price'])[str(i) + '.0%']): - action = {'city': city_now, 'type': 'total_price', - 'price': int((item['total_price'])[str(i) + '.0%']), 'percent': i} - percents_actions.append(action) - for i in range(1, 101): - if str(i) + '%' in item['avg_price']: - if pd.notna((item['avg_price'])[str(i) + '%']): - action = {'city': city_now, 'type': 'avg_price', 'price': (item['avg_price'])[str(i) + '%'], - 'percent': i} - else: - if pd.notna((item['avg_price'])[str(i) + '.0%']): - action = {'city': city_now, 'type': 'avg_price', 'price': (item['avg_price'])[str(i) + '.0%'], - 'percent': i} - percents_actions.append(action) + for column in columns: + for i in range(1, 101): + if str(i) + '%' in item[column]: + if pd.notna((item[column])[str(i) + '%']): + action = {'city': city_now, 'type': column, + 'price': int((item[column])[str(i) + '%']), 'percent': i} + else: + if pd.notna((item[column])[str(i) + '.0%']): + action = {'city': city_now, 'type': column, + 'price': int((item[column])[str(i) + '.0%']), 'percent': i} + percents_actions.append(action) percents_df = pd.DataFrame(percents_actions) @@ -64,13 +59,11 @@ def calc_percentile(city=None): try: for i in range(1, 101): if i == 1: - df.loc[(df['avg_price'] <= percents_now_df[(percents_now_df['type'] == 'avg_price') & (percents_now_df['percent'] == i)].iloc[0]['price']), 'avg_price_percentile'] = 101 - i - - df.loc[(df['total_price'] <= percents_now_df[(percents_now_df['type'] == 'total_price') & (percents_now_df['percent'] == i)].iloc[0]['price']), 'total_price_percentile'] = 101 - i + for column in columns: + df.loc[(df[column] <= percents_now_df[(percents_now_df['type'] == column) & (percents_now_df['percent'] == i)].iloc[0]['price']), column + '_percentile'] = 101 - i else: - df.loc[(df['avg_price'] > percents_now_df[(percents_now_df['type'] == 'avg_price') & (percents_now_df['percent'] == i - 1)].iloc[0]['price']) & (df['avg_price'] <= percents_now_df[(percents_now_df['type'] == 'avg_price') & (percents_now_df['percent'] == i)].iloc[0]['price']), 'avg_price_percentile'] = 101 - i - - df.loc[(df['total_price'] > percents_now_df[(percents_now_df['type'] == 'total_price') & (percents_now_df['percent'] == i - 1)].iloc[0]['price']) & (df['total_price'] <= percents_now_df[(percents_now_df['type'] == 'total_price') & (percents_now_df['percent'] == i)].iloc[0]['price']), 'total_price_percentile'] = 101 - i + for column in columns: + df.loc[(df[column] > percents_now_df[(percents_now_df['type'] == column) & (percents_now_df['percent'] == i - 1)].iloc[0]['price']) & (df[column] <= percents_now_df[(percents_now_df['type'] == column) & (percents_now_df['percent'] == i)].iloc[0]['price']), column + '_percentile'] = 101 - i except Exception as e: log.error("解析错误{}", e) continue @@ -78,10 +71,13 @@ def calc_percentile(city=None): for index, item in df.iterrows(): row = dict() _id = str(item["id"]) - if pd.notna(item['total_price_percentile']): - row['total_price_percentile'] = int(item['total_price_percentile']) - if pd.notna(item['avg_price_percentile']): - row['avg_price_percentile'] = int(item['avg_price_percentile']) + for column in columns: + if pd.notna(item[column + '_percentile']): + if 'estate_charge_fix' == column: + db_column = "estate_charge_percentile" + else: + db_column = column + '_percentile' + row[db_column] = int(item[column + '_percentile']) action = {'_op_type': 'update', '_index': index_name, diff --git a/dags/business/business/community/person_rating.py b/dags/business/business/community/person_rating.py index 2868bc7..a74ed8a 100644 --- a/dags/business/business/community/person_rating.py +++ b/dags/business/business/community/person_rating.py @@ -11,7 +11,7 @@ from sklearn.mixture import GaussianMixture log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index 35d1238..7d1b1cc 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -8,7 +8,7 @@ from elasticsearch import helpers import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/price_rating.py b/dags/business/business/community/price_rating.py index 47536d5..01740f8 100644 --- a/dags/business/business/community/price_rating.py +++ b/dags/business/business/community/price_rating.py @@ -14,7 +14,7 @@ from sklearn.mixture import GaussianMixture log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" diff --git a/dags/business/business/community/relative_price_ranking.py b/dags/business/business/community/relative_price_ranking.py index 5dc1b3c..9a79b75 100644 --- a/dags/business/business/community/relative_price_ranking.py +++ b/dags/business/business/community/relative_price_ranking.py @@ -11,7 +11,7 @@ import sys from threading import Lock log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/relative_price_ranking_fix.py b/dags/business/business/community/relative_price_ranking_fix.py index 5e06a68..d66b857 100644 --- a/dags/business/business/community/relative_price_ranking_fix.py +++ b/dags/business/business/community/relative_price_ranking_fix.py @@ -9,7 +9,7 @@ import sys from threading import Lock log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/home/airflow/data/community" diff --git a/dags/business/business/community/sell.py b/dags/business/business/community/sell.py index 6de2b7c..bc93af3 100644 --- a/dags/business/business/community/sell.py +++ b/dags/business/business/community/sell.py @@ -11,7 +11,7 @@ from threading import Lock log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/total_price.py b/dags/business/business/community/total_price.py index 254fec3..951c1b2 100644 --- a/dags/business/business/community/total_price.py +++ b/dags/business/business/community/total_price.py @@ -9,7 +9,7 @@ import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/home/airflow/data/community" diff --git a/dags/business/business/community/total_price_complete.py b/dags/business/business/community/total_price_complete.py index 0589ad5..aaf6696 100644 --- a/dags/business/business/community/total_price_complete.py +++ b/dags/business/business/community/total_price_complete.py @@ -10,7 +10,7 @@ from concurrent.futures import ThreadPoolExecutor from threading import Lock log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/opt/airflow/dags/repo/dags/business/business/community/data" diff --git a/dags/business/business/community/trade.py b/dags/business/business/community/trade.py index 8036ec8..4d88e35 100644 --- a/dags/business/business/community/trade.py +++ b/dags/business/business/community/trade.py @@ -8,7 +8,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseCl from elasticsearch import helpers log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index 7158f68..f38024d 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -10,7 +10,7 @@ from threading import Lock import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary"