diff --git a/dags/business/business/community/avg_cost.py b/dags/business/business/community/avg_cost.py index f7381e2..73fa72e 100644 --- a/dags/business/business/community/avg_cost.py +++ b/dags/business/business/community/avg_cost.py @@ -11,7 +11,7 @@ from threading import Lock mutex = Lock() log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/business_district.py b/dags/business/business/community/business_district.py index 3c26117..8ccf599 100644 --- a/dags/business/business/community/business_district.py +++ b/dags/business/business/community/business_district.py @@ -10,7 +10,7 @@ from threading import Lock import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database="tmp") index_name = "community_summary" diff --git a/dags/business/business/community/city_level_and_price_rating.py b/dags/business/business/community/city_level_and_price_rating.py index a5c0175..1ac246f 100644 --- a/dags/business/business/community/city_level_and_price_rating.py +++ b/dags/business/business/community/city_level_and_price_rating.py @@ -11,7 +11,7 @@ import sys def sync(): - es206_client = ElasticSearchClient(host='192.168.0.206') + es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') log = logging.getLogger(__name__) path = "/opt/airflow/dags/repo/dags/business/business/community/data" diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index a008cf5..892248f 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -10,7 +10,7 @@ import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/consumption.py b/dags/business/business/community/consumption.py index 9a6d5c8..284a905 100644 --- a/dags/business/business/community/consumption.py +++ b/dags/business/business/community/consumption.py @@ -15,7 +15,7 @@ from sklearn.preprocessing import QuantileTransformer log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/opt/airflow/dags/repo/dags/business/business/community/data" save_path_prefix = "/home/airflow/data/community" diff --git a/dags/business/business/community/credit.py b/dags/business/business/community/credit.py index 2fe336e..649427a 100644 --- a/dags/business/business/community/credit.py +++ b/dags/business/business/community/credit.py @@ -9,7 +9,7 @@ import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/home/airflow/data/community" diff --git a/dags/business/business/community/del_comm.py b/dags/business/business/community/del_comm.py index 658cffc..b6d46e4 100644 --- a/dags/business/business/community/del_comm.py +++ b/dags/business/business/community/del_comm.py @@ -10,7 +10,7 @@ import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/education.py b/dags/business/business/community/education.py index 916c9c1..b9135b4 100644 --- a/dags/business/business/community/education.py +++ b/dags/business/business/community/education.py @@ -10,7 +10,7 @@ from threading import Lock import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database="tmp") index_name = "community_summary" diff --git a/dags/business/business/community/education2.py b/dags/business/business/community/education2.py index 7d83350..b49168b 100644 --- a/dags/business/business/community/education2.py +++ b/dags/business/business/community/education2.py @@ -10,7 +10,7 @@ from threading import Lock import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database="tmp") index_name = "community_summary" diff --git a/dags/business/business/community/estate_charge_complete.py b/dags/business/business/community/estate_charge_complete.py index 9f2ad7c..2033e5f 100644 --- a/dags/business/business/community/estate_charge_complete.py +++ b/dags/business/business/community/estate_charge_complete.py @@ -11,7 +11,7 @@ from threading import Lock log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" file_path = "/opt/airflow/dags/repo/dags/business/business/community/data" mutex = Lock() diff --git a/dags/business/business/community/estate_charge_ranking.py b/dags/business/business/community/estate_charge_ranking.py index a5319af..5e09847 100644 --- a/dags/business/business/community/estate_charge_ranking.py +++ b/dags/business/business/community/estate_charge_ranking.py @@ -14,7 +14,7 @@ from sklearn.preprocessing import QuantileTransformer log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/opt/airflow/dags/repo/dags/business/business/community/data" diff --git a/dags/business/business/community/extra.py b/dags/business/business/community/extra.py index 7561eac..baa6bf1 100644 --- a/dags/business/business/community/extra.py +++ b/dags/business/business/community/extra.py @@ -17,7 +17,7 @@ from threading import Lock import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/home/airflow/data/community" diff --git a/dags/business/business/community/full_name.py b/dags/business/business/community/full_name.py index 867262d..f65988e 100644 --- a/dags/business/business/community/full_name.py +++ b/dags/business/business/community/full_name.py @@ -7,7 +7,6 @@ import pandas as pd from elasticsearch import helpers import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') es89_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" @@ -41,7 +40,7 @@ def deal(actions, x): if len(doc_info) > 0: actions.append(action_doc) if len(actions) == 5000: - helpers.bulk(es206_client.elastic_client, actions) + helpers.bulk(es89_client.elastic_client, actions) count += len(actions) log.info(f"已更新{count}") actions.clear() @@ -53,13 +52,13 @@ def sync(city=None): sql = "select city, region from community_summary" if pd.notna(city): sql += " where city='" + city + "'" - community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + community_summary_df = es89_client.scan_by_sql_to_data_frame(sql) actions = list() community_summary_df.apply(lambda x: deal(actions, x), axis=1) if len(actions) > 0: - helpers.bulk(es206_client.elastic_client, actions) + helpers.bulk(es89_client.elastic_client, actions) actions.clear() log.info(f"完成转换省市区全拼至206 {index_name}") diff --git a/dags/business/business/community/life.py b/dags/business/business/community/life.py index 7d0ba49..55ee981 100644 --- a/dags/business/business/community/life.py +++ b/dags/business/business/community/life.py @@ -10,7 +10,7 @@ from threading import Lock import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/medical.py b/dags/business/business/community/medical.py index 0de3227..ab9b194 100644 --- a/dags/business/business/community/medical.py +++ b/dags/business/business/community/medical.py @@ -11,7 +11,7 @@ import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/other.py b/dags/business/business/community/other.py index 6a67cff..3e42cb2 100644 --- a/dags/business/business/community/other.py +++ b/dags/business/business/community/other.py @@ -7,7 +7,7 @@ import pandas as pd from elasticsearch import helpers import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/home/airflow/data/community" diff --git a/dags/business/business/community/percentile.py b/dags/business/business/community/percentile.py index e8f5ac4..dd8480f 100644 --- a/dags/business/business/community/percentile.py +++ b/dags/business/business/community/percentile.py @@ -17,7 +17,7 @@ def calc_percentile(city=None): """ 计算消费等级 """ - sql = "select id, city, region, name, avg_price, total_price from community_summary" + sql = "select id, city, region, name, avg_price, total_price, avg_cost, rent_price, relative_price_ranking, finish_building_date, estate_charge_fix, household_count from community_summary" if pd.notna(city): sql += " where city='" + city + "'" community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) diff --git a/dags/business/business/community/person_rating.py b/dags/business/business/community/person_rating.py index 2868bc7..a74ed8a 100644 --- a/dags/business/business/community/person_rating.py +++ b/dags/business/business/community/person_rating.py @@ -11,7 +11,7 @@ from sklearn.mixture import GaussianMixture log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index 35d1238..7d1b1cc 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -8,7 +8,7 @@ from elasticsearch import helpers import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/price_rating.py b/dags/business/business/community/price_rating.py index 47536d5..01740f8 100644 --- a/dags/business/business/community/price_rating.py +++ b/dags/business/business/community/price_rating.py @@ -14,7 +14,7 @@ from sklearn.mixture import GaussianMixture log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" diff --git a/dags/business/business/community/relative_price_ranking.py b/dags/business/business/community/relative_price_ranking.py index 5dc1b3c..9a79b75 100644 --- a/dags/business/business/community/relative_price_ranking.py +++ b/dags/business/business/community/relative_price_ranking.py @@ -11,7 +11,7 @@ import sys from threading import Lock log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/relative_price_ranking_fix.py b/dags/business/business/community/relative_price_ranking_fix.py index 5e06a68..d66b857 100644 --- a/dags/business/business/community/relative_price_ranking_fix.py +++ b/dags/business/business/community/relative_price_ranking_fix.py @@ -9,7 +9,7 @@ import sys from threading import Lock log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/home/airflow/data/community" diff --git a/dags/business/business/community/sell.py b/dags/business/business/community/sell.py index 6de2b7c..bc93af3 100644 --- a/dags/business/business/community/sell.py +++ b/dags/business/business/community/sell.py @@ -11,7 +11,7 @@ from threading import Lock log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/total_price.py b/dags/business/business/community/total_price.py index 254fec3..951c1b2 100644 --- a/dags/business/business/community/total_price.py +++ b/dags/business/business/community/total_price.py @@ -9,7 +9,7 @@ import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/home/airflow/data/community" diff --git a/dags/business/business/community/total_price_complete.py b/dags/business/business/community/total_price_complete.py index 0589ad5..aaf6696 100644 --- a/dags/business/business/community/total_price_complete.py +++ b/dags/business/business/community/total_price_complete.py @@ -10,7 +10,7 @@ from concurrent.futures import ThreadPoolExecutor from threading import Lock log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/opt/airflow/dags/repo/dags/business/business/community/data" diff --git a/dags/business/business/community/trade.py b/dags/business/business/community/trade.py index 8036ec8..4d88e35 100644 --- a/dags/business/business/community/trade.py +++ b/dags/business/business/community/trade.py @@ -8,7 +8,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseCl from elasticsearch import helpers log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index 7158f68..f38024d 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -10,7 +10,7 @@ from threading import Lock import sys log = logging.getLogger(__name__) -es206_client = ElasticSearchClient(host='192.168.0.206') +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary"