财力评级206改为89

This commit is contained in:
沈健
2024-08-26 10:10:11 +08:00
parent ac8efee40b
commit aa0f3719a5
27 changed files with 29 additions and 30 deletions
+1 -1
View File
@@ -11,7 +11,7 @@ from threading import Lock
mutex = Lock()
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
@@ -10,7 +10,7 @@ from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database="tmp")
index_name = "community_summary"
@@ -11,7 +11,7 @@ import sys
def sync():
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
log = logging.getLogger(__name__)
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
+1 -1
View File
@@ -10,7 +10,7 @@ import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
@@ -15,7 +15,7 @@ from sklearn.preprocessing import QuantileTransformer
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
index_name = "community_summary"
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
save_path_prefix = "/home/airflow/data/community"
+1 -1
View File
@@ -9,7 +9,7 @@ import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
index_name = "community_summary"
path = "/home/airflow/data/community"
+1 -1
View File
@@ -10,7 +10,7 @@ import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
@@ -10,7 +10,7 @@ from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database="tmp")
index_name = "community_summary"
@@ -10,7 +10,7 @@ from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database="tmp")
index_name = "community_summary"
@@ -11,7 +11,7 @@ from threading import Lock
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
index_name = "community_summary"
file_path = "/opt/airflow/dags/repo/dags/business/business/community/data"
mutex = Lock()
@@ -14,7 +14,7 @@ from sklearn.preprocessing import QuantileTransformer
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
index_name = "community_summary"
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
+1 -1
View File
@@ -17,7 +17,7 @@ from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -7,7 +7,6 @@ import pandas as pd
from elasticsearch import helpers
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es89_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
index_name = "community_summary"
@@ -41,7 +40,7 @@ def deal(actions, x):
if len(doc_info) > 0:
actions.append(action_doc)
if len(actions) == 5000:
helpers.bulk(es206_client.elastic_client, actions)
helpers.bulk(es89_client.elastic_client, actions)
count += len(actions)
log.info(f"已更新{count}")
actions.clear()
@@ -53,13 +52,13 @@ def sync(city=None):
sql = "select city, region from community_summary"
if pd.notna(city):
sql += " where city='" + city + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
community_summary_df = es89_client.scan_by_sql_to_data_frame(sql)
actions = list()
community_summary_df.apply(lambda x: deal(actions, x), axis=1)
if len(actions) > 0:
helpers.bulk(es206_client.elastic_client, actions)
helpers.bulk(es89_client.elastic_client, actions)
actions.clear()
log.info(f"完成转换省市区全拼至206 {index_name}")
+1 -1
View File
@@ -10,7 +10,7 @@ from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
+1 -1
View File
@@ -11,7 +11,7 @@ import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
+1 -1
View File
@@ -7,7 +7,7 @@ import pandas as pd
from elasticsearch import helpers
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -17,7 +17,7 @@ def calc_percentile(city=None):
"""
计算消费等级
"""
sql = "select id, city, region, name, avg_price, total_price from community_summary"
sql = "select id, city, region, name, avg_price, total_price, avg_cost, rent_price, relative_price_ranking, finish_building_date, estate_charge_fix, household_count from community_summary"
if pd.notna(city):
sql += " where city='" + city + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
@@ -11,7 +11,7 @@ from sklearn.mixture import GaussianMixture
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
index_name = "community_summary"
+1 -1
View File
@@ -8,7 +8,7 @@ from elasticsearch import helpers
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
@@ -14,7 +14,7 @@ from sklearn.mixture import GaussianMixture
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
index_name = "community_summary"
@@ -11,7 +11,7 @@ import sys
from threading import Lock
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
@@ -9,7 +9,7 @@ import sys
from threading import Lock
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
index_name = "community_summary"
path = "/home/airflow/data/community"
+1 -1
View File
@@ -11,7 +11,7 @@ from threading import Lock
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
@@ -9,7 +9,7 @@ import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -10,7 +10,7 @@ from concurrent.futures import ThreadPoolExecutor
from threading import Lock
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
index_name = "community_summary"
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
+1 -1
View File
@@ -8,7 +8,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseCl
from elasticsearch import helpers
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
+1 -1
View File
@@ -10,7 +10,7 @@ from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"