Merge remote-tracking branch 'origin/master'
This commit is contained in:
@@ -11,7 +11,7 @@ from threading import Lock
|
||||
|
||||
mutex = Lock()
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
|
||||
@@ -10,7 +10,7 @@ from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database="tmp")
|
||||
|
||||
index_name = "community_summary"
|
||||
|
||||
@@ -11,7 +11,7 @@ import sys
|
||||
|
||||
|
||||
def sync():
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
log = logging.getLogger(__name__)
|
||||
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
|
||||
@@ -15,7 +15,7 @@ from sklearn.preprocessing import QuantileTransformer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
index_name = "community_summary"
|
||||
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||
save_path_prefix = "/home/airflow/data/community"
|
||||
|
||||
@@ -9,7 +9,7 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
|
||||
@@ -10,7 +10,7 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
|
||||
@@ -10,7 +10,7 @@ from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database="tmp")
|
||||
|
||||
index_name = "community_summary"
|
||||
|
||||
@@ -10,7 +10,7 @@ from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database="tmp")
|
||||
|
||||
index_name = "community_summary"
|
||||
|
||||
@@ -11,7 +11,7 @@ from threading import Lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
index_name = "community_summary"
|
||||
file_path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||
mutex = Lock()
|
||||
|
||||
@@ -14,7 +14,7 @@ from sklearn.preprocessing import QuantileTransformer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
index_name = "community_summary"
|
||||
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
|
||||
@@ -7,7 +7,6 @@ import pandas as pd
|
||||
from elasticsearch import helpers
|
||||
import sys
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es89_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
|
||||
index_name = "community_summary"
|
||||
@@ -41,7 +40,7 @@ def deal(actions, x):
|
||||
if len(doc_info) > 0:
|
||||
actions.append(action_doc)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
helpers.bulk(es89_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
log.info(f"已更新{count}")
|
||||
actions.clear()
|
||||
@@ -53,13 +52,13 @@ def sync(city=None):
|
||||
sql = "select city, region from community_summary"
|
||||
if pd.notna(city):
|
||||
sql += " where city='" + city + "'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
community_summary_df = es89_client.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
actions = list()
|
||||
community_summary_df.apply(lambda x: deal(actions, x), axis=1)
|
||||
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
helpers.bulk(es89_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
|
||||
log.info(f"完成转换省市区全拼至206 {index_name}")
|
||||
|
||||
@@ -10,7 +10,7 @@ from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
|
||||
@@ -11,7 +11,7 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
|
||||
@@ -7,7 +7,7 @@ import pandas as pd
|
||||
from elasticsearch import helpers
|
||||
import sys
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
|
||||
@@ -9,7 +9,7 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
index_name = "community_summary"
|
||||
|
||||
|
||||
@@ -17,39 +17,34 @@ def calc_percentile(city=None):
|
||||
"""
|
||||
计算消费等级
|
||||
"""
|
||||
sql = "select id, city, region, name, avg_price, total_price from community_summary"
|
||||
sql = "select id, city, region, name, avg_price, total_price, avg_cost, rent_price, relative_price_ranking, finish_building_date, estate_charge, estate_charge_fix, household_count from community_summary"
|
||||
if pd.notna(city):
|
||||
sql += " where city='" + city + "'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
consumption_aim_df = community_summary_df[['id', 'city', 'avg_price', 'total_price']]
|
||||
|
||||
community_summary_df['estate_charge_fix'] = community_summary_df['estate_charge'].combine_first(community_summary_df['estate_charge_fix'])
|
||||
community_summary_df['relative_price_ranking'] = community_summary_df['relative_price_ranking'] * 100
|
||||
consumption_aim_df = community_summary_df[['id', 'city', 'avg_price', 'total_price', 'avg_cost', 'rent_price', 'relative_price_ranking', 'finish_building_date',
|
||||
'estate_charge_fix', 'household_count']]
|
||||
number = []
|
||||
for i in range(1, 101):
|
||||
number.append(i / 100)
|
||||
|
||||
percents_actions = list()
|
||||
# 计算百分位的字段
|
||||
columns = ['avg_price', 'total_price', 'avg_cost', 'rent_price', 'relative_price_ranking', 'finish_building_date',
|
||||
'estate_charge_fix', 'household_count']
|
||||
for city_now, item in consumption_aim_df.groupby(by=['city']).describe(percentiles=number).iterrows():
|
||||
for i in range(1, 101):
|
||||
if str(i) + '%' in item['total_price']:
|
||||
if pd.notna((item['total_price'])[str(i) + '%']):
|
||||
action = {'city': city_now, 'type': 'total_price',
|
||||
'price': int((item['total_price'])[str(i) + '%']), 'percent': i}
|
||||
else:
|
||||
if pd.notna((item['total_price'])[str(i) + '.0%']):
|
||||
action = {'city': city_now, 'type': 'total_price',
|
||||
'price': int((item['total_price'])[str(i) + '.0%']), 'percent': i}
|
||||
percents_actions.append(action)
|
||||
for i in range(1, 101):
|
||||
if str(i) + '%' in item['avg_price']:
|
||||
if pd.notna((item['avg_price'])[str(i) + '%']):
|
||||
action = {'city': city_now, 'type': 'avg_price', 'price': (item['avg_price'])[str(i) + '%'],
|
||||
'percent': i}
|
||||
else:
|
||||
if pd.notna((item['avg_price'])[str(i) + '.0%']):
|
||||
action = {'city': city_now, 'type': 'avg_price', 'price': (item['avg_price'])[str(i) + '.0%'],
|
||||
'percent': i}
|
||||
percents_actions.append(action)
|
||||
for column in columns:
|
||||
for i in range(1, 101):
|
||||
if str(i) + '%' in item[column]:
|
||||
if pd.notna((item[column])[str(i) + '%']):
|
||||
action = {'city': city_now, 'type': column,
|
||||
'price': int((item[column])[str(i) + '%']), 'percent': i}
|
||||
else:
|
||||
if pd.notna((item[column])[str(i) + '.0%']):
|
||||
action = {'city': city_now, 'type': column,
|
||||
'price': int((item[column])[str(i) + '.0%']), 'percent': i}
|
||||
percents_actions.append(action)
|
||||
|
||||
percents_df = pd.DataFrame(percents_actions)
|
||||
|
||||
@@ -64,13 +59,11 @@ def calc_percentile(city=None):
|
||||
try:
|
||||
for i in range(1, 101):
|
||||
if i == 1:
|
||||
df.loc[(df['avg_price'] <= percents_now_df[(percents_now_df['type'] == 'avg_price') & (percents_now_df['percent'] == i)].iloc[0]['price']), 'avg_price_percentile'] = 101 - i
|
||||
|
||||
df.loc[(df['total_price'] <= percents_now_df[(percents_now_df['type'] == 'total_price') & (percents_now_df['percent'] == i)].iloc[0]['price']), 'total_price_percentile'] = 101 - i
|
||||
for column in columns:
|
||||
df.loc[(df[column] <= percents_now_df[(percents_now_df['type'] == column) & (percents_now_df['percent'] == i)].iloc[0]['price']), column + '_percentile'] = 101 - i
|
||||
else:
|
||||
df.loc[(df['avg_price'] > percents_now_df[(percents_now_df['type'] == 'avg_price') & (percents_now_df['percent'] == i - 1)].iloc[0]['price']) & (df['avg_price'] <= percents_now_df[(percents_now_df['type'] == 'avg_price') & (percents_now_df['percent'] == i)].iloc[0]['price']), 'avg_price_percentile'] = 101 - i
|
||||
|
||||
df.loc[(df['total_price'] > percents_now_df[(percents_now_df['type'] == 'total_price') & (percents_now_df['percent'] == i - 1)].iloc[0]['price']) & (df['total_price'] <= percents_now_df[(percents_now_df['type'] == 'total_price') & (percents_now_df['percent'] == i)].iloc[0]['price']), 'total_price_percentile'] = 101 - i
|
||||
for column in columns:
|
||||
df.loc[(df[column] > percents_now_df[(percents_now_df['type'] == column) & (percents_now_df['percent'] == i - 1)].iloc[0]['price']) & (df[column] <= percents_now_df[(percents_now_df['type'] == column) & (percents_now_df['percent'] == i)].iloc[0]['price']), column + '_percentile'] = 101 - i
|
||||
except Exception as e:
|
||||
log.error("解析错误{}", e)
|
||||
continue
|
||||
@@ -78,10 +71,13 @@ def calc_percentile(city=None):
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["id"])
|
||||
if pd.notna(item['total_price_percentile']):
|
||||
row['total_price_percentile'] = int(item['total_price_percentile'])
|
||||
if pd.notna(item['avg_price_percentile']):
|
||||
row['avg_price_percentile'] = int(item['avg_price_percentile'])
|
||||
for column in columns:
|
||||
if pd.notna(item[column + '_percentile']):
|
||||
if 'estate_charge_fix' == column:
|
||||
db_column = "estate_charge_percentile"
|
||||
else:
|
||||
db_column = column + '_percentile'
|
||||
row[db_column] = int(item[column + '_percentile'])
|
||||
|
||||
action = {'_op_type': 'update',
|
||||
'_index': index_name,
|
||||
|
||||
@@ -11,7 +11,7 @@ from sklearn.mixture import GaussianMixture
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
index_name = "community_summary"
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from elasticsearch import helpers
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
|
||||
@@ -14,7 +14,7 @@ from sklearn.mixture import GaussianMixture
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
index_name = "community_summary"
|
||||
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ import sys
|
||||
from threading import Lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
|
||||
@@ -9,7 +9,7 @@ import sys
|
||||
from threading import Lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
|
||||
@@ -11,7 +11,7 @@ from threading import Lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
|
||||
@@ -9,7 +9,7 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
|
||||
@@ -10,7 +10,7 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||
|
||||
@@ -8,7 +8,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseCl
|
||||
from elasticsearch import helpers
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
|
||||
@@ -10,7 +10,7 @@ from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
|
||||
Reference in New Issue
Block a user