新增省市区全拼转换

This commit is contained in:
sjshenjian
2024-07-09 16:07:55 +08:00
parent 8c5b16e519
commit 16999f22b2
2 changed files with 75 additions and 1 deletions
@@ -0,0 +1,68 @@
"""
省市区简拼转全拼
"""
from goojia_utils import ElasticSearchClient, dateutils
import logging
import pandas as pd
from elasticsearch import helpers
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es89_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
count = 0
short_df = es89_client.scan_by_sql_to_data_frame("select city, city_short, district, district_short from administrative_mapping where type='区域' ")
short_map = {}
def deal_short_map(x):
short_map[x['city_short'] + x['district_short']] = x['city'] + '|' + x['district']
short_df.apply(lambda x: deal_short_map(x), axis=1)
def deal(actions, x):
global count
doc_info = dict()
if pd.notna(x['city']) and pd.notna(x['region']):
if x['city'] + x['region'] in short_map:
short_info = short_map[x['city'] + x['region']]
x['city'] = short_info.split('|')[0]
x['region'] = short_info.split('|')[1]
action_doc = {
"_op_type": "update",
"_index": index_name,
"_id": x['_id'],
"doc": doc_info
}
if len(doc_info) > 0:
actions.append(action_doc)
if len(actions) == 5000:
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
log.info(f"已更新{count}")
actions.clear()
def sync(city=None):
log.info(f"开始转换省市区全拼至206 {index_name}")
sql = "select city, region from community_summary"
if pd.notna(city):
sql += " where city='" + city + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
actions = list()
community_summary_df.apply(lambda x: deal(actions, x), axis=1)
if len(actions) > 0:
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
log.info(f"完成转换省市区全拼至206 {index_name}")
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+7 -1
View File
@@ -110,5 +110,11 @@ with DAG(
task_id='other',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
# 统一输出为省市区全拼
full_name = BashOperator(
task_id='full_name',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/full_name.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
# 人均消费并行,值为空
comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> city_level_and_price_rating >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other
comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> city_level_and_price_rating >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other \
>> full_name