新增省市区全拼转换
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
"""
|
||||
省市区简拼转全拼
|
||||
"""
|
||||
from goojia_utils import ElasticSearchClient, dateutils
|
||||
import logging
|
||||
import pandas as pd
|
||||
from elasticsearch import helpers
|
||||
import sys
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es89_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
m_date = dateutils.iso_format()
|
||||
|
||||
count = 0
|
||||
|
||||
short_df = es89_client.scan_by_sql_to_data_frame("select city, city_short, district, district_short from administrative_mapping where type='区域' ")
|
||||
short_map = {}
|
||||
def deal_short_map(x):
|
||||
short_map[x['city_short'] + x['district_short']] = x['city'] + '|' + x['district']
|
||||
short_df.apply(lambda x: deal_short_map(x), axis=1)
|
||||
|
||||
def deal(actions, x):
|
||||
global count
|
||||
doc_info = dict()
|
||||
if pd.notna(x['city']) and pd.notna(x['region']):
|
||||
if x['city'] + x['region'] in short_map:
|
||||
short_info = short_map[x['city'] + x['region']]
|
||||
x['city'] = short_info.split('|')[0]
|
||||
x['region'] = short_info.split('|')[1]
|
||||
|
||||
action_doc = {
|
||||
"_op_type": "update",
|
||||
"_index": index_name,
|
||||
"_id": x['_id'],
|
||||
"doc": doc_info
|
||||
}
|
||||
if len(doc_info) > 0:
|
||||
actions.append(action_doc)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
log.info(f"已更新{count}")
|
||||
actions.clear()
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始转换省市区全拼至206 {index_name}")
|
||||
|
||||
sql = "select city, region from community_summary"
|
||||
if pd.notna(city):
|
||||
sql += " where city='" + city + "'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
actions = list()
|
||||
community_summary_df.apply(lambda x: deal(actions, x), axis=1)
|
||||
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
|
||||
log.info(f"完成转换省市区全拼至206 {index_name}")
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -110,5 +110,11 @@ with DAG(
|
||||
task_id='other',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
# 统一输出为省市区全拼
|
||||
full_name = BashOperator(
|
||||
task_id='full_name',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/full_name.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
# 人均消费并行,值为空
|
||||
comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> city_level_and_price_rating >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other
|
||||
comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> city_level_and_price_rating >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other \
|
||||
>> full_name
|
||||
|
||||
Reference in New Issue
Block a user