diff --git a/dags/business/business/community/full_name.py b/dags/business/business/community/full_name.py new file mode 100644 index 0000000..b58c223 --- /dev/null +++ b/dags/business/business/community/full_name.py @@ -0,0 +1,68 @@ +""" +省市区简拼转全拼 +""" +from goojia_utils import ElasticSearchClient, dateutils +import logging +import pandas as pd +from elasticsearch import helpers +import sys +log = logging.getLogger(__name__) +es206_client = ElasticSearchClient(host='192.168.0.206') +es89_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') + +index_name = "community_summary" +path = "/home/airflow/data/community" +m_date = dateutils.iso_format() + +count = 0 + +short_df = es89_client.scan_by_sql_to_data_frame("select city, city_short, district, district_short from administrative_mapping where type='区域' ") +short_map = {} +def deal_short_map(x): + short_map[x['city_short'] + x['district_short']] = x['city'] + '|' + x['district'] +short_df.apply(lambda x: deal_short_map(x), axis=1) + +def deal(actions, x): + global count + doc_info = dict() + if pd.notna(x['city']) and pd.notna(x['region']): + if x['city'] + x['region'] in short_map: + short_info = short_map[x['city'] + x['region']] + x['city'] = short_info.split('|')[0] + x['region'] = short_info.split('|')[1] + + action_doc = { + "_op_type": "update", + "_index": index_name, + "_id": x['_id'], + "doc": doc_info + } + if len(doc_info) > 0: + actions.append(action_doc) + if len(actions) == 5000: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info(f"已更新{count}") + actions.clear() + + +def sync(city=None): + log.info(f"开始转换省市区全拼至206 {index_name}") + + sql = "select city, region from community_summary" + if pd.notna(city): + sql += " where city='" + city + "'" + community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + + actions = list() + community_summary_df.apply(lambda x: deal(actions, x), axis=1) + + if len(actions) > 0: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() + + log.info(f"完成转换省市区全拼至206 {index_name}") + + +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/dag_community_summary.py b/dags/business/dag_community_summary.py index c2df35b..97500bb 100644 --- a/dags/business/dag_community_summary.py +++ b/dags/business/dag_community_summary.py @@ -110,5 +110,11 @@ with DAG( task_id='other', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) + # 统一输出为省市区全拼 + full_name = BashOperator( + task_id='full_name', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/full_name.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', + ) # 人均消费并行,值为空 - comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> city_level_and_price_rating >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other + comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> city_level_and_price_rating >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other \ + >> full_name