diff --git a/dags/company/company/bd2gaode.py b/dags/company/company/bd2gaode.py index 6fcb923..82259ac 100644 --- a/dags/company/company/bd2gaode.py +++ b/dags/company/company/bd2gaode.py @@ -11,7 +11,30 @@ flag = False for province in areautils.get_all_province(): print(f"开始处理{province}数据") # 百度经纬度转高德经纬度 - df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select location from didata_enterprise where province='{province}'") + df = DIDATA_DEV_ELASTIC_CLIENT.scan_to_data_frame(index='didata_enterprise', query={ + "query": { + "bool": { + "must": [ + { + "term": { + "province": { + "value": "" + province + "" + } + } + } + ], + "must_not": [ + { + "exists": { + "field": "gaode_location" + } + } + ] + } + } + }) + if len(df) == 0: + continue def deal_location(item): if pd.isna(item['location']): return item diff --git a/dags/company/company/fill_company_dev.py b/dags/company/company/fill_company_dev.py index 6d6b691..ad5490f 100644 --- a/dags/company/company/fill_company_dev.py +++ b/dags/company/company/fill_company_dev.py @@ -1,3 +1,68 @@ +from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, threads, geohashutils, areautils +import pandas as pd +from elasticsearch import helpers +import numpy as np +import re +import logging +log = logging.getLogger(__name__) + """ 通过开发区围栏更新企业开发区 TODO -""" \ No newline at end of file +""" +def sync(province): + log.info(f"开始同步{province}企业数据的开发区数据") + # 先临时禁用索引刷新 + DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( + index="didata_company", + body={"index": {"refresh_interval": "-1"}} + ) + + shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select name, shape from development_zone where province='{province}'") + actions = list() + count = 0 + for index, item in shape_df.iterrows(): + if pd.isna(item['shape']): + continue + ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_company', item['shape']['coordinates'][0], + field='location') + for info in ret: + doc_info = {'dev': item['name']} + action = { + "_op_type": "update", + "_index": "didata_company", + "_id": info['_id'], + "doc": doc_info, + "_retry_on_conflict": 5 + } + + actions.append(action) + if len(actions) == 10000: + for success, info in helpers.parallel_bulk( + DIDATA_DEV_ELASTIC_CLIENT.elastic_client, + actions, + thread_count=5, # 并行线程数 + chunk_size=1000 # 批量大小 + ): + if not success: + print(f"索引失败: {info}") + count += len(actions) + print("更新:" + str(count)) + actions.clear() + if len(actions) > 0: + for success, info in helpers.parallel_bulk( + DIDATA_DEV_ELASTIC_CLIENT.elastic_client, + actions, + thread_count=5, # 并行线程数 + chunk_size=1000 # 批量大小 + ): + if not success: + print(f"索引失败: {info}") + count += len(actions) + print("更新:" + str(count)) + actions.clear() + log.info(f"完成同步{province}企业数据的开发区数据") + # 恢复索引刷新 + DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( + index="didata_company", + body={"index": {"refresh_interval": "5s"}} + ) \ No newline at end of file diff --git a/dags/company/company/mark_right_dev.py b/dags/company/company/mark_right_dev.py new file mode 100644 index 0000000..22818ad --- /dev/null +++ b/dags/company/company/mark_right_dev.py @@ -0,0 +1,130 @@ +from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, threads, geohashutils, areautils +import pandas as pd +from elasticsearch import helpers +import numpy as np +import re +import logging +log = logging.getLogger(__name__) + +""" +对于没有围栏结果的,通过match_phrase匹配出来的开发区一定正确,如果有围栏出来的开发区,则企业经纬度不正确 +""" + +dev_alias = {'福建永定工业园区': '永定区永定工业园', '福建漳州古雷港经济开发区': '古雷港经济开发区', '福建莆田湄洲湾北岸经济开发区': '湄洲湾北岸经济开发区', + '福建南平工业园区': '南平工业园区', '泉州台商投资区': '泉州市台商投资区', '福建洛江经济开发区': '洛江园区', '福建武平工业园区': '武平工业园', + '福建晋江经济开发区': '晋江市经济开发区', '漳州台商投资区': '龙池开发区', '莆田高新技术产业开发区': '莆田市高新技术', + '莆田高新技术产业开发区': '莆田市涵江区高新', '绥安工业开发区': '绥安工业园', '绥安工业开发区': '黄仓开发区', '绥安工业开发区': '黄仓工业园', + '绥安工业开发区': '旧镇工业园', '绥安工业开发区': '大南坂开发区', '绥安工业开发区': '大南坂工业开发区', '福建龙州工业园区': '龙州工业园', + '长汀经济开发区':'腾飞开发区', '福建福州金山工业园区': '金山投资区', '厦门象屿综合保税区': '厦门现代物流园', '福清融侨经济技术开发区': '融侨经济', + '福建洛江经济开发区': '万安开发区'} +dev_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame("select province, city, district, name, alias from development_zone where province='福建省'") +# 不要去重,有可能存在别名, 如福建云霄常山经济开发区 +#dev_df = dev_df[dev_df['name'] == '福建政和经济开发区'] +actions = list() +for dev in dev_df.itertuples(): + if dev.name in ['中国(福建)自由贸易试验区厦门片区', '中国(福建)自由贸易试验区福州片区', '中国(福建)自由贸易试验区平潭片区']: + # 中国(福建)自由贸易试验区 已包含,防止反复更新 + continue + if dev.name in ['政和经济开发区']: + continue + + count = 0 + names = list() + names.append(dev.name) + if str(dev.alias) != 'nan': + names.extend(dev.alias) + #东侨经济技术开发区 福建省宁德市东侨经济开发区塔山村塔兴路25号 + if '经济技术开发区' in dev.name: + names.append(re.sub('经济技术开发区', '经济开发区', dev.name)) + # 福州经济技术开发区 福州开发区君竹路83号科技发展中心大楼第四层R472室(自贸试验区内) + names.append(re.sub('经济技术开发区', '开发区', dev.name)) + # 长汀经济开发区 长汀县经济开发区工业新区河梁大山体 + if '经济开发区' in dev.name: + if pd.notna(dev.city) and pd.notna(dev.district): + district_short = areautils.get_district_short(dev.city, dev.district) + if pd.notna(district_short): + names.append(re.sub(district_short, dev.district, dev.name)) + # 漳州高新技术产业开发区 福建省漳州高新区九湖镇木棉村后壁山401-1号 + if '高新技术产业开发区' in dev.name: + names.append(re.sub('高新技术产业开发区', '高新区', dev.name)) + # 厦门火炬高技术产业开发区 厦门火炬高新区软件园三期溪西山尾路63号1303室-3单元之三 + if '高技术产业开发区' in dev.name: + names.append(re.sub('高技术产业开发区', '高新区', dev.name)) + #areautils.get_district_display() + # 福建诏安工业园区 福建省漳州市诏安工业园区丹诏大道15号 + if pd.notna(dev.city): + province_short = areautils.get_province(dev.city) + new_name = re.sub(province_short, '', dev.name) + if pd.notna(dev.city) and pd.notna(dev.district): + city_short = areautils.get_city_short(dev.city) + district_short = areautils.get_district_short(dev.city, dev.district) + if pd.notna(city_short) and city_short in new_name: + names.append(new_name) + if pd.notna(district_short) and district_short in new_name: + names.append(new_name) + # 福建省漳州市长泰县经济开发区林墩办事处帝君路 福建长泰经济开发区 + new_new_name = re.sub(district_short, district_short + '县', new_name) + names.append(new_new_name) + new_new_name = re.sub(district_short, district_short + '区', new_name) + names.append(new_new_name) + + for key in dev_alias: + if dev.name == key: + names.append(dev_alias[key]) + for name in names: + query = { + "query": { + "bool": { + "must_not": [ + { + "term": { + "dev": { + "value": dev.name + } + } + }, + { + "term": { + "dev_status": { + "value": 0 + } + } + } + ], + "must": [ + { + "term": { + "province": { + "value": dev.province + } + } + }, + { + "match_phrase": { + "registered_address": name + } + } + ] + } + } + } + count_data = DIDATA_DEV_ELASTIC_CLIENT.elastic_client.count(index='didata_company',body=query) + if count_data['count'] > 0: + for doc in DIDATA_DEV_ELASTIC_CLIENT.scan(index='didata_company',query=query): + action = { + "_op_type": "update", + "_index": "didata_company", + "_id": doc['_id'], + "doc": {'dev': dev.name, 'dev_status': 0} + } + actions.append(action) + if len(actions) == 5000: + helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions) + count += len(actions) + print(f"已更新 {dev.name} 开发区" + str(count) + "记录") + actions.clear() + if len(actions) > 0: + helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions) + count += len(actions) + print(f"已更新 {dev.name} 开发区" + str(count) + "记录") + actions.clear() \ No newline at end of file diff --git a/dags/company/dag_company.py b/dags/company/dag_company.py index d2d09fa..773a51a 100644 --- a/dags/company/dag_company.py +++ b/dags/company/dag_company.py @@ -15,4 +15,12 @@ with DAG( task_id='fill_commpany_comm', bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_company_comm.py', ) - t1 \ No newline at end of file + t2 = BashOperator( + task_id='fill_commpany_dev', + bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_commpany_dev.py', + ) + t3 = BashOperator( + task_id='mark_right_dev', + bash_command='python /opt/airflow/dags/repo/dags/company/company/mark_right_dev.py', + ) + t1 >> t2 >> t3 \ No newline at end of file