Merge remote-tracking branch 'origin/master'

This commit is contained in:
xuhe
2023-02-24 18:26:11 +08:00
22 changed files with 138 additions and 84 deletions
+3 -1
View File
@@ -7,6 +7,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
from elasticsearch import helpers from elasticsearch import helpers
import numpy as np import numpy as np
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
import sys
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206') es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -150,4 +151,5 @@ def sync(city=None):
log.info("完成同步消费数据...") log.info("完成同步消费数据...")
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -3,10 +3,11 @@
""" """
import logging import logging
import pandas as pd import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils from goojia_utils import ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from threading import Lock from threading import Lock
import sys
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206') es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -104,4 +105,5 @@ def sync(city=None):
log.info(f"完成同步商圈至206 {index_name}") log.info(f"完成同步商圈至206 {index_name}")
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+7 -2
View File
@@ -7,6 +7,7 @@ import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers from elasticsearch import helpers
import os import os
import sys
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -93,7 +94,10 @@ def sync(city=None):
'estate_charge': 1, 'finish_building_date': 1}) 'estate_charge': 1, 'finish_building_date': 1})
.batch_size(100000))) .batch_size(100000)))
community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}") sql = f"select id from {index_name} "
if pd.notna(city):
sql += f" where city='{city}'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
seaweed_df['_id'] = seaweed_df['_id'].astype(str) seaweed_df['_id'] = seaweed_df['_id'].astype(str)
if len(community_summary_df) > 0: if len(community_summary_df) > 0:
@@ -154,4 +158,5 @@ def sync(city=None):
log.info(f"完成同步小区基本属性至206 {index_name}") log.info(f"完成同步小区基本属性至206 {index_name}")
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -8,6 +8,7 @@ import logging
import os import os
from elasticsearch import helpers from elasticsearch import helpers
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import sys
from sklearn.mixture import GaussianMixture from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import QuantileTransformer from sklearn.preprocessing import QuantileTransformer
@@ -249,5 +250,7 @@ def save_png(community_city):
# 关窗口 # 关窗口
plt.close() plt.close()
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
#dingutils.send_text("【分布图生成完毕】财力评级已完成, 请点击 https://nextcloud.fangjia.com/s/Jqs5BzXi8xLYeH9 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78") #dingutils.send_text("【分布图生成完毕】财力评级已完成, 请点击 https://nextcloud.fangjia.com/s/Jqs5BzXi8xLYeH9 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
@@ -3,10 +3,11 @@
""" """
import logging import logging
import pandas as pd import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from threading import Lock from threading import Lock
import sys
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206') es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -62,11 +63,10 @@ def deal(actions, item):
def sync(city=None): def sync(city=None):
log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}") log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}")
seaweed_query = {'status': 0, 'cat': 'district'}
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
actions = list() actions = list()
with ThreadPoolExecutor(max_workers=30) as pool: with ThreadPoolExecutor(max_workers=30) as pool:
@@ -83,4 +83,5 @@ def sync(city=None):
log.info(f"完成同步教育资源至206 {index_name}") log.info(f"完成同步教育资源至206 {index_name}")
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -3,10 +3,11 @@
""" """
import logging import logging
import pandas as pd import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from threading import Lock from threading import Lock
import sys
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206') es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -95,4 +96,5 @@ def sync(city=None):
log.info(f"完成同步教育资源至206 {index_name}") log.info(f"完成同步教育资源至206 {index_name}")
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -4,6 +4,7 @@
from goojia_utils import ElasticSearchClient, areautils, dateutils from goojia_utils import ElasticSearchClient, areautils, dateutils
import pandas as pd import pandas as pd
import logging import logging
import sys
from elasticsearch import helpers from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
@@ -140,4 +141,5 @@ def sync(city=None):
sync_by_city(city) sync_by_city(city)
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -5,6 +5,7 @@ from goojia_utils import ElasticSearchClient, nextcloudutils
import pandas as pd import pandas as pd
import logging import logging
import os import os
import sys
from elasticsearch import helpers from elasticsearch import helpers
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
@@ -163,6 +164,6 @@ def save_png(community_city):
plt.close() plt.close()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync() sync(city)
#dingutils.send_text("【分布图生成完毕】物业费评级已完成, 请点击 https://nextcloud.fangjia.com/s/HbifLoEDc5wi8HR 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78") #dingutils.send_text("【分布图生成完毕】物业费评级已完成, 请点击 https://nextcloud.fangjia.com/s/HbifLoEDc5wi8HR 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
+3 -1
View File
@@ -14,6 +14,7 @@ import pandas as pd
from elasticsearch import helpers from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from threading import Lock from threading import Lock
import sys
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206') es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -140,4 +141,5 @@ def sync(city=None):
log.info(f"完成同步附加标签至206 {index_name}") log.info(f"完成同步附加标签至206 {index_name}")
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+4 -2
View File
@@ -3,10 +3,11 @@
""" """
import logging import logging
import pandas as pd import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from threading import Lock from threading import Lock
import sys
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206') es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -96,4 +97,5 @@ def sync(city=None):
log.info(f"完成同步生活配套至206 {index_name}") log.info(f"完成同步生活配套至206 {index_name}")
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -1
View File
@@ -7,6 +7,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date
from elasticsearch import helpers from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from threading import Lock from threading import Lock
import sys
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -83,4 +84,5 @@ def sync(city=None):
log.info(f"完成同步医院至206 {index_name}") log.info(f"完成同步医院至206 {index_name}")
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -2
View File
@@ -5,7 +5,7 @@ from goojia_utils import ElasticSearchClient, dateutils
import logging import logging
import pandas as pd import pandas as pd
from elasticsearch import helpers from elasticsearch import helpers
import datetime import sys
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206') es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -57,4 +57,5 @@ def sync(city=None):
log.info(f"完成同步售租比、人口数至206 {index_name}") log.info(f"完成同步售租比、人口数至206 {index_name}")
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -1
View File
@@ -5,6 +5,7 @@ import logging
import pandas as pd import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
from elasticsearch import helpers from elasticsearch import helpers
import sys
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206') es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -139,4 +140,5 @@ def sync(city=None):
log.info(f"完成同步价格信息至206 {index_name}") log.info(f"完成同步价格信息至206 {index_name}")
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -8,6 +8,7 @@ from goojia_utils import ElasticSearchClient
import pandas as pd import pandas as pd
import logging import logging
from elasticsearch import helpers from elasticsearch import helpers
import sys
from sklearn.mixture import GaussianMixture from sklearn.mixture import GaussianMixture
@@ -83,4 +84,5 @@ def sync(city=None):
calc_price_rating(city) calc_price_rating(city)
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -7,6 +7,7 @@ import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils from goojia_utils import ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
import sys
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206') es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -112,4 +113,5 @@ def sync(city=None):
log.info("完成同步相对价格排名数据...") log.info("完成同步相对价格排名数据...")
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -1
View File
@@ -6,6 +6,7 @@ import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
import sys
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -84,4 +85,5 @@ def sync(city=None):
log.info(f"完成同步出售信息至206 {index_name}") log.info(f"完成同步出售信息至206 {index_name}")
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -7,6 +7,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date
from elasticsearch import helpers from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
import re import re
import sys
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -106,4 +107,5 @@ def sync(city=None):
log.info(f"完成同步户均总价信息至206 {index_name}") log.info(f"完成同步户均总价信息至206 {index_name}")
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -5,6 +5,7 @@ import logging
import pandas as pd import pandas as pd
from goojia_utils import ElasticSearchClient, dateutils from goojia_utils import ElasticSearchClient, dateutils
from elasticsearch import helpers from elasticsearch import helpers
import sys
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206') es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -61,6 +62,9 @@ def complete_by_build_year_area(community_factor_df):
for index, build_year_area in build_year_area_df.iterrows(): for index, build_year_area in build_year_area_df.iterrows():
build_year_area_dict[ build_year_area_dict[
build_year_area['city'] + '|' + build_year_area['region'] + "|" + build_year_area['buildyear']] = build_year_area['factor'] build_year_area['city'] + '|' + build_year_area['region'] + "|" + build_year_area['buildyear']] = build_year_area['factor']
# 修复按城市跑批数据字段可能不存在问题
if 'finish_building_date' not in community_factor_df.columns or 'total_price' not in community_factor_df.columns:
return False
for index, item in community_factor_df[ for index, item in community_factor_df[
(community_factor_df['total_price'].isna()) & (community_factor_df['finish_building_date'].notna())].iterrows(): (community_factor_df['total_price'].isna()) & (community_factor_df['finish_building_date'].notna())].iterrows():
finish_building_date = item['finish_building_date'] finish_building_date = item['finish_building_date']
@@ -101,18 +105,22 @@ def complete_by_region_area(community_factor_df):
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor'] community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
def sync(): def sync(city=None):
log.info(f"开始补全户均总价信息至206 {index_name}") log.info(f"开始补全户均总价信息至206 {index_name}")
community_df = es206_client.scan_by_sql_to_data_frame("select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null") sql = "select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null "
if pd.notna(city):
sql += " and city='" + city + "'"
community_df = es206_client.scan_by_sql_to_data_frame(sql)
community_factor_df = complete_by_area(community_df) if len(community_df) > 0:
complete_by_build_year_area(community_factor_df) community_factor_df = complete_by_area(community_df)
complete_by_region_area(community_factor_df) complete_by_build_year_area(community_factor_df)
complete_by_region_area(community_factor_df)
save(community_factor_df) save(community_factor_df)
log.info(f"完成补全户均总价信息至206 {index_name}") log.info(f"完成补全户均总价信息至206 {index_name}")
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+13 -5
View File
@@ -3,6 +3,7 @@
""" """
import logging import logging
import pandas as pd import pandas as pd
import sys
from goojia_utils import ElasticSearchClient, commutils, dateutils from goojia_utils import ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers from elasticsearch import helpers
@@ -24,14 +25,20 @@ def build_doc(item):
def sync(city=None): def sync(city=None):
log.info(f"开始同步成交信息至206 {index_name}") log.info(f"开始同步成交信息至206 {index_name}")
seaweed_query = {'cat': 'district', 'status': 0}
cities = None cities = None
if pd.notna(city):
seaweed_query['city'] = city
cities = [city]
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
if pd.notna(city):
cities = [city]
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
deal_price_df = commutils.get_deal_avg_price(cities, 3) deal_price_df = commutils.get_deal_avg_price(cities, 3)
if len(deal_price_df) <= 0:
log.info(f"无成交,无需同步")
return
if 'match_city' not in deal_price_df.columns or 'match_region' not in deal_price_df.columns or 'match_name' not in deal_price_df.columns:
log.info(f"无成交,无需同步")
return
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']] deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name', deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name',
'avg_price': 'three_month_trade_avg_price'}) 'avg_price': 'three_month_trade_avg_price'})
@@ -68,4 +75,5 @@ def sync(city=None):
log.info(f"完成同步成交信息至206 {index_name}") log.info(f"完成同步成交信息至206 {index_name}")
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -1
View File
@@ -7,6 +7,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from threading import Lock from threading import Lock
import sys
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206') es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -86,4 +87,5 @@ def sync(city=None):
log.info(f"完成同步交通至206 {index_name}") log.info(f"完成同步交通至206 {index_name}")
sync() city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+42 -41
View File
@@ -10,84 +10,85 @@ with DAG(
catchup=True, catchup=True,
description="地址输出信息分类数据定时更新" description="地址输出信息分类数据定时更新"
) as dag: ) as dag:
t1 = BashOperator( comm = BashOperator(
task_id='comm', task_id='comm',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t2 = BashOperator( price = BashOperator(
task_id='price', task_id='price',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t3 = BashOperator( sell = BashOperator(
task_id='sell', task_id='sell',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t4 = BashOperator( avg_cost = BashOperator(
task_id='avg_cost', task_id='avg_cost',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t5 = BashOperator( trade = BashOperator(
task_id='trade', task_id='trade',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t6 = BashOperator( total_price = BashOperator(
task_id='total_price', task_id='total_price',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t6_1 = BashOperator( total_price_complete = BashOperator(
task_id='total_price_complete', task_id='total_price_complete',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t7 = BashOperator( traffic = BashOperator(
task_id='traffic', task_id='traffic',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t7_1 = BashOperator( business_district = BashOperator(
task_id='business_district', task_id='business_district',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t8 = BashOperator( medical = BashOperator(
task_id='medical', task_id='medical',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t9 = BashOperator( life = BashOperator(
task_id='life', task_id='life',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t10 = BashOperator( relative_price_ranking = BashOperator(
task_id='relative_price_ranking', task_id='relative_price_ranking',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t10_1 = BashOperator( estate_charge_complete = BashOperator(
task_id='estate_charge_complete', task_id='estate_charge_complete',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t10_2 = BashOperator( estate_charge_ranking = BashOperator(
task_id='estate_charge_ranking', task_id='estate_charge_ranking',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t11 = BashOperator( consumption = BashOperator(
task_id='consumption', task_id='consumption',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t12 = BashOperator( price_rating = BashOperator(
task_id='price_rating', task_id='price_rating',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t13 = BashOperator( education = BashOperator(
task_id='education', task_id='education',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t14 = BashOperator( education2 = BashOperator(
task_id='education2', task_id='education2',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t15 = BashOperator( extra = BashOperator(
task_id='extra', task_id='extra',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t16 = BashOperator( other = BashOperator(
task_id='other', task_id='other',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
) )
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t10_1 >> t10_2 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16 comm >> price >> avg_cost >> total_price >> total_price_complete >> relative_price_ranking >> estate_charge_complete >> estate_charge_ranking \
>> consumption >> price_rating >> sell >> trade >> traffic >> business_district >> medical >> life >> education >> education2 >> extra >> other
+1 -1
View File
@@ -146,7 +146,7 @@ for index, item in city_count_df.iterrows():
city_count_df = city_count_df.rename(columns={'小区ID': '小区数量'}) city_count_df = city_count_df.rename(columns={'小区ID': '小区数量'})
city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度') city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度')
excel_writer.save() #excel_writer.save()
excel_writer.close() excel_writer.close()
up_to_nextcloud() up_to_nextcloud()