Merge remote-tracking branch 'origin/master'
This commit is contained in:
@@ -7,6 +7,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
|
||||
from elasticsearch import helpers
|
||||
import numpy as np
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
@@ -150,4 +151,5 @@ def sync(city=None):
|
||||
log.info("完成同步消费数据...")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -3,10 +3,11 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
@@ -104,4 +105,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步商圈至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -7,6 +7,7 @@ import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||
from elasticsearch import helpers
|
||||
import os
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
@@ -93,7 +94,10 @@ def sync(city=None):
|
||||
'estate_charge': 1, 'finish_building_date': 1})
|
||||
.batch_size(100000)))
|
||||
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}")
|
||||
sql = f"select id from {index_name} "
|
||||
if pd.notna(city):
|
||||
sql += f" where city='{city}'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
||||
|
||||
if len(community_summary_df) > 0:
|
||||
@@ -154,4 +158,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步小区基本属性至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
|
||||
@@ -8,6 +8,7 @@ import logging
|
||||
import os
|
||||
from elasticsearch import helpers
|
||||
import matplotlib.pyplot as plt
|
||||
import sys
|
||||
|
||||
from sklearn.mixture import GaussianMixture
|
||||
from sklearn.preprocessing import QuantileTransformer
|
||||
@@ -249,5 +250,7 @@ def save_png(community_city):
|
||||
# 关窗口
|
||||
plt.close()
|
||||
|
||||
sync()
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
#dingutils.send_text("【分布图生成完毕】财力评级已完成, 请点击 https://nextcloud.fangjia.com/s/Jqs5BzXi8xLYeH9 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
|
||||
|
||||
@@ -3,10 +3,11 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
@@ -62,11 +63,10 @@ def deal(actions, item):
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}")
|
||||
seaweed_query = {'status': 0, 'cat': 'district'}
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
@@ -83,4 +83,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步教育资源至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -3,10 +3,11 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
@@ -95,4 +96,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步教育资源至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -4,6 +4,7 @@
|
||||
from goojia_utils import ElasticSearchClient, areautils, dateutils
|
||||
import pandas as pd
|
||||
import logging
|
||||
import sys
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
@@ -140,4 +141,5 @@ def sync(city=None):
|
||||
sync_by_city(city)
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -5,6 +5,7 @@ from goojia_utils import ElasticSearchClient, nextcloudutils
|
||||
import pandas as pd
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from elasticsearch import helpers
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
@@ -163,6 +164,6 @@ def save_png(community_city):
|
||||
plt.close()
|
||||
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
#dingutils.send_text("【分布图生成完毕】物业费评级已完成, 请点击 https://nextcloud.fangjia.com/s/HbifLoEDc5wi8HR 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
|
||||
@@ -14,6 +14,7 @@ import pandas as pd
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
@@ -140,4 +141,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步附加标签至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -3,10 +3,11 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
@@ -96,4 +97,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步生活配套至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -7,6 +7,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
@@ -83,4 +84,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步医院至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -5,7 +5,7 @@ from goojia_utils import ElasticSearchClient, dateutils
|
||||
import logging
|
||||
import pandas as pd
|
||||
from elasticsearch import helpers
|
||||
import datetime
|
||||
import sys
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
@@ -57,4 +57,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步售租比、人口数至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -5,6 +5,7 @@ import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
|
||||
from elasticsearch import helpers
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
@@ -139,4 +140,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步价格信息至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -8,6 +8,7 @@ from goojia_utils import ElasticSearchClient
|
||||
import pandas as pd
|
||||
import logging
|
||||
from elasticsearch import helpers
|
||||
import sys
|
||||
|
||||
from sklearn.mixture import GaussianMixture
|
||||
|
||||
@@ -83,4 +84,5 @@ def sync(city=None):
|
||||
calc_price_rating(city)
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
|
||||
@@ -7,6 +7,7 @@ import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
@@ -112,4 +113,5 @@ def sync(city=None):
|
||||
log.info("完成同步相对价格排名数据...")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -6,6 +6,7 @@ import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
@@ -84,4 +85,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步出售信息至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -7,6 +7,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import re
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
@@ -106,4 +107,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步户均总价信息至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
|
||||
@@ -5,6 +5,7 @@ import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, dateutils
|
||||
from elasticsearch import helpers
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
@@ -61,6 +62,9 @@ def complete_by_build_year_area(community_factor_df):
|
||||
for index, build_year_area in build_year_area_df.iterrows():
|
||||
build_year_area_dict[
|
||||
build_year_area['city'] + '|' + build_year_area['region'] + "|" + build_year_area['buildyear']] = build_year_area['factor']
|
||||
# 修复按城市跑批数据字段可能不存在问题
|
||||
if 'finish_building_date' not in community_factor_df.columns or 'total_price' not in community_factor_df.columns:
|
||||
return False
|
||||
for index, item in community_factor_df[
|
||||
(community_factor_df['total_price'].isna()) & (community_factor_df['finish_building_date'].notna())].iterrows():
|
||||
finish_building_date = item['finish_building_date']
|
||||
@@ -101,18 +105,22 @@ def complete_by_region_area(community_factor_df):
|
||||
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
|
||||
|
||||
|
||||
def sync():
|
||||
def sync(city=None):
|
||||
log.info(f"开始补全户均总价信息至206 {index_name}")
|
||||
|
||||
community_df = es206_client.scan_by_sql_to_data_frame("select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null")
|
||||
sql = "select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null "
|
||||
if pd.notna(city):
|
||||
sql += " and city='" + city + "'"
|
||||
community_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
community_factor_df = complete_by_area(community_df)
|
||||
complete_by_build_year_area(community_factor_df)
|
||||
complete_by_region_area(community_factor_df)
|
||||
|
||||
save(community_factor_df)
|
||||
if len(community_df) > 0:
|
||||
community_factor_df = complete_by_area(community_df)
|
||||
complete_by_build_year_area(community_factor_df)
|
||||
complete_by_region_area(community_factor_df)
|
||||
save(community_factor_df)
|
||||
|
||||
log.info(f"完成补全户均总价信息至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
import sys
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||
from elasticsearch import helpers
|
||||
|
||||
@@ -24,14 +25,20 @@ def build_doc(item):
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步成交信息至206 {index_name}")
|
||||
|
||||
seaweed_query = {'cat': 'district', 'status': 0}
|
||||
cities = None
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
cities = [city]
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
if pd.notna(city):
|
||||
cities = [city]
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
|
||||
deal_price_df = commutils.get_deal_avg_price(cities, 3)
|
||||
if len(deal_price_df) <= 0:
|
||||
log.info(f"无成交,无需同步")
|
||||
return
|
||||
if 'match_city' not in deal_price_df.columns or 'match_region' not in deal_price_df.columns or 'match_name' not in deal_price_df.columns:
|
||||
log.info(f"无成交,无需同步")
|
||||
return
|
||||
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
|
||||
deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name',
|
||||
'avg_price': 'three_month_trade_avg_price'})
|
||||
@@ -68,4 +75,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步成交信息至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
|
||||
@@ -7,6 +7,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
@@ -86,4 +87,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步交通至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -10,84 +10,85 @@ with DAG(
|
||||
catchup=True,
|
||||
description="地址输出信息分类数据定时更新"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
comm = BashOperator(
|
||||
task_id='comm',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t2 = BashOperator(
|
||||
price = BashOperator(
|
||||
task_id='price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t3 = BashOperator(
|
||||
sell = BashOperator(
|
||||
task_id='sell',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t4 = BashOperator(
|
||||
avg_cost = BashOperator(
|
||||
task_id='avg_cost',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t5 = BashOperator(
|
||||
trade = BashOperator(
|
||||
task_id='trade',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t6 = BashOperator(
|
||||
total_price = BashOperator(
|
||||
task_id='total_price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t6_1 = BashOperator(
|
||||
total_price_complete = BashOperator(
|
||||
task_id='total_price_complete',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t7 = BashOperator(
|
||||
traffic = BashOperator(
|
||||
task_id='traffic',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t7_1 = BashOperator(
|
||||
business_district = BashOperator(
|
||||
task_id='business_district',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t8 = BashOperator(
|
||||
medical = BashOperator(
|
||||
task_id='medical',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t9 = BashOperator(
|
||||
life = BashOperator(
|
||||
task_id='life',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t10 = BashOperator(
|
||||
relative_price_ranking = BashOperator(
|
||||
task_id='relative_price_ranking',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t10_1 = BashOperator(
|
||||
estate_charge_complete = BashOperator(
|
||||
task_id='estate_charge_complete',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t10_2 = BashOperator(
|
||||
estate_charge_ranking = BashOperator(
|
||||
task_id='estate_charge_ranking',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t11 = BashOperator(
|
||||
consumption = BashOperator(
|
||||
task_id='consumption',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t12 = BashOperator(
|
||||
price_rating = BashOperator(
|
||||
task_id='price_rating',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t13 = BashOperator(
|
||||
education = BashOperator(
|
||||
task_id='education',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t14 = BashOperator(
|
||||
education2 = BashOperator(
|
||||
task_id='education2',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t15 = BashOperator(
|
||||
extra = BashOperator(
|
||||
task_id='extra',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t16 = BashOperator(
|
||||
other = BashOperator(
|
||||
task_id='other',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t10_1 >> t10_2 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16
|
||||
comm >> price >> avg_cost >> total_price >> total_price_complete >> relative_price_ranking >> estate_charge_complete >> estate_charge_ranking \
|
||||
>> consumption >> price_rating >> sell >> trade >> traffic >> business_district >> medical >> life >> education >> education2 >> extra >> other
|
||||
|
||||
@@ -146,7 +146,7 @@ for index, item in city_count_df.iterrows():
|
||||
city_count_df = city_count_df.rename(columns={'小区ID': '小区数量'})
|
||||
city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度')
|
||||
|
||||
excel_writer.save()
|
||||
#excel_writer.save()
|
||||
excel_writer.close()
|
||||
|
||||
up_to_nextcloud()
|
||||
|
||||
Reference in New Issue
Block a user