dag修改

This commit is contained in:
shenjian@fangjia.com
2022-10-25 10:37:39 +08:00
parent f03e3dd81b
commit bf9414e957
13 changed files with 116 additions and 60 deletions
@@ -84,6 +84,7 @@ def update_status(new_df):
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
def sync():
log.info("开始同步消费数据...")
df = es206_client.scan_by_sql_to_data_frame(
@@ -145,3 +146,5 @@ def sync():
log.info("完成同步消费数据...")
sync()
+7 -3
View File
@@ -19,6 +19,7 @@ path = "/home/airflow/data/community"
if not os.path.exists(path):
os.makedirs(path)
def build_doc(item):
"""
构建字典
@@ -64,9 +65,9 @@ def sync(city=None):
seaweed_df = pd.DataFrame(
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1,
'alias': 1, 'estate_type2': 1, 'build_type': 1,
'household_count': 1,
'estate_charge': 1, 'finish_building_date': 1})
'alias': 1, 'estate_type2': 1, 'build_type': 1,
'household_count': 1,
'estate_charge': 1, 'finish_building_date': 1})
.batch_size(100000)))
community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}")
@@ -126,3 +127,6 @@ def sync(city=None):
# 写入csv文件,防止小区变动导致的后续过程更新报错
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
log.info(f"完成同步小区基本属性至206 {index_name}")
sync()
@@ -1,32 +0,0 @@
import logging
import sys
import comm
import price
import sell
import trade
import avg_cost
import total_price
import traffic
import medical
import life
import relative_price_ranking
import consumption_potential
log = logging.getLogger(__name__)
type = sys.argv[1] if len(sys.argv) > 1 else 'all'
log.info("同步类型:" + type)
if type == 'all':
comm.sync()
price.sync()
sell.sync()
avg_cost.sync()
trade.sync()
total_price.sync()
traffic.sync()
medical.sync()
life.sync()
relative_price_ranking.sync()
consumption_potential.sync()
@@ -16,36 +16,50 @@ def calc_consumption_potential(city):
if pd.notna(city):
sql += " where city='" + city + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.mean)
community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform(
np.mean)
community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.std)
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(np.mean)
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(np.std)
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(
np.mean)
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(
np.std)
community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])['relative_price_ranking'].transform(np.mean)
community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])['relative_price_ranking'].transform(np.std)
community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])[
'relative_price_ranking'].transform(np.mean)
community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])[
'relative_price_ranking'].transform(np.std)
community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.mean)
community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.std)
community_summary_df['total_price_score'] = 286.0134196 * (community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df['total_price_std']
community_summary_df['total_price_score'] = 286.0134196 * (
community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df[
'total_price_std']
community_summary_df['total_price_score'].fillna(0, inplace=True)
community_summary_df['estate_charge_score'] = 139.1374524 * (community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std']
community_summary_df['estate_charge_score'] = 139.1374524 * (
community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / \
community_summary_df['estate_charge_std']
community_summary_df['estate_charge_score'].fillna(0, inplace=True)
community_summary_df['relative_price_ranking_score'] = 21.8427926 * (community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / community_summary_df['relative_price_ranking_std']
community_summary_df['relative_price_ranking_score'] = 21.8427926 * (
community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / \
community_summary_df['relative_price_ranking_std']
community_summary_df['relative_price_ranking_score'].fillna(0, inplace=True)
community_summary_df['avg_cost_score'] = 67.0195871700 * (community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std']
community_summary_df['avg_cost_score'] = 67.0195871700 * (
community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df[
'avg_cost_std']
community_summary_df['avg_cost_score'].fillna(0, inplace=True)
community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df['estate_charge_score'] + community_summary_df['relative_price_ranking_score'] \
+ community_summary_df['avg_cost_score'] + community_summary_df['avg_cost_score'] + 4099.978248
community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df[
'estate_charge_score'] + community_summary_df['relative_price_ranking_score'] \
+ community_summary_df['avg_cost_score'] + community_summary_df[
'avg_cost_score'] + 4099.978248
pass
def calc_consumption_rating():
"""
计算消费等级
@@ -55,4 +69,7 @@ def calc_consumption_rating():
def sync(city=None):
calc_consumption_potential(city)
pass
pass
sync()
+4 -1
View File
@@ -94,4 +94,7 @@ def sync(city=None):
log.error(e)
actions.clear()
log.info(f"完成同步生活配套至206 {index_name}")
log.info(f"完成同步生活配套至206 {index_name}")
sync()
+4 -1
View File
@@ -82,4 +82,7 @@ def sync(city=None):
log.error(e)
actions.clear()
log.info(f"完成同步医院至206 {index_name}")
log.info(f"完成同步医院至206 {index_name}")
sync()
@@ -115,3 +115,6 @@ def sync(city=None):
print(e)
actions.clear()
log.info(f"完成同步价格信息至206 {index_name}")
sync()
@@ -107,3 +107,5 @@ def sync():
log.info("完成同步相对价格排名数据...")
sync()
+3
View File
@@ -81,3 +81,6 @@ def sync(city=None):
print(e)
actions.clear()
log.info(f"完成同步出售信息至206 {index_name}")
sync()
@@ -23,6 +23,7 @@ def build_doc(item):
doc_dict['total_price'] = int(item['total_price'])
return doc_dict
def sync(city=None):
log.info(f"开始同步户均总价信息至206 {index_name}")
@@ -66,14 +67,13 @@ def sync(city=None):
if pd.notna(avg_price):
min_avg_price = 0.8 * avg_price
max_avg_rice = 1.2 * avg_price
total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366, min_avg_price=min_avg_price, max_avg_price=max_avg_rice)
total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366,
min_avg_price=min_avg_price, max_avg_price=max_avg_rice)
seaweed_df.loc[index, 'total_price'] = total_price
for index, item in seaweed_df.iterrows():
pool.submit(deal, index, item)
actions = list()
count = 0
for index, item in seaweed_df.iterrows():
@@ -102,3 +102,6 @@ def sync(city=None):
print(e)
actions.clear()
log.info(f"完成同步户均总价信息至206 {index_name}")
sync()
+6 -3
View File
@@ -33,11 +33,11 @@ def sync(city=None):
deal_price_df = commutils.get_deal_avg_price(cities, 3)
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name', 'avg_price': 'three_month_trade_avg_price'})
deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name',
'avg_price': 'three_month_trade_avg_price'})
seaweed_df = pd.merge(seaweed_df, deal_price_df, how='left', on=['city', 'region', 'name'])
actions = list()
count = 0
for index, item in seaweed_df.iterrows():
@@ -65,4 +65,7 @@ def sync(city=None):
except Exception as e:
print(e)
actions.clear()
log.info(f"完成同步成交信息至206 {index_name}")
log.info(f"完成同步成交信息至206 {index_name}")
sync()
+4 -1
View File
@@ -91,4 +91,7 @@ def sync(city=None):
print(e)
actions.clear()
log.info(f"完成同步交通至206 {index_name}")
log.info(f"完成同步交通至206 {index_name}")
sync()
+44 -3
View File
@@ -11,7 +11,48 @@ with DAG(
description="地址输出信息分类数据定时更新"
) as dag:
t1 = BashOperator(
task_id='community_summary_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/community_summary.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
task_id='comm_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t1
t2 = BashOperator(
task_id='price_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t3 = BashOperator(
task_id='sell_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t4 = BashOperator(
task_id='avg_cost_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t5 = BashOperator(
task_id='trade_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t6 = BashOperator(
task_id='total_price_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t7 = BashOperator(
task_id='traffic_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t8 = BashOperator(
task_id='medical_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t9 = BashOperator(
task_id='life_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t10 = BashOperator(
task_id='relative_price_ranking_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t11 = BashOperator(
task_id='consumption_potential_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption_potential.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t7 >> t8 >> t9 >> t10 >> t11