dag修改
This commit is contained in:
@@ -84,6 +84,7 @@ def update_status(new_df):
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
|
||||
|
||||
def sync():
|
||||
log.info("开始同步消费数据...")
|
||||
df = es206_client.scan_by_sql_to_data_frame(
|
||||
@@ -145,3 +146,5 @@ def sync():
|
||||
|
||||
log.info("完成同步消费数据...")
|
||||
|
||||
|
||||
sync()
|
||||
@@ -19,6 +19,7 @@ path = "/home/airflow/data/community"
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
|
||||
|
||||
def build_doc(item):
|
||||
"""
|
||||
构建字典
|
||||
@@ -64,9 +65,9 @@ def sync(city=None):
|
||||
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1,
|
||||
'alias': 1, 'estate_type2': 1, 'build_type': 1,
|
||||
'household_count': 1,
|
||||
'estate_charge': 1, 'finish_building_date': 1})
|
||||
'alias': 1, 'estate_type2': 1, 'build_type': 1,
|
||||
'household_count': 1,
|
||||
'estate_charge': 1, 'finish_building_date': 1})
|
||||
.batch_size(100000)))
|
||||
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}")
|
||||
@@ -126,3 +127,6 @@ def sync(city=None):
|
||||
# 写入csv文件,防止小区变动导致的后续过程更新报错
|
||||
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||
log.info(f"完成同步小区基本属性至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
import logging
|
||||
import sys
|
||||
import comm
|
||||
import price
|
||||
import sell
|
||||
import trade
|
||||
import avg_cost
|
||||
import total_price
|
||||
import traffic
|
||||
import medical
|
||||
import life
|
||||
import relative_price_ranking
|
||||
import consumption_potential
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
type = sys.argv[1] if len(sys.argv) > 1 else 'all'
|
||||
log.info("同步类型:" + type)
|
||||
|
||||
|
||||
if type == 'all':
|
||||
comm.sync()
|
||||
price.sync()
|
||||
sell.sync()
|
||||
avg_cost.sync()
|
||||
trade.sync()
|
||||
total_price.sync()
|
||||
traffic.sync()
|
||||
medical.sync()
|
||||
life.sync()
|
||||
relative_price_ranking.sync()
|
||||
consumption_potential.sync()
|
||||
@@ -16,36 +16,50 @@ def calc_consumption_potential(city):
|
||||
if pd.notna(city):
|
||||
sql += " where city='" + city + "'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.mean)
|
||||
community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform(
|
||||
np.mean)
|
||||
community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.std)
|
||||
|
||||
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(np.mean)
|
||||
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(np.std)
|
||||
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(
|
||||
np.mean)
|
||||
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(
|
||||
np.std)
|
||||
|
||||
community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])['relative_price_ranking'].transform(np.mean)
|
||||
community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])['relative_price_ranking'].transform(np.std)
|
||||
community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])[
|
||||
'relative_price_ranking'].transform(np.mean)
|
||||
community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])[
|
||||
'relative_price_ranking'].transform(np.std)
|
||||
|
||||
community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.mean)
|
||||
community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.std)
|
||||
|
||||
community_summary_df['total_price_score'] = 286.0134196 * (community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df['total_price_std']
|
||||
community_summary_df['total_price_score'] = 286.0134196 * (
|
||||
community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df[
|
||||
'total_price_std']
|
||||
community_summary_df['total_price_score'].fillna(0, inplace=True)
|
||||
|
||||
community_summary_df['estate_charge_score'] = 139.1374524 * (community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std']
|
||||
community_summary_df['estate_charge_score'] = 139.1374524 * (
|
||||
community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / \
|
||||
community_summary_df['estate_charge_std']
|
||||
community_summary_df['estate_charge_score'].fillna(0, inplace=True)
|
||||
|
||||
community_summary_df['relative_price_ranking_score'] = 21.8427926 * (community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / community_summary_df['relative_price_ranking_std']
|
||||
community_summary_df['relative_price_ranking_score'] = 21.8427926 * (
|
||||
community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / \
|
||||
community_summary_df['relative_price_ranking_std']
|
||||
community_summary_df['relative_price_ranking_score'].fillna(0, inplace=True)
|
||||
|
||||
community_summary_df['avg_cost_score'] = 67.0195871700 * (community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std']
|
||||
community_summary_df['avg_cost_score'] = 67.0195871700 * (
|
||||
community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df[
|
||||
'avg_cost_std']
|
||||
community_summary_df['avg_cost_score'].fillna(0, inplace=True)
|
||||
|
||||
community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df['estate_charge_score'] + community_summary_df['relative_price_ranking_score'] \
|
||||
+ community_summary_df['avg_cost_score'] + community_summary_df['avg_cost_score'] + 4099.978248
|
||||
community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df[
|
||||
'estate_charge_score'] + community_summary_df['relative_price_ranking_score'] \
|
||||
+ community_summary_df['avg_cost_score'] + community_summary_df[
|
||||
'avg_cost_score'] + 4099.978248
|
||||
pass
|
||||
|
||||
|
||||
|
||||
def calc_consumption_rating():
|
||||
"""
|
||||
计算消费等级
|
||||
@@ -55,4 +69,7 @@ def calc_consumption_rating():
|
||||
|
||||
def sync(city=None):
|
||||
calc_consumption_potential(city)
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
sync()
|
||||
|
||||
@@ -94,4 +94,7 @@ def sync(city=None):
|
||||
log.error(e)
|
||||
actions.clear()
|
||||
|
||||
log.info(f"完成同步生活配套至206 {index_name}")
|
||||
log.info(f"完成同步生活配套至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
@@ -82,4 +82,7 @@ def sync(city=None):
|
||||
log.error(e)
|
||||
actions.clear()
|
||||
|
||||
log.info(f"完成同步医院至206 {index_name}")
|
||||
log.info(f"完成同步医院至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
@@ -115,3 +115,6 @@ def sync(city=None):
|
||||
print(e)
|
||||
actions.clear()
|
||||
log.info(f"完成同步价格信息至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
@@ -107,3 +107,5 @@ def sync():
|
||||
|
||||
log.info("完成同步相对价格排名数据...")
|
||||
|
||||
|
||||
sync()
|
||||
@@ -81,3 +81,6 @@ def sync(city=None):
|
||||
print(e)
|
||||
actions.clear()
|
||||
log.info(f"完成同步出售信息至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
@@ -23,6 +23,7 @@ def build_doc(item):
|
||||
doc_dict['total_price'] = int(item['total_price'])
|
||||
return doc_dict
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步户均总价信息至206 {index_name}")
|
||||
|
||||
@@ -66,14 +67,13 @@ def sync(city=None):
|
||||
if pd.notna(avg_price):
|
||||
min_avg_price = 0.8 * avg_price
|
||||
max_avg_rice = 1.2 * avg_price
|
||||
total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366, min_avg_price=min_avg_price, max_avg_price=max_avg_rice)
|
||||
total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366,
|
||||
min_avg_price=min_avg_price, max_avg_price=max_avg_rice)
|
||||
seaweed_df.loc[index, 'total_price'] = total_price
|
||||
|
||||
for index, item in seaweed_df.iterrows():
|
||||
pool.submit(deal, index, item)
|
||||
|
||||
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in seaweed_df.iterrows():
|
||||
@@ -102,3 +102,6 @@ def sync(city=None):
|
||||
print(e)
|
||||
actions.clear()
|
||||
log.info(f"完成同步户均总价信息至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
|
||||
@@ -33,11 +33,11 @@ def sync(city=None):
|
||||
|
||||
deal_price_df = commutils.get_deal_avg_price(cities, 3)
|
||||
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
|
||||
deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name', 'avg_price': 'three_month_trade_avg_price'})
|
||||
deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name',
|
||||
'avg_price': 'three_month_trade_avg_price'})
|
||||
|
||||
seaweed_df = pd.merge(seaweed_df, deal_price_df, how='left', on=['city', 'region', 'name'])
|
||||
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in seaweed_df.iterrows():
|
||||
@@ -65,4 +65,7 @@ def sync(city=None):
|
||||
except Exception as e:
|
||||
print(e)
|
||||
actions.clear()
|
||||
log.info(f"完成同步成交信息至206 {index_name}")
|
||||
log.info(f"完成同步成交信息至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
|
||||
@@ -91,4 +91,7 @@ def sync(city=None):
|
||||
print(e)
|
||||
actions.clear()
|
||||
|
||||
log.info(f"完成同步交通至206 {index_name}")
|
||||
log.info(f"完成同步交通至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
@@ -11,7 +11,48 @@ with DAG(
|
||||
description="地址输出信息分类数据定时更新"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='community_summary_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/community_summary.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
task_id='comm_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t1
|
||||
t2 = BashOperator(
|
||||
task_id='price_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t3 = BashOperator(
|
||||
task_id='sell_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t4 = BashOperator(
|
||||
task_id='avg_cost_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t5 = BashOperator(
|
||||
task_id='trade_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t6 = BashOperator(
|
||||
task_id='total_price_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t7 = BashOperator(
|
||||
task_id='traffic_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t8 = BashOperator(
|
||||
task_id='medical_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t9 = BashOperator(
|
||||
task_id='life_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t10 = BashOperator(
|
||||
task_id='relative_price_ranking_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t11 = BashOperator(
|
||||
task_id='consumption_potential_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption_potential.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
|
||||
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t7 >> t8 >> t9 >> t10 >> t11
|
||||
Reference in New Issue
Block a user