修复文件读取bug
This commit is contained in:
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||
from elasticsearch import helpers
|
||||
import numpy as np
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
@@ -13,6 +13,7 @@ es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
m_date = dateutils.iso_format()
|
||||
path = "/home/airflow/data/community"
|
||||
|
||||
|
||||
def find_around_price(lng, lat):
|
||||
@@ -100,7 +101,7 @@ def sync():
|
||||
# 按最新的百分位更新库中价格状态
|
||||
update_status(new_df)
|
||||
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
def find_around_price_item(index, item):
|
||||
if 'lng' not in item:
|
||||
|
||||
@@ -6,6 +6,7 @@ import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||
from elasticsearch import helpers
|
||||
import os
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
@@ -14,6 +15,9 @@ es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
index_name = "community_summary"
|
||||
m_date = dateutils.iso_format()
|
||||
|
||||
path = "/home/airflow/data/community"
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
|
||||
def build_doc(item):
|
||||
"""
|
||||
@@ -120,5 +124,5 @@ def sync(city=None):
|
||||
actions.clear()
|
||||
|
||||
# 写入csv文件,防止小区变动导致的后续过程更新报错
|
||||
seaweed_df.to_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||
log.info(f"完成同步小区基本属性至206 {index_name}")
|
||||
|
||||
@@ -10,6 +10,7 @@ import traffic
|
||||
import medical
|
||||
import life
|
||||
import relative_price_ranking
|
||||
import consumption_potential
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -28,3 +29,4 @@ if type == 'all':
|
||||
medical.sync()
|
||||
life.sync()
|
||||
relative_price_ranking.sync()
|
||||
consumption_potential.sync()
|
||||
|
||||
@@ -12,9 +12,9 @@ def calc_consumption_potential(city):
|
||||
"""
|
||||
计算消费潜力
|
||||
"""
|
||||
sql = "select id, city, region, name, total_price, estate_charge, avg_price_region_ranking, avg_cost from community_summary"
|
||||
sql = "select id, city, region, name, total_price, estate_charge, relative_price_ranking, avg_cost from community_summary"
|
||||
if pd.notna(city):
|
||||
sql += " where city=" + city
|
||||
sql += " where city='" + city + "'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.mean)
|
||||
community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.std)
|
||||
@@ -22,16 +22,26 @@ def calc_consumption_potential(city):
|
||||
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(np.mean)
|
||||
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(np.std)
|
||||
|
||||
community_summary_df['avg_price_region_ranking_mean'] = community_summary_df.groupby(by=['city'])['avg_price_region_ranking'].transform(np.mean)
|
||||
community_summary_df['avg_price_region_ranking_std'] = community_summary_df.groupby(by=['city'])['avg_price_region_ranking'].transform(np.std)
|
||||
community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])['relative_price_ranking'].transform(np.mean)
|
||||
community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])['relative_price_ranking'].transform(np.std)
|
||||
|
||||
community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.mean)
|
||||
community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.std)
|
||||
|
||||
community_summary_df['consumption_potential'] = 286.0134196 * (community_summary_df['total_price'] - community_summary_df['total_price_avg']) / community_summary_df['total_price_std'] \
|
||||
+ 139.1374524 * (community_summary_df['estate_charge'] - community_summary_df['estate_charge_avg']) / community_summary_df['estate_charge_std'] \
|
||||
+ 21.8427926 * (community_summary_df['avg_price_region_ranking'] - community_summary_df['avg_price_region_ranking_avg']) / community_summary_df['avg_price_region_ranking_std'] \
|
||||
+ 67.0195871700 * (community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std'] + 4099.978248
|
||||
community_summary_df['total_price_score'] = 286.0134196 * (community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df['total_price_std']
|
||||
community_summary_df['total_price_score'].fillna(0, inplace=True)
|
||||
|
||||
community_summary_df['estate_charge_score'] = 139.1374524 * (community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std']
|
||||
community_summary_df['estate_charge_score'].fillna(0, inplace=True)
|
||||
|
||||
community_summary_df['relative_price_ranking_score'] = 21.8427926 * (community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / community_summary_df['relative_price_ranking_std']
|
||||
community_summary_df['relative_price_ranking_score'].fillna(0, inplace=True)
|
||||
|
||||
community_summary_df['avg_cost_score'] = 67.0195871700 * (community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std']
|
||||
community_summary_df['avg_cost_score'].fillna(0, inplace=True)
|
||||
|
||||
community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df['estate_charge_score'] + community_summary_df['relative_price_ranking_score'] \
|
||||
+ community_summary_df['avg_cost_score'] + community_summary_df['avg_cost_score'] + 4099.978248
|
||||
pass
|
||||
|
||||
|
||||
@@ -42,3 +52,7 @@ def calc_consumption_rating():
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
calc_consumption_potential(city)
|
||||
pass
|
||||
@@ -12,6 +12,7 @@ log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
m_date = dateutils.iso_format()
|
||||
|
||||
count = 0
|
||||
@@ -79,7 +80,7 @@ def sync(city=None):
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
|
||||
@@ -13,6 +13,7 @@ seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
m_date = dateutils.iso_format()
|
||||
|
||||
count = 0
|
||||
@@ -67,7 +68,7 @@ def sync(city=None):
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
|
||||
@@ -10,6 +10,7 @@ log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
m_date = dateutils.iso_format()
|
||||
|
||||
|
||||
@@ -42,7 +43,7 @@ def sync(city=None):
|
||||
cities = [city]
|
||||
else:
|
||||
cities = None
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
# 小区所在区域均价
|
||||
region_price_df = commutils.get_region_price(city=city)
|
||||
|
||||
@@ -13,6 +13,7 @@ log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
m_date = dateutils.iso_format()
|
||||
|
||||
|
||||
@@ -57,7 +58,7 @@ def build_doc(item):
|
||||
def sync():
|
||||
log.info("开始同步相对价格排名数据...")
|
||||
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
def find_around_price_item(index, item):
|
||||
if 'lng' not in item:
|
||||
@@ -66,8 +67,11 @@ def sync():
|
||||
lat = commutils.format_lat_lng(item['lat'])
|
||||
if pd.notna(lng):
|
||||
around_seaweed_df = find_around_seaweed(lng, lat)
|
||||
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
|
||||
seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0]
|
||||
if len(around_seaweed_df) > 1:
|
||||
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
|
||||
seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0]
|
||||
elif len(around_seaweed_df) == 1:
|
||||
seaweed_df.loc[index, 'relative_price_ranking'] = 0.5
|
||||
|
||||
with ThreadPoolExecutor(max_workers=50) as pool:
|
||||
for index, item in seaweed_df.iterrows():
|
||||
|
||||
@@ -12,6 +12,7 @@ seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
m_date = dateutils.iso_format()
|
||||
|
||||
|
||||
@@ -28,7 +29,7 @@ def sync(city=None):
|
||||
seaweed_query = {'cat': 'district', 'status': 0}
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
with ThreadPoolExecutor(50) as pool:
|
||||
def deal(index, item):
|
||||
|
||||
@@ -13,6 +13,7 @@ seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
m_date = dateutils.iso_format()
|
||||
|
||||
|
||||
@@ -28,7 +29,7 @@ def sync(city=None):
|
||||
seaweed_query = {'cat': 'district', 'status': 0}
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
# 加载库中均价数据
|
||||
query_sql = "SELECT city, region, name, avg_price FROM community_summary "
|
||||
|
||||
@@ -10,6 +10,7 @@ log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
m_date = dateutils.iso_format()
|
||||
|
||||
|
||||
@@ -28,7 +29,7 @@ def sync(city=None):
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
cities = [city]
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
deal_price_df = commutils.get_deal_avg_price(cities, 3)
|
||||
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
|
||||
|
||||
@@ -12,6 +12,7 @@ log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
m_date = dateutils.iso_format()
|
||||
|
||||
count = 0
|
||||
@@ -76,8 +77,7 @@ def sync(city=None):
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
|
||||
seaweed_df = seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@',
|
||||
encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
|
||||
Reference in New Issue
Block a user