修复文件读取bug

This commit is contained in:
shenjian@fangjia.com
2022-10-25 09:33:16 +08:00
parent 21e5c5fdac
commit f03e3dd81b
13 changed files with 53 additions and 22 deletions
+3 -2
View File
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from goojia_utils import ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
import numpy as np
from concurrent.futures import ThreadPoolExecutor
@@ -13,6 +13,7 @@ es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
m_date = dateutils.iso_format()
path = "/home/airflow/data/community"
def find_around_price(lng, lat):
@@ -100,7 +101,7 @@ def sync():
# 按最新的百分位更新库中价格状态
update_status(new_df)
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
def find_around_price_item(index, item):
if 'lng' not in item:
+5 -1
View File
@@ -6,6 +6,7 @@ import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
import os
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -14,6 +15,9 @@ es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
m_date = dateutils.iso_format()
path = "/home/airflow/data/community"
if not os.path.exists(path):
os.makedirs(path)
def build_doc(item):
"""
@@ -120,5 +124,5 @@ def sync(city=None):
actions.clear()
# 写入csv文件,防止小区变动导致的后续过程更新报错
seaweed_df.to_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8', index=False)
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
log.info(f"完成同步小区基本属性至206 {index_name}")
@@ -10,6 +10,7 @@ import traffic
import medical
import life
import relative_price_ranking
import consumption_potential
log = logging.getLogger(__name__)
@@ -28,3 +29,4 @@ if type == 'all':
medical.sync()
life.sync()
relative_price_ranking.sync()
consumption_potential.sync()
@@ -12,9 +12,9 @@ def calc_consumption_potential(city):
"""
计算消费潜力
"""
sql = "select id, city, region, name, total_price, estate_charge, avg_price_region_ranking, avg_cost from community_summary"
sql = "select id, city, region, name, total_price, estate_charge, relative_price_ranking, avg_cost from community_summary"
if pd.notna(city):
sql += " where city=" + city
sql += " where city='" + city + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.mean)
community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.std)
@@ -22,16 +22,26 @@ def calc_consumption_potential(city):
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(np.mean)
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(np.std)
community_summary_df['avg_price_region_ranking_mean'] = community_summary_df.groupby(by=['city'])['avg_price_region_ranking'].transform(np.mean)
community_summary_df['avg_price_region_ranking_std'] = community_summary_df.groupby(by=['city'])['avg_price_region_ranking'].transform(np.std)
community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])['relative_price_ranking'].transform(np.mean)
community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])['relative_price_ranking'].transform(np.std)
community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.mean)
community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.std)
community_summary_df['consumption_potential'] = 286.0134196 * (community_summary_df['total_price'] - community_summary_df['total_price_avg']) / community_summary_df['total_price_std'] \
+ 139.1374524 * (community_summary_df['estate_charge'] - community_summary_df['estate_charge_avg']) / community_summary_df['estate_charge_std'] \
+ 21.8427926 * (community_summary_df['avg_price_region_ranking'] - community_summary_df['avg_price_region_ranking_avg']) / community_summary_df['avg_price_region_ranking_std'] \
+ 67.0195871700 * (community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std'] + 4099.978248
community_summary_df['total_price_score'] = 286.0134196 * (community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df['total_price_std']
community_summary_df['total_price_score'].fillna(0, inplace=True)
community_summary_df['estate_charge_score'] = 139.1374524 * (community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std']
community_summary_df['estate_charge_score'].fillna(0, inplace=True)
community_summary_df['relative_price_ranking_score'] = 21.8427926 * (community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / community_summary_df['relative_price_ranking_std']
community_summary_df['relative_price_ranking_score'].fillna(0, inplace=True)
community_summary_df['avg_cost_score'] = 67.0195871700 * (community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std']
community_summary_df['avg_cost_score'].fillna(0, inplace=True)
community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df['estate_charge_score'] + community_summary_df['relative_price_ranking_score'] \
+ community_summary_df['avg_cost_score'] + community_summary_df['avg_cost_score'] + 4099.978248
pass
@@ -42,3 +52,7 @@ def calc_consumption_rating():
"""
pass
def sync(city=None):
calc_consumption_potential(city)
pass
+2 -1
View File
@@ -12,6 +12,7 @@ log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
count = 0
@@ -79,7 +80,7 @@ def sync(city=None):
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
+2 -1
View File
@@ -13,6 +13,7 @@ seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
count = 0
@@ -67,7 +68,7 @@ def sync(city=None):
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
+2 -1
View File
@@ -10,6 +10,7 @@ log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
@@ -42,7 +43,7 @@ def sync(city=None):
cities = [city]
else:
cities = None
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
# 小区所在区域均价
region_price_df = commutils.get_region_price(city=city)
@@ -13,6 +13,7 @@ log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
@@ -57,7 +58,7 @@ def build_doc(item):
def sync():
log.info("开始同步相对价格排名数据...")
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
def find_around_price_item(index, item):
if 'lng' not in item:
@@ -66,8 +67,11 @@ def sync():
lat = commutils.format_lat_lng(item['lat'])
if pd.notna(lng):
around_seaweed_df = find_around_seaweed(lng, lat)
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0]
if len(around_seaweed_df) > 1:
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0]
elif len(around_seaweed_df) == 1:
seaweed_df.loc[index, 'relative_price_ranking'] = 0.5
with ThreadPoolExecutor(max_workers=50) as pool:
for index, item in seaweed_df.iterrows():
+2 -1
View File
@@ -12,6 +12,7 @@ seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
@@ -28,7 +29,7 @@ def sync(city=None):
seaweed_query = {'cat': 'district', 'status': 0}
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
with ThreadPoolExecutor(50) as pool:
def deal(index, item):
@@ -13,6 +13,7 @@ seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
@@ -28,7 +29,7 @@ def sync(city=None):
seaweed_query = {'cat': 'district', 'status': 0}
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
# 加载库中均价数据
query_sql = "SELECT city, region, name, avg_price FROM community_summary "
+2 -1
View File
@@ -10,6 +10,7 @@ log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
@@ -28,7 +29,7 @@ def sync(city=None):
if pd.notna(city):
seaweed_query['city'] = city
cities = [city]
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
deal_price_df = commutils.get_deal_avg_price(cities, 3)
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
+2 -2
View File
@@ -12,6 +12,7 @@ log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
count = 0
@@ -76,8 +77,7 @@ def sync(city=None):
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@',
encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool: