修复文件读取bug

This commit is contained in:
shenjian@fangjia.com
2022-10-25 09:33:16 +08:00
parent 21e5c5fdac
commit 212fb10edb
11 changed files with 24 additions and 12 deletions
+3 -2
View File
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from goojia_utils import ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
import numpy as np
from concurrent.futures import ThreadPoolExecutor
@@ -13,6 +13,7 @@ es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
m_date = dateutils.iso_format()
path = "/home/airflow/data/community"
def find_around_price(lng, lat):
@@ -100,7 +101,7 @@ def sync():
# 按最新的百分位更新库中价格状态
update_status(new_df)
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
def find_around_price_item(index, item):
if 'lng' not in item:
+5 -1
View File
@@ -6,6 +6,7 @@ import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
import os
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -14,6 +15,9 @@ es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
m_date = dateutils.iso_format()
path = "/home/airflow/data/community"
if not os.path.exists(path):
os.makedirs(path)
def build_doc(item):
"""
@@ -120,5 +124,5 @@ def sync(city=None):
actions.clear()
# 写入csv文件,防止小区变动导致的后续过程更新报错
seaweed_df.to_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8', index=False)
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
log.info(f"完成同步小区基本属性至206 {index_name}")
+2 -1
View File
@@ -12,6 +12,7 @@ log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
count = 0
@@ -79,7 +80,7 @@ def sync(city=None):
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
+2 -1
View File
@@ -13,6 +13,7 @@ seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
count = 0
@@ -67,7 +68,7 @@ def sync(city=None):
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
+2 -1
View File
@@ -10,6 +10,7 @@ log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
@@ -42,7 +43,7 @@ def sync(city=None):
cities = [city]
else:
cities = None
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
# 小区所在区域均价
region_price_df = commutils.get_region_price(city=city)
@@ -13,6 +13,7 @@ log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
@@ -57,7 +58,7 @@ def build_doc(item):
def sync():
log.info("开始同步相对价格排名数据...")
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
def find_around_price_item(index, item):
if 'lng' not in item:
+2 -1
View File
@@ -12,6 +12,7 @@ seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
@@ -28,7 +29,7 @@ def sync(city=None):
seaweed_query = {'cat': 'district', 'status': 0}
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
with ThreadPoolExecutor(50) as pool:
def deal(index, item):
@@ -13,6 +13,7 @@ seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
@@ -28,7 +29,7 @@ def sync(city=None):
seaweed_query = {'cat': 'district', 'status': 0}
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
# 加载库中均价数据
query_sql = "SELECT city, region, name, avg_price FROM community_summary "
+2 -1
View File
@@ -10,6 +10,7 @@ log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
@@ -28,7 +29,7 @@ def sync(city=None):
if pd.notna(city):
seaweed_query['city'] = city
cities = [city]
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
deal_price_df = commutils.get_deal_avg_price(cities, 3)
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
+2 -2
View File
@@ -12,6 +12,7 @@ log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
count = 0
@@ -76,8 +77,7 @@ def sync(city=None):
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@',
encoding='UTF-8')
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool: