优化中间表
This commit is contained in:
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
@@ -12,6 +12,7 @@ from threading import Lock
|
||||
mutex = Lock()
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
m_date = dateutils.iso_format()
|
||||
@@ -111,10 +112,10 @@ def find_around_price_item(seaweed):
|
||||
|
||||
def sync(city=None):
|
||||
log.info("开始同步消费数据...")
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
seaweed_list = list()
|
||||
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
|
||||
@@ -153,5 +154,5 @@ def sync(city=None):
|
||||
log.info("完成同步消费数据...")
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
@@ -11,6 +11,7 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database="tmp")
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -89,10 +90,10 @@ def deal(actions, item):
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步商圈至206 {index_name}")
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
@@ -105,5 +106,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步商圈至206 {index_name}")
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
@@ -1,10 +1,10 @@
|
||||
"""
|
||||
同步小区基本信息至ES206
|
||||
"""
|
||||
# import sys
|
||||
# sys.path.insert(0, '/opt/airflow/dags/repo')
|
||||
# import dags.install as install
|
||||
# install.install()
|
||||
import sys
|
||||
sys.path.insert(0, '/opt/airflow/dags/repo')
|
||||
import dags.install as install
|
||||
install.install()
|
||||
|
||||
import logging
|
||||
import pandas as pd
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
@@ -11,6 +11,7 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database="tmp")
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -63,10 +64,10 @@ def deal(actions, item):
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}")
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
@@ -83,5 +84,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步教育资源至206 {index_name}")
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
@@ -11,6 +11,7 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database="tmp")
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -76,10 +77,10 @@ def deal(actions, item):
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步教育资源【小学、中学】至206 {index_name}")
|
||||
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
@@ -96,5 +97,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步教育资源至206 {index_name}")
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
@@ -11,6 +11,7 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -77,10 +78,10 @@ def deal(actions, item):
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步生活配套至206 {index_name}")
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
@@ -97,5 +98,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步生活配套至206 {index_name}")
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient, poiutils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
@@ -12,6 +12,7 @@ import sys
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -63,11 +64,10 @@ def deal(actions, item):
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步医院至206 {index_name}")
|
||||
seaweed_query = {'status': 0, 'cat': 'district'}
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
@@ -84,5 +84,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步医院至206 {index_name}")
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
@@ -3,12 +3,13 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -38,11 +39,10 @@ def build_doc(item):
|
||||
return doc_dict
|
||||
|
||||
|
||||
def sync_by_city(city, total_seaweed_df):
|
||||
seaweed_df = total_seaweed_df[total_seaweed_df['city'] == city].copy()
|
||||
def sync_by_city(city):
|
||||
seaweed_df = tmp_ck.read_sql_query(f"select * from community_summary where city='{city}'")
|
||||
if len(seaweed_df) == 0:
|
||||
return
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
|
||||
# 小区所在城市均价
|
||||
city_price_df = commutils.get_city_price(city=city)
|
||||
@@ -67,10 +67,11 @@ def sync_by_city(city, total_seaweed_df):
|
||||
|
||||
# 小区均价区域排名
|
||||
seaweed_df['avg_price_region_ranking'] = \
|
||||
seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
|
||||
seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)[
|
||||
'avg_price']
|
||||
# 小区均价城市排名
|
||||
seaweed_df['avg_price_ranking'] = \
|
||||
seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
|
||||
seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
|
||||
|
||||
# 12个月均价涨跌幅
|
||||
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-12)
|
||||
@@ -125,20 +126,19 @@ def sync_by_city(city, total_seaweed_df):
|
||||
actions.clear()
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
def sync(curr_city=None):
|
||||
log.info(f"开始同步价格信息至206 {index_name}")
|
||||
if pd.notna(city):
|
||||
cities = [city]
|
||||
if pd.notna(curr_city):
|
||||
cities = [curr_city]
|
||||
else:
|
||||
cities = areautils.get_all_cities()
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
for city in cities:
|
||||
log.info(f"开始同步{city}")
|
||||
sync_by_city(city, seaweed_df)
|
||||
log.info(f"完成同步{city}")
|
||||
for city_now in cities:
|
||||
log.info(f"开始同步{city_now}")
|
||||
sync_by_city(city_now)
|
||||
log.info(f"完成同步{city_now}")
|
||||
|
||||
log.info(f"完成同步价格信息至206 {index_name}")
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
@@ -12,6 +12,7 @@ from threading import Lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -92,10 +93,10 @@ def find_around_price_item(seaweed):
|
||||
def sync(city=None):
|
||||
log.info("开始同步相对价格排名数据...")
|
||||
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
seaweed_list = list()
|
||||
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
|
||||
@@ -138,5 +139,5 @@ def sync(city=None):
|
||||
log.info("完成同步相对价格排名数据...")
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
@@ -12,6 +12,7 @@ from threading import Lock
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -32,10 +33,10 @@ def build_doc(item):
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步出售信息至206 {index_name}")
|
||||
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
with ThreadPoolExecutor(30) as pool:
|
||||
def deal(index, item):
|
||||
|
||||
@@ -4,11 +4,12 @@
|
||||
import logging
|
||||
import pandas as pd
|
||||
import sys
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -26,11 +27,12 @@ def sync(city=None):
|
||||
log.info(f"开始同步成交信息至206 {index_name}")
|
||||
|
||||
cities = None
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
cities = [city]
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
deal_price_df = commutils.get_deal_avg_price(cities, 3)
|
||||
if len(deal_price_df) <= 0:
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
@@ -11,6 +11,7 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -71,10 +72,10 @@ def deal(actions, item):
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步交通至206 {index_name}")
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
@@ -87,5 +88,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步交通至206 {index_name}")
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
Reference in New Issue
Block a user