优化中间表

This commit is contained in:
shenjian@fangjia.com
2023-03-17 11:26:43 +08:00
parent fe592e20ac
commit b8e8d79821
12 changed files with 87 additions and 77 deletions
+7 -6
View File
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, ClickHouseClient
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import sys
@@ -12,6 +12,7 @@ from threading import Lock
mutex = Lock()
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
m_date = dateutils.iso_format()
@@ -111,10 +112,10 @@ def find_around_price_item(seaweed):
def sync(city=None):
log.info("开始同步消费数据...")
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
seaweed_list = list()
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
@@ -153,5 +154,5 @@ def sync(city=None):
log.info("完成同步消费数据...")
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
@@ -11,6 +11,7 @@ import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database="tmp")
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -89,10 +90,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步商圈至206 {index_name}")
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -105,5 +106,5 @@ def sync(city=None):
log.info(f"完成同步商圈至206 {index_name}")
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
+4 -4
View File
@@ -1,10 +1,10 @@
"""
同步小区基本信息至ES206
"""
# import sys
# sys.path.insert(0, '/opt/airflow/dags/repo')
# import dags.install as install
# install.install()
import sys
sys.path.insert(0, '/opt/airflow/dags/repo')
import dags.install as install
install.install()
import logging
import pandas as pd
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
@@ -11,6 +11,7 @@ import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database="tmp")
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -63,10 +64,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}")
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -83,5 +84,5 @@ def sync(city=None):
log.info(f"完成同步教育资源至206 {index_name}")
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
@@ -11,6 +11,7 @@ import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database="tmp")
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -76,10 +77,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步教育资源【小学、中学】至206 {index_name}")
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -96,5 +97,5 @@ def sync(city=None):
log.info(f"完成同步教育资源至206 {index_name}")
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
+7 -6
View File
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
@@ -11,6 +11,7 @@ import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -77,10 +78,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步生活配套至206 {index_name}")
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -97,5 +98,5 @@ def sync(city=None):
log.info(f"完成同步生活配套至206 {index_name}")
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
+7 -7
View File
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient, poiutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
@@ -12,6 +12,7 @@ import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -63,11 +64,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步医院至206 {index_name}")
seaweed_query = {'status': 0, 'cat': 'district'}
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -84,5 +84,5 @@ def sync(city=None):
log.info(f"完成同步医院至206 {index_name}")
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
+16 -16
View File
@@ -3,12 +3,13 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, ClickHouseClient
from elasticsearch import helpers
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -38,11 +39,10 @@ def build_doc(item):
return doc_dict
def sync_by_city(city, total_seaweed_df):
seaweed_df = total_seaweed_df[total_seaweed_df['city'] == city].copy()
def sync_by_city(city):
seaweed_df = tmp_ck.read_sql_query(f"select * from community_summary where city='{city}'")
if len(seaweed_df) == 0:
return
seaweed_df = seaweed_df.reset_index(drop=True)
# 小区所在城市均价
city_price_df = commutils.get_city_price(city=city)
@@ -67,10 +67,11 @@ def sync_by_city(city, total_seaweed_df):
# 小区均价区域排名
seaweed_df['avg_price_region_ranking'] = \
seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)[
'avg_price']
# 小区均价城市排名
seaweed_df['avg_price_ranking'] = \
seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
# 12个月均价涨跌幅
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-12)
@@ -125,20 +126,19 @@ def sync_by_city(city, total_seaweed_df):
actions.clear()
def sync(city=None):
def sync(curr_city=None):
log.info(f"开始同步价格信息至206 {index_name}")
if pd.notna(city):
cities = [city]
if pd.notna(curr_city):
cities = [curr_city]
else:
cities = areautils.get_all_cities()
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
for city in cities:
log.info(f"开始同步{city}")
sync_by_city(city, seaweed_df)
log.info(f"完成同步{city}")
for city_now in cities:
log.info(f"开始同步{city_now}")
sync_by_city(city_now)
log.info(f"完成同步{city_now}")
log.info(f"完成同步价格信息至206 {index_name}")
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
@@ -4,7 +4,7 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import sys
@@ -12,6 +12,7 @@ from threading import Lock
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -92,10 +93,10 @@ def find_around_price_item(seaweed):
def sync(city=None):
log.info("开始同步相对价格排名数据...")
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
seaweed_list = list()
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
@@ -138,5 +139,5 @@ def sync(city=None):
log.info("完成同步相对价格排名数据...")
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
+5 -4
View File
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import sys
@@ -12,6 +12,7 @@ from threading import Lock
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -32,10 +33,10 @@ def build_doc(item):
def sync(city=None):
log.info(f"开始同步出售信息至206 {index_name}")
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
with ThreadPoolExecutor(30) as pool:
def deal(index, item):
+6 -4
View File
@@ -4,11 +4,12 @@
import logging
import pandas as pd
import sys
from goojia_utils import ElasticSearchClient, commutils, dateutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
from elasticsearch import helpers
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -26,11 +27,12 @@ def sync(city=None):
log.info(f"开始同步成交信息至206 {index_name}")
cities = None
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql = f"select * from community_summary "
if pd.notna(city):
cities = [city]
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
deal_price_df = commutils.get_deal_avg_price(cities, 3)
if len(deal_price_df) <= 0:
+7 -6
View File
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
@@ -11,6 +11,7 @@ import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -71,10 +72,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步交通至206 {index_name}")
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -87,5 +88,5 @@ def sync(city=None):
log.info(f"完成同步交通至206 {index_name}")
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)