优化同步过程中小区变更问题

This commit is contained in:
shenjian@fangjia.com
2022-10-24 16:18:26 +08:00
parent 7ca680108e
commit f2650b287d
10 changed files with 38 additions and 58 deletions
+1 -5
View File
@@ -9,7 +9,6 @@ import numpy as np
from concurrent.futures import ThreadPoolExecutor
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
@@ -101,8 +100,7 @@ def sync():
# 按最新的百分位更新库中价格状态
update_status(new_df)
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'lng': 1, 'lat': 1}).batch_size(100000)))
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
def find_around_price_item(index, item):
if 'lng' not in item:
@@ -116,8 +114,6 @@ def sync():
for index, item in seaweed_df.iterrows():
pool.submit(find_around_price_item, index, item)
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
actions = list()
count = 0
for index, item in seaweed_df.iterrows():
+14 -9
View File
@@ -22,7 +22,7 @@ def build_doc(item):
:return:
"""
doc_dict = dict()
doc_dict['id'] = item["id"]
doc_dict['id'] = item["_id"]
doc_dict['m_date'] = m_date
doc_dict['city'] = item["city"]
doc_dict['region'] = item["region"]
@@ -52,23 +52,25 @@ def build_doc(item):
return doc_dict
def sync():
def sync(city=None):
log.info(f"开始同步小区基本属性至206 {index_name}...")
seaweed_query = {'status': 0, 'cat': 'district'}
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1,
'estate_type2': 1, 'build_type': 1,
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1,
'alias': 1, 'estate_type2': 1, 'build_type': 1,
'household_count': 1,
'estate_charge': 1, 'finish_building_date': 1})
.batch_size(100000)))
community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}")
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
seaweed_df = seaweed_df.rename(columns={"_id": "id"})
if len(community_summary_df) > 0:
community_summary_df['flag'] = 1
seaweed_new_df = pd.merge(seaweed_df, community_summary_df, how='left', on=['id'])
seaweed_new_df = pd.merge(seaweed_df, community_summary_df, how='left', on=['_id'])
seaweed_existed_df = seaweed_new_df[seaweed_new_df['flag'] == 1]
seaweed_not_existed_df = seaweed_new_df[seaweed_new_df['flag'] != 1]
else:
@@ -84,7 +86,7 @@ def sync():
action = {
"_op_type": "update",
"_index": index_name,
"_id": item['id'],
"_id": item['_id'],
"doc": doc_info
}
@@ -103,7 +105,7 @@ def sync():
action = {
"_op_type": "index",
"_index": index_name,
"_id": item['id'],
"_id": item['_id'],
"_source": doc_info
}
@@ -116,4 +118,7 @@ def sync():
if len(actions) > 0:
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
# 写入csv文件,防止小区变动导致的后续过程更新报错
seaweed_df.to_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8', index=False)
log.info(f"完成同步小区基本属性至206 {index_name}")
@@ -17,12 +17,12 @@ log.info("同步类型:" + type)
if type == 'all':
#comm.sync()
comm.sync('上海')
#price.sync()
#sell.sync()
#avg_cost.sync()
#trade.sync()
#total_price.sync()
total_price.sync()
#traffic.sync()
#medical.sync()
life.sync()
+1 -6
View File
@@ -9,7 +9,6 @@ from concurrent.futures import ThreadPoolExecutor
from threading import Lock
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
@@ -80,11 +79,7 @@ def sync(city=None):
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.DataFrame(
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1})
.batch_size(100000)))
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
+1 -5
View File
@@ -67,11 +67,7 @@ def sync(city=None):
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.DataFrame(
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1})
.batch_size(100000)))
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
+2 -7
View File
@@ -3,11 +3,10 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils
from goojia_utils import ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
@@ -43,9 +42,7 @@ def sync(city=None):
cities = [city]
else:
cities = None
seaweed_df = pd.DataFrame(
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1})
.batch_size(100000)))
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
# 小区所在区域均价
region_price_df = commutils.get_region_price(city=city)
@@ -89,8 +86,6 @@ def sync(city=None):
gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'})
seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name'])
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
actions = list()
count = 0
for index, item in seaweed_df.iterrows():
+2 -4
View File
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
@@ -28,8 +28,7 @@ def sync(city=None):
seaweed_query = {'cat': 'district', 'status': 0}
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.DataFrame(
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1}).batch_size(100000)))
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
with ThreadPoolExecutor(50) as pool:
def deal(index, item):
@@ -52,7 +51,6 @@ def sync(city=None):
for index, item in seaweed_df.iterrows():
pool.submit(deal, index, item)
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
actions = list()
count = 0
@@ -6,7 +6,7 @@ import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import datetime
import re
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -28,8 +28,7 @@ def sync(city=None):
seaweed_query = {'cat': 'district', 'status': 0}
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.DataFrame(
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1}).batch_size(100000)))
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
# 加载库中均价数据
query_sql = "SELECT city, region, name, avg_price FROM community_summary "
@@ -53,9 +52,12 @@ def sync(city=None):
curr_region = item['region']
curr_name = item['name']
names = [curr_name]
if type(item['alias']) == list:
for alias_item in item['alias']:
names.append(alias_item)
if pd.notna(item['alias']) and '[]' != item['alias']:
alias_list = re.sub("\[|\]|'", '', item['alias']).split(',')
if len(alias_list) > 0:
for alias_item in alias_list:
names.append(alias_item)
# 户均总价
if (curr_city + curr_region + curr_name) in avg_price_map:
@@ -63,13 +65,13 @@ def sync(city=None):
if pd.notna(avg_price):
min_avg_price = 0.8 * avg_price
max_avg_rice = 1.2 * avg_price
total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=93, min_avg_price=min_avg_price, max_avg_price=max_avg_rice)
total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366, min_avg_price=min_avg_price, max_avg_price=max_avg_rice)
seaweed_df.loc[index, 'total_price'] = total_price
for index, item in seaweed_df.iterrows():
pool.submit(deal, index, item)
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
actions = list()
count = 0
+2 -5
View File
@@ -3,11 +3,10 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils
from goojia_utils import ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
@@ -29,9 +28,7 @@ def sync(city=None):
if pd.notna(city):
seaweed_query['city'] = city
cities = [city]
seaweed_df = pd.DataFrame(
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1}).batch_size(100000)))
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
deal_price_df = commutils.get_deal_avg_price(cities, 3)
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
+3 -7
View File
@@ -3,13 +3,12 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
@@ -77,11 +76,8 @@ def sync(city=None):
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.DataFrame(
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1})
.batch_size(100000)))
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
seaweed_df = seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@',
encoding='UTF-8')
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool: