优化同步过程中小区变更问题
This commit is contained in:
@@ -9,7 +9,6 @@ import numpy as np
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
@@ -101,8 +100,7 @@ def sync():
|
||||
# 按最新的百分位更新库中价格状态
|
||||
update_status(new_df)
|
||||
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'lng': 1, 'lat': 1}).batch_size(100000)))
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
def find_around_price_item(index, item):
|
||||
if 'lng' not in item:
|
||||
@@ -116,8 +114,6 @@ def sync():
|
||||
for index, item in seaweed_df.iterrows():
|
||||
pool.submit(find_around_price_item, index, item)
|
||||
|
||||
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in seaweed_df.iterrows():
|
||||
|
||||
@@ -22,7 +22,7 @@ def build_doc(item):
|
||||
:return:
|
||||
"""
|
||||
doc_dict = dict()
|
||||
doc_dict['id'] = item["id"]
|
||||
doc_dict['id'] = item["_id"]
|
||||
doc_dict['m_date'] = m_date
|
||||
doc_dict['city'] = item["city"]
|
||||
doc_dict['region'] = item["region"]
|
||||
@@ -52,23 +52,25 @@ def build_doc(item):
|
||||
return doc_dict
|
||||
|
||||
|
||||
def sync():
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步小区基本属性至206 {index_name}...")
|
||||
seaweed_query = {'status': 0, 'cat': 'district'}
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1,
|
||||
'estate_type2': 1, 'build_type': 1,
|
||||
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1,
|
||||
'alias': 1, 'estate_type2': 1, 'build_type': 1,
|
||||
'household_count': 1,
|
||||
'estate_charge': 1, 'finish_building_date': 1})
|
||||
.batch_size(100000)))
|
||||
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}")
|
||||
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
||||
seaweed_df = seaweed_df.rename(columns={"_id": "id"})
|
||||
|
||||
if len(community_summary_df) > 0:
|
||||
community_summary_df['flag'] = 1
|
||||
|
||||
seaweed_new_df = pd.merge(seaweed_df, community_summary_df, how='left', on=['id'])
|
||||
seaweed_new_df = pd.merge(seaweed_df, community_summary_df, how='left', on=['_id'])
|
||||
seaweed_existed_df = seaweed_new_df[seaweed_new_df['flag'] == 1]
|
||||
seaweed_not_existed_df = seaweed_new_df[seaweed_new_df['flag'] != 1]
|
||||
else:
|
||||
@@ -84,7 +86,7 @@ def sync():
|
||||
action = {
|
||||
"_op_type": "update",
|
||||
"_index": index_name,
|
||||
"_id": item['id'],
|
||||
"_id": item['_id'],
|
||||
"doc": doc_info
|
||||
}
|
||||
|
||||
@@ -103,7 +105,7 @@ def sync():
|
||||
action = {
|
||||
"_op_type": "index",
|
||||
"_index": index_name,
|
||||
"_id": item['id'],
|
||||
"_id": item['_id'],
|
||||
"_source": doc_info
|
||||
}
|
||||
|
||||
@@ -116,4 +118,7 @@ def sync():
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
|
||||
# 写入csv文件,防止小区变动导致的后续过程更新报错
|
||||
seaweed_df.to_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||
log.info(f"完成同步小区基本属性至206 {index_name}")
|
||||
|
||||
@@ -17,12 +17,12 @@ log.info("同步类型:" + type)
|
||||
|
||||
|
||||
if type == 'all':
|
||||
#comm.sync()
|
||||
comm.sync('上海')
|
||||
#price.sync()
|
||||
#sell.sync()
|
||||
#avg_cost.sync()
|
||||
#trade.sync()
|
||||
#total_price.sync()
|
||||
total_price.sync()
|
||||
#traffic.sync()
|
||||
#medical.sync()
|
||||
life.sync()
|
||||
|
||||
@@ -9,7 +9,6 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
@@ -80,11 +79,7 @@ def sync(city=None):
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1})
|
||||
.batch_size(100000)))
|
||||
|
||||
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
|
||||
@@ -67,11 +67,7 @@ def sync(city=None):
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1})
|
||||
.batch_size(100000)))
|
||||
|
||||
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
|
||||
@@ -3,11 +3,10 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||
from elasticsearch import helpers
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
@@ -43,9 +42,7 @@ def sync(city=None):
|
||||
cities = [city]
|
||||
else:
|
||||
cities = None
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1})
|
||||
.batch_size(100000)))
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
# 小区所在区域均价
|
||||
region_price_df = commutils.get_region_price(city=city)
|
||||
@@ -89,8 +86,6 @@ def sync(city=None):
|
||||
gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'})
|
||||
seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name'])
|
||||
|
||||
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in seaweed_df.iterrows():
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
@@ -28,8 +28,7 @@ def sync(city=None):
|
||||
seaweed_query = {'cat': 'district', 'status': 0}
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1}).batch_size(100000)))
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
with ThreadPoolExecutor(50) as pool:
|
||||
def deal(index, item):
|
||||
@@ -52,7 +51,6 @@ def sync(city=None):
|
||||
for index, item in seaweed_df.iterrows():
|
||||
pool.submit(deal, index, item)
|
||||
|
||||
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
@@ -6,7 +6,7 @@ import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import datetime
|
||||
import re
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
@@ -28,8 +28,7 @@ def sync(city=None):
|
||||
seaweed_query = {'cat': 'district', 'status': 0}
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1}).batch_size(100000)))
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
# 加载库中均价数据
|
||||
query_sql = "SELECT city, region, name, avg_price FROM community_summary "
|
||||
@@ -53,9 +52,12 @@ def sync(city=None):
|
||||
curr_region = item['region']
|
||||
curr_name = item['name']
|
||||
names = [curr_name]
|
||||
if type(item['alias']) == list:
|
||||
for alias_item in item['alias']:
|
||||
names.append(alias_item)
|
||||
|
||||
if pd.notna(item['alias']) and '[]' != item['alias']:
|
||||
alias_list = re.sub("\[|\]|'", '', item['alias']).split(',')
|
||||
if len(alias_list) > 0:
|
||||
for alias_item in alias_list:
|
||||
names.append(alias_item)
|
||||
|
||||
# 户均总价
|
||||
if (curr_city + curr_region + curr_name) in avg_price_map:
|
||||
@@ -63,13 +65,13 @@ def sync(city=None):
|
||||
if pd.notna(avg_price):
|
||||
min_avg_price = 0.8 * avg_price
|
||||
max_avg_rice = 1.2 * avg_price
|
||||
total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=93, min_avg_price=min_avg_price, max_avg_price=max_avg_rice)
|
||||
total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366, min_avg_price=min_avg_price, max_avg_price=max_avg_rice)
|
||||
seaweed_df.loc[index, 'total_price'] = total_price
|
||||
|
||||
for index, item in seaweed_df.iterrows():
|
||||
pool.submit(deal, index, item)
|
||||
|
||||
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
||||
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
@@ -3,11 +3,10 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||
from elasticsearch import helpers
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
@@ -29,9 +28,7 @@ def sync(city=None):
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
cities = [city]
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1}).batch_size(100000)))
|
||||
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
deal_price_df = commutils.get_deal_avg_price(cities, 3)
|
||||
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
|
||||
|
||||
@@ -3,13 +3,12 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
@@ -77,11 +76,8 @@ def sync(city=None):
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1})
|
||||
.batch_size(100000)))
|
||||
|
||||
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
||||
seaweed_df = seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@',
|
||||
encoding='UTF-8')
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
|
||||
Reference in New Issue
Block a user