加锁
This commit is contained in:
@@ -5,13 +5,12 @@ import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
|
||||
from elasticsearch import helpers
|
||||
import numpy as np
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
import threading
|
||||
from threading import Lock
|
||||
|
||||
# 创建线程锁
|
||||
lock = threading.Lock()
|
||||
mutex = Lock()
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
@@ -71,12 +70,12 @@ def find_around_price_item(seaweed_df, index, item):
|
||||
lat = commutils.format_lat_lng(item['lat'])
|
||||
city_now = areautils.get_city_display(item['city'])
|
||||
price = find_around_price(city_now, lng, lat)
|
||||
lock.acquire()
|
||||
mutex.acquire(timeout=60)
|
||||
if pd.notna(price):
|
||||
seaweed_df.loc[index, 'avg_cost'] = int(price)
|
||||
else:
|
||||
seaweed_df.loc[index, 'avg_cost'] = None
|
||||
lock.release()
|
||||
mutex.release()
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
|
||||
@@ -7,12 +7,14 @@ import logging
|
||||
import sys
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
index_name = "community_summary"
|
||||
|
||||
mutex = Lock()
|
||||
|
||||
def save(community_summary_df):
|
||||
if 'estate_charge' not in community_summary_df.columns:
|
||||
@@ -127,7 +129,9 @@ def sync_by_city(city_now):
|
||||
# 取周边均值
|
||||
estate_charge = docs['estate_charge'].mean()
|
||||
if pd.notna(estate_charge):
|
||||
mutex.acquire(timeout=60)
|
||||
community_summary_df.loc[index, "estate_charge"] = estate_charge
|
||||
mutex.release()
|
||||
for index, item in community_summary_df.iterrows():
|
||||
pool.submit(deal, index, item)
|
||||
save(community_summary_df)
|
||||
|
||||
@@ -8,6 +8,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
from threading import Lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
@@ -15,7 +16,7 @@ es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
m_date = dateutils.iso_format()
|
||||
|
||||
mutex = Lock()
|
||||
|
||||
def find_around_seaweed(lng, lat):
|
||||
"""
|
||||
@@ -72,11 +73,13 @@ def sync(city=None):
|
||||
lat = commutils.format_lat_lng(item['lat'])
|
||||
if pd.notna(lng):
|
||||
around_seaweed_df = find_around_seaweed(lng, lat)
|
||||
mutex.acquire(timeout=60)
|
||||
if len(around_seaweed_df) > 1:
|
||||
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
|
||||
seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0]
|
||||
elif len(around_seaweed_df) == 1:
|
||||
seaweed_df.loc[index, 'relative_price_ranking'] = 0.5
|
||||
mutex.release()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=20) as pool:
|
||||
for index, item in seaweed_df.iterrows():
|
||||
|
||||
@@ -7,6 +7,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
from threading import Lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
@@ -15,6 +16,7 @@ es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
m_date = dateutils.iso_format()
|
||||
mutex = Lock()
|
||||
|
||||
|
||||
def build_doc(item):
|
||||
@@ -51,11 +53,14 @@ def sync(city=None):
|
||||
|
||||
# 近三个月挂牌量
|
||||
sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=93)
|
||||
seaweed_df.loc[index, 'three_months_sell_num'] = sample_num
|
||||
|
||||
# 近半年挂牌量
|
||||
sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=186)
|
||||
seaweed_df.loc[index, 'half_on_year_sell_num'] = sample_num
|
||||
sample_half_year_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=186)
|
||||
|
||||
mutex.acquire()
|
||||
seaweed_df.loc[index, 'three_months_sell_num'] = sample_num
|
||||
seaweed_df.loc[index, 'half_on_year_sell_num'] = sample_half_year_num
|
||||
mutex.release()
|
||||
|
||||
for index, item in seaweed_df.iterrows():
|
||||
pool.submit(deal, index, item)
|
||||
|
||||
@@ -96,4 +96,4 @@ with DAG(
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
# 人均消费并行,值为空
|
||||
comm >> price >> avg_cost >> [total_price, relative_price_ranking, estate_charge_complete] >> consumption >> [estate_charge_ranking, price_rating, person_rating] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other
|
||||
comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> consumption >> [estate_charge_ranking, price_rating, person_rating] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other
|
||||
|
||||
Reference in New Issue
Block a user