From e079380c500578b876caa35fb37b3dc5226743b4 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 20 Feb 2023 18:29:04 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9Ecommunity=5Fsummary=E5=9F=8E?= =?UTF-8?q?=E5=B8=82=E4=BC=A0=E5=8F=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/avg_cost.py | 4 +++- .../business/community/business_district.py | 6 ++++-- dags/business/business/community/comm.py | 9 +++++++-- dags/business/business/community/consumption.py | 5 ++++- dags/business/business/community/education.py | 13 +++++++------ dags/business/business/community/education2.py | 6 ++++-- .../business/community/estate_charge_complete.py | 4 +++- .../business/community/estate_charge_ranking.py | 5 +++-- dags/business/business/community/extra.py | 4 +++- dags/business/business/community/life.py | 6 ++++-- dags/business/business/community/medical.py | 4 +++- dags/business/business/community/other.py | 5 +++-- dags/business/business/community/price.py | 4 +++- dags/business/business/community/price_rating.py | 4 +++- .../business/community/relative_price_ranking.py | 4 +++- dags/business/business/community/sell.py | 4 +++- dags/business/business/community/total_price.py | 4 +++- .../business/community/total_price_complete.py | 11 ++++++++--- dags/business/business/community/trade.py | 12 +++++++----- dags/business/business/community/traffic.py | 4 +++- 20 files changed, 81 insertions(+), 37 deletions(-) diff --git a/dags/business/business/community/avg_cost.py b/dags/business/business/community/avg_cost.py index 0b705c9..316d15f 100644 --- a/dags/business/business/community/avg_cost.py +++ b/dags/business/business/community/avg_cost.py @@ -7,6 +7,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils from elasticsearch import helpers import numpy as np from concurrent.futures import ThreadPoolExecutor +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -150,4 +151,5 @@ def sync(city=None): log.info("完成同步消费数据...") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/business_district.py b/dags/business/business/community/business_district.py index 0a21ad1..ccece89 100644 --- a/dags/business/business/community/business_district.py +++ b/dags/business/business/community/business_district.py @@ -3,10 +3,11 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils +from goojia_utils import ElasticSearchClient, commutils, dateutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -104,4 +105,5 @@ def sync(city=None): log.info(f"完成同步商圈至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index f08d67f..7134e3d 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -7,6 +7,7 @@ import pandas as pd from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils from elasticsearch import helpers import os +import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -93,7 +94,10 @@ def sync(city=None): 'estate_charge': 1, 'finish_building_date': 1}) .batch_size(100000))) - community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}") + sql = f"select id from {index_name} " + if pd.notna(city): + sql += f" from city={city}" + community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) seaweed_df['_id'] = seaweed_df['_id'].astype(str) if len(community_summary_df) > 0: @@ -154,4 +158,5 @@ def sync(city=None): log.info(f"完成同步小区基本属性至206 {index_name}") -sync() +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) diff --git a/dags/business/business/community/consumption.py b/dags/business/business/community/consumption.py index 11af5a5..f40b273 100644 --- a/dags/business/business/community/consumption.py +++ b/dags/business/business/community/consumption.py @@ -8,6 +8,7 @@ import logging import os from elasticsearch import helpers import matplotlib.pyplot as plt +import sys from sklearn.mixture import GaussianMixture from sklearn.preprocessing import QuantileTransformer @@ -249,5 +250,7 @@ def save_png(community_city): # 关窗口 plt.close() -sync() + +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) #dingutils.send_text("【分布图生成完毕】财力评级已完成, 请点击 https://nextcloud.fangjia.com/s/Jqs5BzXi8xLYeH9 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78") diff --git a/dags/business/business/community/education.py b/dags/business/business/community/education.py index 4af97d0..f00e718 100644 --- a/dags/business/business/community/education.py +++ b/dags/business/business/community/education.py @@ -3,10 +3,11 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -62,11 +63,10 @@ def deal(actions, item): def sync(city=None): log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}") - seaweed_query = {'status': 0, 'cat': 'district'} - if pd.notna(city): - seaweed_query['city'] = city - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + if pd.notna(city): + seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() + seaweed_df = seaweed_df.reset_index(drop=True) actions = list() with ThreadPoolExecutor(max_workers=30) as pool: @@ -83,4 +83,5 @@ def sync(city=None): log.info(f"完成同步教育资源至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/education2.py b/dags/business/business/community/education2.py index 3fd27ba..ec37083 100644 --- a/dags/business/business/community/education2.py +++ b/dags/business/business/community/education2.py @@ -3,10 +3,11 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -95,4 +96,5 @@ def sync(city=None): log.info(f"完成同步教育资源至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/estate_charge_complete.py b/dags/business/business/community/estate_charge_complete.py index 23c2460..becd95e 100644 --- a/dags/business/business/community/estate_charge_complete.py +++ b/dags/business/business/community/estate_charge_complete.py @@ -4,6 +4,7 @@ from goojia_utils import ElasticSearchClient, areautils, dateutils import pandas as pd import logging +import sys from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor @@ -140,4 +141,5 @@ def sync(city=None): sync_by_city(city) -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/estate_charge_ranking.py b/dags/business/business/community/estate_charge_ranking.py index db6489a..a5319af 100644 --- a/dags/business/business/community/estate_charge_ranking.py +++ b/dags/business/business/community/estate_charge_ranking.py @@ -5,6 +5,7 @@ from goojia_utils import ElasticSearchClient, nextcloudutils import pandas as pd import logging import os +import sys from elasticsearch import helpers import matplotlib.pyplot as plt @@ -163,6 +164,6 @@ def save_png(community_city): plt.close() - -sync() +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) #dingutils.send_text("【分布图生成完毕】物业费评级已完成, 请点击 https://nextcloud.fangjia.com/s/HbifLoEDc5wi8HR 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78") \ No newline at end of file diff --git a/dags/business/business/community/extra.py b/dags/business/business/community/extra.py index 4ddad24..7561eac 100644 --- a/dags/business/business/community/extra.py +++ b/dags/business/business/community/extra.py @@ -14,6 +14,7 @@ import pandas as pd from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -140,4 +141,5 @@ def sync(city=None): log.info(f"完成同步附加标签至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/life.py b/dags/business/business/community/life.py index 3c62f6f..0e62d00 100644 --- a/dags/business/business/community/life.py +++ b/dags/business/business/community/life.py @@ -3,10 +3,11 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -96,4 +97,5 @@ def sync(city=None): log.info(f"完成同步生活配套至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/medical.py b/dags/business/business/community/medical.py index 832a4bf..536b871 100644 --- a/dags/business/business/community/medical.py +++ b/dags/business/business/community/medical.py @@ -7,6 +7,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock +import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -83,4 +84,5 @@ def sync(city=None): log.info(f"完成同步医院至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/other.py b/dags/business/business/community/other.py index 0f02d66..6a67cff 100644 --- a/dags/business/business/community/other.py +++ b/dags/business/business/community/other.py @@ -5,7 +5,7 @@ from goojia_utils import ElasticSearchClient, dateutils import logging import pandas as pd from elasticsearch import helpers -import datetime +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -57,4 +57,5 @@ def sync(city=None): log.info(f"完成同步售租比、人口数至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index e148e92..7d967ad 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -5,6 +5,7 @@ import logging import pandas as pd from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils from elasticsearch import helpers +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -139,4 +140,5 @@ def sync(city=None): log.info(f"完成同步价格信息至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/price_rating.py b/dags/business/business/community/price_rating.py index 2860934..3020b87 100644 --- a/dags/business/business/community/price_rating.py +++ b/dags/business/business/community/price_rating.py @@ -8,6 +8,7 @@ from goojia_utils import ElasticSearchClient import pandas as pd import logging from elasticsearch import helpers +import sys from sklearn.mixture import GaussianMixture @@ -83,4 +84,5 @@ def sync(city=None): calc_price_rating(city) -sync() +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) diff --git a/dags/business/business/community/relative_price_ranking.py b/dags/business/business/community/relative_price_ranking.py index dd408df..2bc9117 100644 --- a/dags/business/business/community/relative_price_ranking.py +++ b/dags/business/business/community/relative_price_ranking.py @@ -7,6 +7,7 @@ import pandas as pd from goojia_utils import ElasticSearchClient, commutils, dateutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -112,4 +113,5 @@ def sync(city=None): log.info("完成同步相对价格排名数据...") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/sell.py b/dags/business/business/community/sell.py index 408cf31..79ca482 100644 --- a/dags/business/business/community/sell.py +++ b/dags/business/business/community/sell.py @@ -6,6 +6,7 @@ import pandas as pd from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor +import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -84,4 +85,5 @@ def sync(city=None): log.info(f"完成同步出售信息至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/total_price.py b/dags/business/business/community/total_price.py index 3b6f867..91082fb 100644 --- a/dags/business/business/community/total_price.py +++ b/dags/business/business/community/total_price.py @@ -7,6 +7,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor import re +import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -106,4 +107,5 @@ def sync(city=None): log.info(f"完成同步户均总价信息至206 {index_name}") -sync() +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) diff --git a/dags/business/business/community/total_price_complete.py b/dags/business/business/community/total_price_complete.py index 3a94f63..a2f07a6 100644 --- a/dags/business/business/community/total_price_complete.py +++ b/dags/business/business/community/total_price_complete.py @@ -5,6 +5,7 @@ import logging import pandas as pd from goojia_utils import ElasticSearchClient, dateutils from elasticsearch import helpers +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -101,10 +102,13 @@ def complete_by_region_area(community_factor_df): community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor'] -def sync(): +def sync(city=None): log.info(f"开始补全户均总价信息至206 {index_name}") - community_df = es206_client.scan_by_sql_to_data_frame("select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null") + sql = "select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null " + if pd.notna(city): + sql += " and city='" + city + "'" + community_df = es206_client.scan_by_sql_to_data_frame(sql) community_factor_df = complete_by_area(community_df) complete_by_build_year_area(community_factor_df) @@ -115,4 +119,5 @@ def sync(): log.info(f"完成补全户均总价信息至206 {index_name}") -sync() +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) diff --git a/dags/business/business/community/trade.py b/dags/business/business/community/trade.py index ad2aa5d..0694d6c 100644 --- a/dags/business/business/community/trade.py +++ b/dags/business/business/community/trade.py @@ -3,6 +3,7 @@ """ import logging import pandas as pd +import sys from goojia_utils import ElasticSearchClient, commutils, dateutils from elasticsearch import helpers @@ -24,12 +25,12 @@ def build_doc(item): def sync(city=None): log.info(f"开始同步成交信息至206 {index_name}") - seaweed_query = {'cat': 'district', 'status': 0} cities = None - if pd.notna(city): - seaweed_query['city'] = city - cities = [city] seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + if pd.notna(city): + cities = [city] + seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() + seaweed_df = seaweed_df.reset_index(drop=True) deal_price_df = commutils.get_deal_avg_price(cities, 3) deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']] @@ -68,4 +69,5 @@ def sync(city=None): log.info(f"完成同步成交信息至206 {index_name}") -sync() +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index dade577..34e90ad 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -7,6 +7,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -86,4 +87,5 @@ def sync(city=None): log.info(f"完成同步交通至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file