新增community_summary城市传参

This commit is contained in:
shenjian@fangjia.com
2023-02-20 18:29:04 +08:00
parent 8a49c13e45
commit e079380c50
20 changed files with 81 additions and 37 deletions
+3 -1
View File
@@ -7,6 +7,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
from elasticsearch import helpers
import numpy as np
from concurrent.futures import ThreadPoolExecutor
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -150,4 +151,5 @@ def sync(city=None):
log.info("完成同步消费数据...")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -3,10 +3,11 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -104,4 +105,5 @@ def sync(city=None):
log.info(f"完成同步商圈至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+7 -2
View File
@@ -7,6 +7,7 @@ import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
import os
import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -93,7 +94,10 @@ def sync(city=None):
'estate_charge': 1, 'finish_building_date': 1})
.batch_size(100000)))
community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}")
sql = f"select id from {index_name} "
if pd.notna(city):
sql += f" from city={city}"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
if len(community_summary_df) > 0:
@@ -154,4 +158,5 @@ def sync(city=None):
log.info(f"完成同步小区基本属性至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -8,6 +8,7 @@ import logging
import os
from elasticsearch import helpers
import matplotlib.pyplot as plt
import sys
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import QuantileTransformer
@@ -249,5 +250,7 @@ def save_png(community_city):
# 关窗口
plt.close()
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
#dingutils.send_text("【分布图生成完毕】财力评级已完成, 请点击 https://nextcloud.fangjia.com/s/Jqs5BzXi8xLYeH9 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
@@ -3,10 +3,11 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -62,11 +63,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}")
seaweed_query = {'status': 0, 'cat': 'district'}
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -83,4 +83,5 @@ def sync(city=None):
log.info(f"完成同步教育资源至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -3,10 +3,11 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -95,4 +96,5 @@ def sync(city=None):
log.info(f"完成同步教育资源至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -4,6 +4,7 @@
from goojia_utils import ElasticSearchClient, areautils, dateutils
import pandas as pd
import logging
import sys
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
@@ -140,4 +141,5 @@ def sync(city=None):
sync_by_city(city)
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -5,6 +5,7 @@ from goojia_utils import ElasticSearchClient, nextcloudutils
import pandas as pd
import logging
import os
import sys
from elasticsearch import helpers
import matplotlib.pyplot as plt
@@ -163,6 +164,6 @@ def save_png(community_city):
plt.close()
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
#dingutils.send_text("【分布图生成完毕】物业费评级已完成, 请点击 https://nextcloud.fangjia.com/s/HbifLoEDc5wi8HR 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
+3 -1
View File
@@ -14,6 +14,7 @@ import pandas as pd
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -140,4 +141,5 @@ def sync(city=None):
log.info(f"完成同步附加标签至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+4 -2
View File
@@ -3,10 +3,11 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -96,4 +97,5 @@ def sync(city=None):
log.info(f"完成同步生活配套至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -1
View File
@@ -7,6 +7,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -83,4 +84,5 @@ def sync(city=None):
log.info(f"完成同步医院至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -2
View File
@@ -5,7 +5,7 @@ from goojia_utils import ElasticSearchClient, dateutils
import logging
import pandas as pd
from elasticsearch import helpers
import datetime
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -57,4 +57,5 @@ def sync(city=None):
log.info(f"完成同步售租比、人口数至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -1
View File
@@ -5,6 +5,7 @@ import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
from elasticsearch import helpers
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -139,4 +140,5 @@ def sync(city=None):
log.info(f"完成同步价格信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -8,6 +8,7 @@ from goojia_utils import ElasticSearchClient
import pandas as pd
import logging
from elasticsearch import helpers
import sys
from sklearn.mixture import GaussianMixture
@@ -83,4 +84,5 @@ def sync(city=None):
calc_price_rating(city)
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -7,6 +7,7 @@ import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -112,4 +113,5 @@ def sync(city=None):
log.info("完成同步相对价格排名数据...")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -1
View File
@@ -6,6 +6,7 @@ import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -84,4 +85,5 @@ def sync(city=None):
log.info(f"完成同步出售信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -7,6 +7,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import re
import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -106,4 +107,5 @@ def sync(city=None):
log.info(f"完成同步户均总价信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -5,6 +5,7 @@ import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, dateutils
from elasticsearch import helpers
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -101,10 +102,13 @@ def complete_by_region_area(community_factor_df):
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
def sync():
def sync(city=None):
log.info(f"开始补全户均总价信息至206 {index_name}")
community_df = es206_client.scan_by_sql_to_data_frame("select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null")
sql = "select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null "
if pd.notna(city):
sql += " and city='" + city + "'"
community_df = es206_client.scan_by_sql_to_data_frame(sql)
community_factor_df = complete_by_area(community_df)
complete_by_build_year_area(community_factor_df)
@@ -115,4 +119,5 @@ def sync():
log.info(f"完成补全户均总价信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+7 -5
View File
@@ -3,6 +3,7 @@
"""
import logging
import pandas as pd
import sys
from goojia_utils import ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
@@ -24,12 +25,12 @@ def build_doc(item):
def sync(city=None):
log.info(f"开始同步成交信息至206 {index_name}")
seaweed_query = {'cat': 'district', 'status': 0}
cities = None
if pd.notna(city):
seaweed_query['city'] = city
cities = [city]
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
if pd.notna(city):
cities = [city]
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
deal_price_df = commutils.get_deal_avg_price(cities, 3)
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
@@ -68,4 +69,5 @@ def sync(city=None):
log.info(f"完成同步成交信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -1
View File
@@ -7,6 +7,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -86,4 +87,5 @@ def sync(city=None):
log.info(f"完成同步交通至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)