Merge remote-tracking branch 'origin/master'
This commit is contained in:
@@ -7,6 +7,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
|
|||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||||
@@ -150,4 +151,5 @@ def sync(city=None):
|
|||||||
log.info("完成同步消费数据...")
|
log.info("完成同步消费数据...")
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
@@ -3,10 +3,11 @@
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from threading import Lock
|
from threading import Lock
|
||||||
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||||
@@ -104,4 +105,5 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步商圈至206 {index_name}")
|
log.info(f"完成同步商圈至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
@@ -7,6 +7,7 @@ import pandas as pd
|
|||||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
import os
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||||
@@ -93,7 +94,10 @@ def sync(city=None):
|
|||||||
'estate_charge': 1, 'finish_building_date': 1})
|
'estate_charge': 1, 'finish_building_date': 1})
|
||||||
.batch_size(100000)))
|
.batch_size(100000)))
|
||||||
|
|
||||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}")
|
sql = f"select id from {index_name} "
|
||||||
|
if pd.notna(city):
|
||||||
|
sql += f" where city='{city}'"
|
||||||
|
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||||
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
||||||
|
|
||||||
if len(community_summary_df) > 0:
|
if len(community_summary_df) > 0:
|
||||||
@@ -154,4 +158,5 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步小区基本属性至206 {index_name}")
|
log.info(f"完成同步小区基本属性至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import logging
|
|||||||
import os
|
import os
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
|
import sys
|
||||||
|
|
||||||
from sklearn.mixture import GaussianMixture
|
from sklearn.mixture import GaussianMixture
|
||||||
from sklearn.preprocessing import QuantileTransformer
|
from sklearn.preprocessing import QuantileTransformer
|
||||||
@@ -249,5 +250,7 @@ def save_png(community_city):
|
|||||||
# 关窗口
|
# 关窗口
|
||||||
plt.close()
|
plt.close()
|
||||||
|
|
||||||
sync()
|
|
||||||
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
#dingutils.send_text("【分布图生成完毕】财力评级已完成, 请点击 https://nextcloud.fangjia.com/s/Jqs5BzXi8xLYeH9 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
|
#dingutils.send_text("【分布图生成完毕】财力评级已完成, 请点击 https://nextcloud.fangjia.com/s/Jqs5BzXi8xLYeH9 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
|
||||||
|
|||||||
@@ -3,10 +3,11 @@
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
|
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from threading import Lock
|
from threading import Lock
|
||||||
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||||
@@ -62,11 +63,10 @@ def deal(actions, item):
|
|||||||
|
|
||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}")
|
log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}")
|
||||||
seaweed_query = {'status': 0, 'cat': 'district'}
|
|
||||||
if pd.notna(city):
|
|
||||||
seaweed_query['city'] = city
|
|
||||||
|
|
||||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||||
|
if pd.notna(city):
|
||||||
|
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||||
|
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||||
|
|
||||||
actions = list()
|
actions = list()
|
||||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||||
@@ -83,4 +83,5 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步教育资源至206 {index_name}")
|
log.info(f"完成同步教育资源至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
@@ -3,10 +3,11 @@
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
|
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from threading import Lock
|
from threading import Lock
|
||||||
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||||
@@ -95,4 +96,5 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步教育资源至206 {index_name}")
|
log.info(f"完成同步教育资源至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
@@ -4,6 +4,7 @@
|
|||||||
from goojia_utils import ElasticSearchClient, areautils, dateutils
|
from goojia_utils import ElasticSearchClient, areautils, dateutils
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import logging
|
import logging
|
||||||
|
import sys
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
|
||||||
@@ -140,4 +141,5 @@ def sync(city=None):
|
|||||||
sync_by_city(city)
|
sync_by_city(city)
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
@@ -5,6 +5,7 @@ from goojia_utils import ElasticSearchClient, nextcloudutils
|
|||||||
import pandas as pd
|
import pandas as pd
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import sys
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
@@ -163,6 +164,6 @@ def save_png(community_city):
|
|||||||
plt.close()
|
plt.close()
|
||||||
|
|
||||||
|
|
||||||
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
sync()
|
sync(city)
|
||||||
#dingutils.send_text("【分布图生成完毕】物业费评级已完成, 请点击 https://nextcloud.fangjia.com/s/HbifLoEDc5wi8HR 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
|
#dingutils.send_text("【分布图生成完毕】物业费评级已完成, 请点击 https://nextcloud.fangjia.com/s/HbifLoEDc5wi8HR 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
|
||||||
@@ -14,6 +14,7 @@ import pandas as pd
|
|||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from threading import Lock
|
from threading import Lock
|
||||||
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||||
@@ -140,4 +141,5 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步附加标签至206 {index_name}")
|
log.info(f"完成同步附加标签至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
@@ -3,10 +3,11 @@
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
|
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from threading import Lock
|
from threading import Lock
|
||||||
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||||
@@ -96,4 +97,5 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步生活配套至206 {index_name}")
|
log.info(f"完成同步生活配套至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
@@ -7,6 +7,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date
|
|||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from threading import Lock
|
from threading import Lock
|
||||||
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||||
@@ -83,4 +84,5 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步医院至206 {index_name}")
|
log.info(f"完成同步医院至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
@@ -5,7 +5,7 @@ from goojia_utils import ElasticSearchClient, dateutils
|
|||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
import datetime
|
import sys
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||||
|
|
||||||
@@ -57,4 +57,5 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步售租比、人口数至206 {index_name}")
|
log.info(f"完成同步售租比、人口数至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
@@ -5,6 +5,7 @@ import logging
|
|||||||
import pandas as pd
|
import pandas as pd
|
||||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
|
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||||
@@ -139,4 +140,5 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步价格信息至206 {index_name}")
|
log.info(f"完成同步价格信息至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
@@ -8,6 +8,7 @@ from goojia_utils import ElasticSearchClient
|
|||||||
import pandas as pd
|
import pandas as pd
|
||||||
import logging
|
import logging
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
|
import sys
|
||||||
|
|
||||||
from sklearn.mixture import GaussianMixture
|
from sklearn.mixture import GaussianMixture
|
||||||
|
|
||||||
@@ -83,4 +84,5 @@ def sync(city=None):
|
|||||||
calc_price_rating(city)
|
calc_price_rating(city)
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import pandas as pd
|
|||||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||||
@@ -112,4 +113,5 @@ def sync(city=None):
|
|||||||
log.info("完成同步相对价格排名数据...")
|
log.info("完成同步相对价格排名数据...")
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
@@ -6,6 +6,7 @@ import pandas as pd
|
|||||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||||
@@ -84,4 +85,5 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步出售信息至206 {index_name}")
|
log.info(f"完成同步出售信息至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
@@ -7,6 +7,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date
|
|||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
import re
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||||
@@ -106,4 +107,5 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步户均总价信息至206 {index_name}")
|
log.info(f"完成同步户均总价信息至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import logging
|
|||||||
import pandas as pd
|
import pandas as pd
|
||||||
from goojia_utils import ElasticSearchClient, dateutils
|
from goojia_utils import ElasticSearchClient, dateutils
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||||
@@ -61,6 +62,9 @@ def complete_by_build_year_area(community_factor_df):
|
|||||||
for index, build_year_area in build_year_area_df.iterrows():
|
for index, build_year_area in build_year_area_df.iterrows():
|
||||||
build_year_area_dict[
|
build_year_area_dict[
|
||||||
build_year_area['city'] + '|' + build_year_area['region'] + "|" + build_year_area['buildyear']] = build_year_area['factor']
|
build_year_area['city'] + '|' + build_year_area['region'] + "|" + build_year_area['buildyear']] = build_year_area['factor']
|
||||||
|
# 修复按城市跑批数据字段可能不存在问题
|
||||||
|
if 'finish_building_date' not in community_factor_df.columns or 'total_price' not in community_factor_df.columns:
|
||||||
|
return False
|
||||||
for index, item in community_factor_df[
|
for index, item in community_factor_df[
|
||||||
(community_factor_df['total_price'].isna()) & (community_factor_df['finish_building_date'].notna())].iterrows():
|
(community_factor_df['total_price'].isna()) & (community_factor_df['finish_building_date'].notna())].iterrows():
|
||||||
finish_building_date = item['finish_building_date']
|
finish_building_date = item['finish_building_date']
|
||||||
@@ -101,18 +105,22 @@ def complete_by_region_area(community_factor_df):
|
|||||||
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
|
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
|
||||||
|
|
||||||
|
|
||||||
def sync():
|
def sync(city=None):
|
||||||
log.info(f"开始补全户均总价信息至206 {index_name}")
|
log.info(f"开始补全户均总价信息至206 {index_name}")
|
||||||
|
|
||||||
community_df = es206_client.scan_by_sql_to_data_frame("select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null")
|
sql = "select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null "
|
||||||
|
if pd.notna(city):
|
||||||
|
sql += " and city='" + city + "'"
|
||||||
|
community_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||||
|
|
||||||
community_factor_df = complete_by_area(community_df)
|
if len(community_df) > 0:
|
||||||
complete_by_build_year_area(community_factor_df)
|
community_factor_df = complete_by_area(community_df)
|
||||||
complete_by_region_area(community_factor_df)
|
complete_by_build_year_area(community_factor_df)
|
||||||
|
complete_by_region_area(community_factor_df)
|
||||||
save(community_factor_df)
|
save(community_factor_df)
|
||||||
|
|
||||||
log.info(f"完成补全户均总价信息至206 {index_name}")
|
log.info(f"完成补全户均总价信息至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
|
import sys
|
||||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
|
|
||||||
@@ -24,14 +25,20 @@ def build_doc(item):
|
|||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
log.info(f"开始同步成交信息至206 {index_name}")
|
log.info(f"开始同步成交信息至206 {index_name}")
|
||||||
|
|
||||||
seaweed_query = {'cat': 'district', 'status': 0}
|
|
||||||
cities = None
|
cities = None
|
||||||
if pd.notna(city):
|
|
||||||
seaweed_query['city'] = city
|
|
||||||
cities = [city]
|
|
||||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||||
|
if pd.notna(city):
|
||||||
|
cities = [city]
|
||||||
|
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||||
|
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||||
|
|
||||||
deal_price_df = commutils.get_deal_avg_price(cities, 3)
|
deal_price_df = commutils.get_deal_avg_price(cities, 3)
|
||||||
|
if len(deal_price_df) <= 0:
|
||||||
|
log.info(f"无成交,无需同步")
|
||||||
|
return
|
||||||
|
if 'match_city' not in deal_price_df.columns or 'match_region' not in deal_price_df.columns or 'match_name' not in deal_price_df.columns:
|
||||||
|
log.info(f"无成交,无需同步")
|
||||||
|
return
|
||||||
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
|
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
|
||||||
deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name',
|
deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name',
|
||||||
'avg_price': 'three_month_trade_avg_price'})
|
'avg_price': 'three_month_trade_avg_price'})
|
||||||
@@ -68,4 +75,5 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步成交信息至206 {index_name}")
|
log.info(f"完成同步成交信息至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
|||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from threading import Lock
|
from threading import Lock
|
||||||
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||||
@@ -86,4 +87,5 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步交通至206 {index_name}")
|
log.info(f"完成同步交通至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
sync()
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
sync(city)
|
||||||
@@ -10,84 +10,85 @@ with DAG(
|
|||||||
catchup=True,
|
catchup=True,
|
||||||
description="地址输出信息分类数据定时更新"
|
description="地址输出信息分类数据定时更新"
|
||||||
) as dag:
|
) as dag:
|
||||||
t1 = BashOperator(
|
comm = BashOperator(
|
||||||
task_id='comm',
|
task_id='comm',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t2 = BashOperator(
|
price = BashOperator(
|
||||||
task_id='price',
|
task_id='price',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t3 = BashOperator(
|
sell = BashOperator(
|
||||||
task_id='sell',
|
task_id='sell',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t4 = BashOperator(
|
avg_cost = BashOperator(
|
||||||
task_id='avg_cost',
|
task_id='avg_cost',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t5 = BashOperator(
|
trade = BashOperator(
|
||||||
task_id='trade',
|
task_id='trade',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t6 = BashOperator(
|
total_price = BashOperator(
|
||||||
task_id='total_price',
|
task_id='total_price',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t6_1 = BashOperator(
|
total_price_complete = BashOperator(
|
||||||
task_id='total_price_complete',
|
task_id='total_price_complete',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t7 = BashOperator(
|
traffic = BashOperator(
|
||||||
task_id='traffic',
|
task_id='traffic',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t7_1 = BashOperator(
|
business_district = BashOperator(
|
||||||
task_id='business_district',
|
task_id='business_district',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t8 = BashOperator(
|
medical = BashOperator(
|
||||||
task_id='medical',
|
task_id='medical',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t9 = BashOperator(
|
life = BashOperator(
|
||||||
task_id='life',
|
task_id='life',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t10 = BashOperator(
|
relative_price_ranking = BashOperator(
|
||||||
task_id='relative_price_ranking',
|
task_id='relative_price_ranking',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t10_1 = BashOperator(
|
estate_charge_complete = BashOperator(
|
||||||
task_id='estate_charge_complete',
|
task_id='estate_charge_complete',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t10_2 = BashOperator(
|
estate_charge_ranking = BashOperator(
|
||||||
task_id='estate_charge_ranking',
|
task_id='estate_charge_ranking',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t11 = BashOperator(
|
consumption = BashOperator(
|
||||||
task_id='consumption',
|
task_id='consumption',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t12 = BashOperator(
|
price_rating = BashOperator(
|
||||||
task_id='price_rating',
|
task_id='price_rating',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t13 = BashOperator(
|
education = BashOperator(
|
||||||
task_id='education',
|
task_id='education',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t14 = BashOperator(
|
education2 = BashOperator(
|
||||||
task_id='education2',
|
task_id='education2',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t15 = BashOperator(
|
extra = BashOperator(
|
||||||
task_id='extra',
|
task_id='extra',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t16 = BashOperator(
|
other = BashOperator(
|
||||||
task_id='other',
|
task_id='other',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t10_1 >> t10_2 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16
|
comm >> price >> avg_cost >> total_price >> total_price_complete >> relative_price_ranking >> estate_charge_complete >> estate_charge_ranking \
|
||||||
|
>> consumption >> price_rating >> sell >> trade >> traffic >> business_district >> medical >> life >> education >> education2 >> extra >> other
|
||||||
|
|||||||
@@ -146,7 +146,7 @@ for index, item in city_count_df.iterrows():
|
|||||||
city_count_df = city_count_df.rename(columns={'小区ID': '小区数量'})
|
city_count_df = city_count_df.rename(columns={'小区ID': '小区数量'})
|
||||||
city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度')
|
city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度')
|
||||||
|
|
||||||
excel_writer.save()
|
#excel_writer.save()
|
||||||
excel_writer.close()
|
excel_writer.close()
|
||||||
|
|
||||||
up_to_nextcloud()
|
up_to_nextcloud()
|
||||||
|
|||||||
Reference in New Issue
Block a user