Merge remote-tracking branch 'origin/master'

This commit is contained in:
xuhe
2023-02-24 18:26:11 +08:00
22 changed files with 138 additions and 84 deletions
+3 -1
View File
@@ -7,6 +7,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
from elasticsearch import helpers
import numpy as np
from concurrent.futures import ThreadPoolExecutor
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -150,4 +151,5 @@ def sync(city=None):
log.info("完成同步消费数据...")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -3,10 +3,11 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -104,4 +105,5 @@ def sync(city=None):
log.info(f"完成同步商圈至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+7 -2
View File
@@ -7,6 +7,7 @@ import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
import os
import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -93,7 +94,10 @@ def sync(city=None):
'estate_charge': 1, 'finish_building_date': 1})
.batch_size(100000)))
community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}")
sql = f"select id from {index_name} "
if pd.notna(city):
sql += f" where city='{city}'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
if len(community_summary_df) > 0:
@@ -154,4 +158,5 @@ def sync(city=None):
log.info(f"完成同步小区基本属性至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -8,6 +8,7 @@ import logging
import os
from elasticsearch import helpers
import matplotlib.pyplot as plt
import sys
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import QuantileTransformer
@@ -249,5 +250,7 @@ def save_png(community_city):
# 关窗口
plt.close()
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
#dingutils.send_text("【分布图生成完毕】财力评级已完成, 请点击 https://nextcloud.fangjia.com/s/Jqs5BzXi8xLYeH9 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
@@ -3,10 +3,11 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -62,11 +63,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}")
seaweed_query = {'status': 0, 'cat': 'district'}
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -83,4 +83,5 @@ def sync(city=None):
log.info(f"完成同步教育资源至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -3,10 +3,11 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -95,4 +96,5 @@ def sync(city=None):
log.info(f"完成同步教育资源至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -4,6 +4,7 @@
from goojia_utils import ElasticSearchClient, areautils, dateutils
import pandas as pd
import logging
import sys
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
@@ -140,4 +141,5 @@ def sync(city=None):
sync_by_city(city)
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -5,6 +5,7 @@ from goojia_utils import ElasticSearchClient, nextcloudutils
import pandas as pd
import logging
import os
import sys
from elasticsearch import helpers
import matplotlib.pyplot as plt
@@ -163,6 +164,6 @@ def save_png(community_city):
plt.close()
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
#dingutils.send_text("【分布图生成完毕】物业费评级已完成, 请点击 https://nextcloud.fangjia.com/s/HbifLoEDc5wi8HR 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
+3 -1
View File
@@ -14,6 +14,7 @@ import pandas as pd
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -140,4 +141,5 @@ def sync(city=None):
log.info(f"完成同步附加标签至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+4 -2
View File
@@ -3,10 +3,11 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -96,4 +97,5 @@ def sync(city=None):
log.info(f"完成同步生活配套至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -1
View File
@@ -7,6 +7,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -83,4 +84,5 @@ def sync(city=None):
log.info(f"完成同步医院至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -2
View File
@@ -5,7 +5,7 @@ from goojia_utils import ElasticSearchClient, dateutils
import logging
import pandas as pd
from elasticsearch import helpers
import datetime
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -57,4 +57,5 @@ def sync(city=None):
log.info(f"完成同步售租比、人口数至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -1
View File
@@ -5,6 +5,7 @@ import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
from elasticsearch import helpers
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -139,4 +140,5 @@ def sync(city=None):
log.info(f"完成同步价格信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -8,6 +8,7 @@ from goojia_utils import ElasticSearchClient
import pandas as pd
import logging
from elasticsearch import helpers
import sys
from sklearn.mixture import GaussianMixture
@@ -83,4 +84,5 @@ def sync(city=None):
calc_price_rating(city)
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -7,6 +7,7 @@ import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -112,4 +113,5 @@ def sync(city=None):
log.info("完成同步相对价格排名数据...")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -1
View File
@@ -6,6 +6,7 @@ import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -84,4 +85,5 @@ def sync(city=None):
log.info(f"完成同步出售信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -7,6 +7,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import re
import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -106,4 +107,5 @@ def sync(city=None):
log.info(f"完成同步户均总价信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -5,6 +5,7 @@ import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, dateutils
from elasticsearch import helpers
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -61,6 +62,9 @@ def complete_by_build_year_area(community_factor_df):
for index, build_year_area in build_year_area_df.iterrows():
build_year_area_dict[
build_year_area['city'] + '|' + build_year_area['region'] + "|" + build_year_area['buildyear']] = build_year_area['factor']
# 修复按城市跑批数据字段可能不存在问题
if 'finish_building_date' not in community_factor_df.columns or 'total_price' not in community_factor_df.columns:
return False
for index, item in community_factor_df[
(community_factor_df['total_price'].isna()) & (community_factor_df['finish_building_date'].notna())].iterrows():
finish_building_date = item['finish_building_date']
@@ -101,18 +105,22 @@ def complete_by_region_area(community_factor_df):
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
def sync():
def sync(city=None):
log.info(f"开始补全户均总价信息至206 {index_name}")
community_df = es206_client.scan_by_sql_to_data_frame("select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null")
sql = "select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null "
if pd.notna(city):
sql += " and city='" + city + "'"
community_df = es206_client.scan_by_sql_to_data_frame(sql)
community_factor_df = complete_by_area(community_df)
complete_by_build_year_area(community_factor_df)
complete_by_region_area(community_factor_df)
save(community_factor_df)
if len(community_df) > 0:
community_factor_df = complete_by_area(community_df)
complete_by_build_year_area(community_factor_df)
complete_by_region_area(community_factor_df)
save(community_factor_df)
log.info(f"完成补全户均总价信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+13 -5
View File
@@ -3,6 +3,7 @@
"""
import logging
import pandas as pd
import sys
from goojia_utils import ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
@@ -24,14 +25,20 @@ def build_doc(item):
def sync(city=None):
log.info(f"开始同步成交信息至206 {index_name}")
seaweed_query = {'cat': 'district', 'status': 0}
cities = None
if pd.notna(city):
seaweed_query['city'] = city
cities = [city]
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
if pd.notna(city):
cities = [city]
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
deal_price_df = commutils.get_deal_avg_price(cities, 3)
if len(deal_price_df) <= 0:
log.info(f"无成交,无需同步")
return
if 'match_city' not in deal_price_df.columns or 'match_region' not in deal_price_df.columns or 'match_name' not in deal_price_df.columns:
log.info(f"无成交,无需同步")
return
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name',
'avg_price': 'three_month_trade_avg_price'})
@@ -68,4 +75,5 @@ def sync(city=None):
log.info(f"完成同步成交信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -1
View File
@@ -7,6 +7,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -86,4 +87,5 @@ def sync(city=None):
log.info(f"完成同步交通至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+42 -41
View File
@@ -10,84 +10,85 @@ with DAG(
catchup=True,
description="地址输出信息分类数据定时更新"
) as dag:
t1 = BashOperator(
comm = BashOperator(
task_id='comm',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t2 = BashOperator(
price = BashOperator(
task_id='price',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t3 = BashOperator(
sell = BashOperator(
task_id='sell',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t4 = BashOperator(
avg_cost = BashOperator(
task_id='avg_cost',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t5 = BashOperator(
trade = BashOperator(
task_id='trade',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t6 = BashOperator(
total_price = BashOperator(
task_id='total_price',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t6_1 = BashOperator(
total_price_complete = BashOperator(
task_id='total_price_complete',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t7 = BashOperator(
traffic = BashOperator(
task_id='traffic',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t7_1 = BashOperator(
business_district = BashOperator(
task_id='business_district',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t8 = BashOperator(
medical = BashOperator(
task_id='medical',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t9 = BashOperator(
life = BashOperator(
task_id='life',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t10 = BashOperator(
relative_price_ranking = BashOperator(
task_id='relative_price_ranking',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t10_1 = BashOperator(
estate_charge_complete = BashOperator(
task_id='estate_charge_complete',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t10_2 = BashOperator(
estate_charge_ranking = BashOperator(
task_id='estate_charge_ranking',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t11 = BashOperator(
consumption = BashOperator(
task_id='consumption',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t12 = BashOperator(
price_rating = BashOperator(
task_id='price_rating',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t13 = BashOperator(
education = BashOperator(
task_id='education',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t14 = BashOperator(
education2 = BashOperator(
task_id='education2',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t15 = BashOperator(
extra = BashOperator(
task_id='extra',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t16 = BashOperator(
other = BashOperator(
task_id='other',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t10_1 >> t10_2 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16
comm >> price >> avg_cost >> total_price >> total_price_complete >> relative_price_ranking >> estate_charge_complete >> estate_charge_ranking \
>> consumption >> price_rating >> sell >> trade >> traffic >> business_district >> medical >> life >> education >> education2 >> extra >> other
+1 -1
View File
@@ -146,7 +146,7 @@ for index, item in city_count_df.iterrows():
city_count_df = city_count_df.rename(columns={'小区ID': '小区数量'})
city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度')
excel_writer.save()
#excel_writer.save()
excel_writer.close()
up_to_nextcloud()