diff --git a/dags/business/business/change_new_to_old_seaweed.py b/dags/business/business/change_new_to_old_seaweed.py new file mode 100644 index 0000000..14e6e21 --- /dev/null +++ b/dags/business/business/change_new_to_old_seaweed.py @@ -0,0 +1,70 @@ +""" +新房变二手房定时脚本 +创建年代、入住/交付时间三年后,或者有样本就变成二手房 +""" +from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils +from pymongo import UpdateOne +import pandas as pd +import logging +import datetime +import time + +log = logging.getLogger(__name__) + + +def do(): + log.info("开始执行新房变二手房脚本") + seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') + df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'is_new': 1}, + {'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'open_date': 1, 'live_date': 1, + 'create_year': 1}).batch_size(100000))) + if len(df) <= 0: + log.info("执行新房变二手房脚本完毕,未发现新房") + return False + + actions = list() + count = 0 + + def deal(x): + global count + is_old = False + time_now = int(time.mktime(time.strptime(datetime.datetime.now().strftime('%Y-%m-%d'), '%Y-%m-%d'))) + if 'open_date' in x and pd.notna(x['open_date']): + time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(x['open_date'], years=3), '%Y-%m-%d'), '%Y-%m-%d'))) + if time_old < time_now: + is_old = True + if is_old is False and 'live_date' in x and pd.notna(x['live_date']): + time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['live_date']), years=3), '%Y-%m-%d'), '%Y-%m-%d'))) + if time_old < time_now: + is_old = True + if is_old is False and 'create_year' in x and pd.notna(x['create_year']): + time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['create_year'], '%Y-%m'), years=3), '%Y-%m-%d'), '%Y-%m-%d'))) + if time_old < time_now: + is_old = True + # 查询样本库 + names = [x['name']] + if type(x['alias']) == list: + for alias_item in x['alias']: + names.append(alias_item) + names_str = "','".join(names) + names_str = "'" + names_str + "'" + if is_old is False: + nums = commutils.get_sample(x['city'], x['region'], names_str) + if nums > 0: + is_old = True + if is_old: + actions.append(UpdateOne({'_id': x['_id']}, {'$set': {'is_new': 0, 'm_date': datetime.datetime.now()}})) + if len(actions) == 5000: + seaweed_conn.bulk_write(actions) + count += len(actions) + actions.clear() + + df.apply(lambda x: deal(x), axis=1) + if len(actions) > 0: + seaweed_conn.bulk_write(actions) + count += len(actions) + actions.clear() + log.info(f"执行新房变二手房脚本完毕,共更新{count}") + + +do() diff --git a/dags/business/business/community/avg_cost.py b/dags/business/business/community/avg_cost.py index 0b705c9..045e064 100644 --- a/dags/business/business/community/avg_cost.py +++ b/dags/business/business/community/avg_cost.py @@ -5,43 +5,67 @@ import logging import pandas as pd from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils from elasticsearch import helpers -import numpy as np from concurrent.futures import ThreadPoolExecutor +import sys +from threading import Lock +mutex = Lock() log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" m_date = dateutils.iso_format() path = "/home/airflow/data/community" +file_path = "/opt/airflow/dags/repo/dags/business/business/community/data" + +avg_cost_min_max_df = pd.read_excel(file_path + '/全国各区域人均消费中位值.xlsx') +avg_cost_city_region_dict = dict() -def find_around_price(lng, lat): +# 加载各区域人均消费最大最小值 +def deal_avg_cost(x): + city_now = x['city'] + region_now = x['region'] + if pd.notna(city_now) and pd.notna(region_now): + avg_cost_city_region_dict[city_now + region_now] = {'min': x['3'], 'max': x['97']} + elif pd.notna(city_now): + avg_cost_city_region_dict[city_now] = {'min': x['3'], 'max': x['97']} + + +avg_cost_min_max_df.apply(lambda x: deal_avg_cost(x), axis=1) + + +def find_around_price(seaweed): """ 查找周边均价 + 返回取值为10~300,为所有有价格的1-99百分位 """ - if pd.isna(lng): - return None + lng = commutils.format_lat_lng(seaweed['lng']) + lat = commutils.format_lat_lng(seaweed['lat']) + city_short = seaweed['city'] + region_short = seaweed['region'] + city_display = areautils.get_city_display(city_short) + + filter_query = [{"terms": {"tag": ["美食;中餐厅", "美食;外国餐厅"]}}] + if pd.notna(city_display): + filter_query.append({"term": {"city": {"value": city_display}}}) + + min_now = 10 + max_now = 300 + if (city_short+region_short) in avg_cost_city_region_dict: + min_now = avg_cost_city_region_dict[city_short + region_short]['min'] + max_now = avg_cost_city_region_dict[city_short + region_short]['max'] + elif city_short in avg_cost_city_region_dict: + min_now = avg_cost_city_region_dict[city_short]['min'] + max_now = avg_cost_city_region_dict[city_short]['max'] + + filter_query.append({"range": {"price": {"gte": min_now, "lte": max_now}}}) + filter_query.append({"geo_distance": {"distance": 1000, "location": [lng, lat]}}) + ret_es = es206_client.elastic_client.search({ "query": { "bool": { - "must": [ - { - "term": { - "status": { - "value": 0 - } - } - } - ], - "filter": [ - { - "geo_distance": { - "distance": 1000, - "location": [lng, lat] - } - } - ] + "filter": filter_query } }, "size": 0, @@ -58,72 +82,51 @@ def find_around_price(lng, lat): def build_doc(item): doc_dict = dict() - if 'avg_cost' in item and pd.notna(item['avg_cost']): - doc_dict['avg_cost'] = item['avg_cost'] + if 'avg_cost' in item: + if pd.notna(item['avg_cost']) or item['avg_cost'] > 0: + doc_dict['avg_cost'] = int(item['avg_cost']) + else: + doc_dict['avg_cost'] = None return doc_dict -def update_status(new_df): - actions = list() +find_count = 0 - def deal(item): - action_doc = { - "_op_type": "update", - "_index": "bd_poi", - "_id": item['_id'], - "doc": {'status': 0} - } - actions.append(action_doc) - if len(actions) == 5000: - helpers.bulk(es206_client.elastic_client, actions) - actions.clear() - return item - new_df.apply(lambda x: deal(x), axis=1) - if len(actions) > 0: - helpers.bulk(es206_client.elastic_client, actions) - actions.clear() +def find_around_price_item(seaweed): + global find_count + find_count += 1 + if find_count % 1000 == 0: + log.info("已查找" + str(find_count)) + if 'lng' not in seaweed: + return + price = find_around_price(seaweed) + mutex.acquire(timeout=60) + if pd.notna(price): + seaweed['avg_cost'] = int(price) + else: + seaweed['avg_cost'] = None + mutex.release() def sync(city=None): log.info("开始同步消费数据...") - query_sql = "select id, city, tag, type, price from bd_poi where price > 0 and tag in('美食;中餐厅', '美食;外国餐厅') " - if pd.notna(city): - query_sql += " and city='" + areautils.get_city_display(city) + "'" - df = es206_client.scan_by_sql_to_data_frame(query_sql) - - # 按百分位5-95筛选数据 - cities = df['city'].unique() - for city_now in cities: - df.loc[df['city'] == city_now, 'minPrice5'] = np.percentile(a=df[df['city'] == city_now]['price'], q=5, axis=0) - df.loc[df['city'] == city_now, 'minPrice95'] = np.percentile(a=df[df['city'] == city_now]['price'], q=95, axis=0) - - new_df = df[(df['price'] >= df['minPrice5']) & (df['price'] <= df['minPrice95'])] - new_df['new_mean'] = new_df.groupby(by=['city'])['price'].transform(np.mean) - - # 按最新的百分位更新库中价格状态 - update_status(new_df) - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') if pd.notna(city): seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() seaweed_df = seaweed_df.reset_index(drop=True) - def find_around_price_item(index, item): - if index % 1000 == 0: - log.info("已查找" + str(index)) - if 'lng' not in item: - return - lng = commutils.format_lat_lng(item['lng']) - lat = commutils.format_lat_lng(item['lat']) - price = find_around_price(lng, lat) - seaweed_df.loc[index, 'avg_cost'] = price + seaweed_list = list() + seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1) with ThreadPoolExecutor(max_workers=30) as pool: - for index, item in seaweed_df.iterrows(): - pool.submit(find_around_price_item, index, item) - # for index, item in seaweed_df.iterrows(): - # find_around_price_item(index, item) + for seaweed in seaweed_list: + pool.submit(find_around_price_item, seaweed) + + # for seaweed in seaweed_list: + # find_around_price_item(seaweed) + + seaweed_df = pd.DataFrame(seaweed_list) actions = list() count = 0 for index, item in seaweed_df.iterrows(): @@ -150,4 +153,5 @@ def sync(city=None): log.info("完成同步消费数据...") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) diff --git a/dags/business/business/community/business_district.py b/dags/business/business/community/business_district.py index 0a21ad1..ccece89 100644 --- a/dags/business/business/community/business_district.py +++ b/dags/business/business/community/business_district.py @@ -3,10 +3,11 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils +from goojia_utils import ElasticSearchClient, commutils, dateutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -104,4 +105,5 @@ def sync(city=None): log.info(f"完成同步商圈至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index f08d67f..5043402 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -7,6 +7,7 @@ import pandas as pd from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils from elasticsearch import helpers import os +import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -93,7 +94,10 @@ def sync(city=None): 'estate_charge': 1, 'finish_building_date': 1}) .batch_size(100000))) - community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}") + sql = f"select id from {index_name} " + if pd.notna(city): + sql += f" where city='{city}'" + community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) seaweed_df['_id'] = seaweed_df['_id'].astype(str) if len(community_summary_df) > 0: @@ -154,4 +158,5 @@ def sync(city=None): log.info(f"完成同步小区基本属性至206 {index_name}") -sync() +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) diff --git a/dags/business/business/community/consumption.py b/dags/business/business/community/consumption.py index 11af5a5..1d186c9 100644 --- a/dags/business/business/community/consumption.py +++ b/dags/business/business/community/consumption.py @@ -8,6 +8,7 @@ import logging import os from elasticsearch import helpers import matplotlib.pyplot as plt +import sys from sklearn.mixture import GaussianMixture from sklearn.preprocessing import QuantileTransformer @@ -50,9 +51,19 @@ def calc_consumption_potential(city): np.mean) community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.std) - community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform( + # 噪音词处理 + community_summary_df['avg_cost_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform( + community_summary_df[['avg_cost']]) + + # 结合全国物业费情况,对于7以上的占比1% + community_summary_df.loc[community_summary_df['estate_charge'] > 7, 'estate_charge'] = 7 + + community_summary_df['estate_charge_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform( + community_summary_df[['estate_charge']]) + + community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge_fixed'].transform( np.mean) - community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform( + community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge_fixed'].transform( np.std) community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])[ @@ -60,8 +71,8 @@ def calc_consumption_potential(city): community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])[ 'relative_price_ranking'].transform(np.std) - community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.mean) - community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.std) + community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost_fixed'].transform(np.mean) + community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost_fixed'].transform(np.std) # 和田等小城市均值可能为0 if community_summary_df['total_price_std'].unique()[0] != 0: @@ -70,7 +81,7 @@ def calc_consumption_potential(city): community_summary_df['total_price_score'] = 0 if community_summary_df['estate_charge_std'].unique()[0] != 0: - community_summary_df['estate_charge_score'] = 139.1374524 * (community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std'] + community_summary_df['estate_charge_score'] = 139.1374524 * (community_summary_df['estate_charge_fixed'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std'] else: community_summary_df['estate_charge_score'] = 0 @@ -80,7 +91,7 @@ def calc_consumption_potential(city): community_summary_df['relative_price_ranking_score'] = 0 if community_summary_df['avg_cost_std'].unique()[0] != 0: - community_summary_df['avg_cost_score'] = 67.0195871700 * (community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std'] + community_summary_df['avg_cost_score'] = 67.0195871700 * (community_summary_df['avg_cost_fixed'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std'] else: community_summary_df['avg_cost_score'] = 0 @@ -249,5 +260,7 @@ def save_png(community_city): # 关窗口 plt.close() -sync() + +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) #dingutils.send_text("【分布图生成完毕】财力评级已完成, 请点击 https://nextcloud.fangjia.com/s/Jqs5BzXi8xLYeH9 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78") diff --git a/dags/business/business/community/data/全国各区域人均消费中位值.xlsx b/dags/business/business/community/data/全国各区域人均消费中位值.xlsx new file mode 100644 index 0000000..3935a59 Binary files /dev/null and b/dags/business/business/community/data/全国各区域人均消费中位值.xlsx differ diff --git a/dags/business/business/community/education.py b/dags/business/business/community/education.py index 4af97d0..f00e718 100644 --- a/dags/business/business/community/education.py +++ b/dags/business/business/community/education.py @@ -3,10 +3,11 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -62,11 +63,10 @@ def deal(actions, item): def sync(city=None): log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}") - seaweed_query = {'status': 0, 'cat': 'district'} - if pd.notna(city): - seaweed_query['city'] = city - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + if pd.notna(city): + seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() + seaweed_df = seaweed_df.reset_index(drop=True) actions = list() with ThreadPoolExecutor(max_workers=30) as pool: @@ -83,4 +83,5 @@ def sync(city=None): log.info(f"完成同步教育资源至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/education2.py b/dags/business/business/community/education2.py index 3fd27ba..ec37083 100644 --- a/dags/business/business/community/education2.py +++ b/dags/business/business/community/education2.py @@ -3,10 +3,11 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -95,4 +96,5 @@ def sync(city=None): log.info(f"完成同步教育资源至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/estate_charge_complete.py b/dags/business/business/community/estate_charge_complete.py index 23c2460..ed03faa 100644 --- a/dags/business/business/community/estate_charge_complete.py +++ b/dags/business/business/community/estate_charge_complete.py @@ -4,14 +4,17 @@ from goojia_utils import ElasticSearchClient, areautils, dateutils import pandas as pd import logging +import sys from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor +from threading import Lock log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +mutex = Lock() def save(community_summary_df): if 'estate_charge' not in community_summary_df.columns: @@ -109,7 +112,7 @@ def sync_by_city(city_now): log.info(f"{city_now}开始补充物业费....") sql = "select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='" + city_now + "' and (estate_charge is null or estate_charge_flag=1)" community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) - with ThreadPoolExecutor(max_workers=50) as pool: + with ThreadPoolExecutor(max_workers=20) as pool: def deal(index, item): if index % 1000 == 0 and index > 0: log.info(f"已查找{index}") @@ -126,7 +129,9 @@ def sync_by_city(city_now): # 取周边均值 estate_charge = docs['estate_charge'].mean() if pd.notna(estate_charge): + mutex.acquire(timeout=60) community_summary_df.loc[index, "estate_charge"] = estate_charge + mutex.release() for index, item in community_summary_df.iterrows(): pool.submit(deal, index, item) save(community_summary_df) @@ -140,4 +145,5 @@ def sync(city=None): sync_by_city(city) -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/estate_charge_ranking.py b/dags/business/business/community/estate_charge_ranking.py index db6489a..a5319af 100644 --- a/dags/business/business/community/estate_charge_ranking.py +++ b/dags/business/business/community/estate_charge_ranking.py @@ -5,6 +5,7 @@ from goojia_utils import ElasticSearchClient, nextcloudutils import pandas as pd import logging import os +import sys from elasticsearch import helpers import matplotlib.pyplot as plt @@ -163,6 +164,6 @@ def save_png(community_city): plt.close() - -sync() +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) #dingutils.send_text("【分布图生成完毕】物业费评级已完成, 请点击 https://nextcloud.fangjia.com/s/HbifLoEDc5wi8HR 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78") \ No newline at end of file diff --git a/dags/business/business/community/extra.py b/dags/business/business/community/extra.py index 4ddad24..7561eac 100644 --- a/dags/business/business/community/extra.py +++ b/dags/business/business/community/extra.py @@ -14,6 +14,7 @@ import pandas as pd from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -140,4 +141,5 @@ def sync(city=None): log.info(f"完成同步附加标签至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/life.py b/dags/business/business/community/life.py index 3c62f6f..0e62d00 100644 --- a/dags/business/business/community/life.py +++ b/dags/business/business/community/life.py @@ -3,10 +3,11 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -96,4 +97,5 @@ def sync(city=None): log.info(f"完成同步生活配套至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/medical.py b/dags/business/business/community/medical.py index 832a4bf..536b871 100644 --- a/dags/business/business/community/medical.py +++ b/dags/business/business/community/medical.py @@ -7,6 +7,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock +import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -83,4 +84,5 @@ def sync(city=None): log.info(f"完成同步医院至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/other.py b/dags/business/business/community/other.py index 0f02d66..6a67cff 100644 --- a/dags/business/business/community/other.py +++ b/dags/business/business/community/other.py @@ -5,7 +5,7 @@ from goojia_utils import ElasticSearchClient, dateutils import logging import pandas as pd from elasticsearch import helpers -import datetime +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -57,4 +57,5 @@ def sync(city=None): log.info(f"完成同步售租比、人口数至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/person_rating.py b/dags/business/business/community/person_rating.py new file mode 100644 index 0000000..2868bc7 --- /dev/null +++ b/dags/business/business/community/person_rating.py @@ -0,0 +1,79 @@ +""" +人口评级 +""" +from goojia_utils import ElasticSearchClient +import pandas as pd +import logging +from elasticsearch import helpers +import sys + +from sklearn.mixture import GaussianMixture + +log = logging.getLogger(__name__) + +es206_client = ElasticSearchClient(host='192.168.0.206') +index_name = "community_summary" + + +def gmm_train(total_rate, consumption_df, source, target): + # 高斯混合模型 + gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]]) + consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]]) + consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean') + consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int) + consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1) + return consumption_df + + +def calc_person_rating(city=None): + """ + 计算消费等级 + """ + sql = "select id, city, region, name, household_count from community_summary" + if pd.notna(city): + sql += " where city='" + city + "'" + community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + + consumption_aim_df = community_summary_df[['_id', 'city', 'household_count']] + consumption_aim_df = consumption_aim_df.dropna(subset=['household_count']) + cities = consumption_aim_df['city'].unique() + source = "household_count" + target = "person_rating" + actions = list() + count = 0 + + for city in cities: + df = consumption_aim_df[consumption_aim_df['city'] == city].copy() + if len(df) < 5: + continue + df = gmm_train(5, df, source, target) + + # 批量更新 + for index, item in df.iterrows(): + row = dict() + _id = str(item["_id"]) + row['person_rating'] = item['person_rating'] + + action = {'_op_type': 'update', + '_index': index_name, + '_id': _id, # 请勿忘记ID + 'doc': row} + actions.append(action) + if len(actions) == 5000: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("评级中: 当前已评级{}".format(count)) + if len(actions) > 0: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("评级完成,共更新: {}".format(count)) + + +def sync(city=None): + calc_person_rating(city) + + +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index 2d2112f..7d967ad 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -3,8 +3,9 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils from elasticsearch import helpers +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -37,78 +38,69 @@ def build_doc(item): return doc_dict -def sync(city=None): - log.info(f"开始同步价格信息至206 {index_name}") - if pd.notna(city): - cities = [city] - else: - cities = None - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') - if pd.notna(city): - seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() - seaweed_df = seaweed_df.reset_index(drop=True) +def sync_by_city(city, total_seaweed_df): + seaweed_df = total_seaweed_df[total_seaweed_df['city'] == city].copy() + if len(seaweed_df) == 0: + return + seaweed_df = seaweed_df.reset_index(drop=True) # 小区所在城市均价 - log.info('开始导出小区所在城市均价') city_price_df = commutils.get_city_price(city=city) - city_price_df = city_price_df[['city', 'avg_price']] - city_price_df = city_price_df.rename(columns={'avg_price': 'city_avg_price'}) - seaweed_df = pd.merge(seaweed_df, city_price_df, how='left', on=['city']) - log.info('小区所在城市均价导出完毕') + if len(city_price_df) > 0: + city_price_df = city_price_df[['city', 'avg_price']] + city_price_df = city_price_df.rename(columns={'avg_price': 'city_avg_price'}) + seaweed_df = pd.merge(seaweed_df, city_price_df, how='left', on=['city']) # 小区所在区域均价 - log.info('开始导出小区所在区域均价') region_price_df = commutils.get_region_price(city=city) - region_price_df = region_price_df[['city', 'region', 'avg_price']] - region_price_df = region_price_df.rename(columns={'avg_price': 'region_avg_price'}) - seaweed_df = pd.merge(seaweed_df, region_price_df, how='left', on=['city', 'region']) - log.info('小区所在区域均价导出完毕') + if len(region_price_df) > 0: + region_price_df = region_price_df[['city', 'region', 'avg_price']] + region_price_df = region_price_df.rename(columns={'avg_price': 'region_avg_price'}) + seaweed_df = pd.merge(seaweed_df, region_price_df, how='left', on=['city', 'region']) # 小区均价 - log.info('开始导出小区均价') avg_price_df = commutils.get_avg_price(city=city) - avg_price_df = avg_price_df[['city', 'district_name', 'region', 'avg_price']] - avg_price_df = avg_price_df.rename(columns={'district_name': 'name'}) - seaweed_df = pd.merge(seaweed_df, avg_price_df, how='left', on=['city', 'region', 'name']) - log.info('小区均价导出完毕') + if len(avg_price_df) > 0: + avg_price_df = avg_price_df[['city', 'district_name', 'region', 'avg_price']] + avg_price_df = avg_price_df.rename(columns={'district_name': 'name'}) + seaweed_df = pd.merge(seaweed_df, avg_price_df, how='left', on=['city', 'region', 'name']) - # 小区均价区域排名 - seaweed_df['avg_price_region_ranking'] = seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)['avg_price'] - # 小区均价城市排名 - seaweed_df['avg_price_ranking'] = seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price'] + # 小区均价区域排名 + seaweed_df['avg_price_region_ranking'] = \ + seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)['avg_price'] + # 小区均价城市排名 + seaweed_df['avg_price_ranking'] = \ + seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price'] # 12个月均价涨跌幅 - log.info('开始导出12个月均价涨跌幅') - avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=cities, months=-12) - avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']] - avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(columns={'district_name': 'name', 'up_and_downs': 'year_on_year'}) - seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name']) - log.info('12个月均价涨跌幅导出完毕') + avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-12) + if len(avg_price_up_and_downs_df) > 0: + avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']] + avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename( + columns={'district_name': 'name', 'up_and_downs': 'year_on_year'}) + seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name']) # 近半年均价涨跌幅 - log.info('开始导出近半年均价涨跌幅') - avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=cities, months=-6) - avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']] - avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename( - columns={'district_name': 'name', 'up_and_downs': 'half_on_year'}) - seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name']) - log.info('近半年均价涨跌幅导出完毕') + avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-6) + if len(avg_price_up_and_downs_df) > 0: + avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']] + avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename( + columns={'district_name': 'name', 'up_and_downs': 'half_on_year'}) + seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name']) # 平米租金 - log.info('开始导出平米租金') rent_price_df = commutils.get_rent_price(city=city) - rent_price_df = rent_price_df[['city', 'region', 'district_name', 'avg_price']] - rent_price_df = rent_price_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_price'}) - seaweed_df = pd.merge(seaweed_df, rent_price_df, how='left', on=['city', 'region', 'name']) - log.info('平米租金导出完毕') + if len(rent_price_df) > 0: + rent_price_df = rent_price_df[['city', 'region', 'district_name', 'avg_price']] + rent_price_df = rent_price_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_price'}) + seaweed_df = pd.merge(seaweed_df, rent_price_df, how='left', on=['city', 'region', 'name']) # 政府指导价 - log.info('开始导出政府指导价') - gov_price_df = commutils.get_gov_guide_price() - gov_price_df = gov_price_df[['city', 'region', 'name', 'avg_price']] - gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'}) - seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name']) - log.info('政府指导价导出完毕') + gov_price_df = commutils.get_gov_guide_price(city=city) + if len(gov_price_df) > 0: + gov_price_df = gov_price_df[['city', 'region', 'name', 'avg_price']] + gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'}) + seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name']) actions = list() count = 0 @@ -131,7 +123,22 @@ def sync(city=None): if len(actions) > 0: helpers.bulk(es206_client.elastic_client, actions) actions.clear() + + +def sync(city=None): + log.info(f"开始同步价格信息至206 {index_name}") + if pd.notna(city): + cities = [city] + else: + cities = areautils.get_all_cities() + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + for city in cities: + log.info(f"开始同步{city}") + sync_by_city(city, seaweed_df) + log.info(f"完成同步{city}") + log.info(f"完成同步价格信息至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/price_rating.py b/dags/business/business/community/price_rating.py index d6669c6..3020b87 100644 --- a/dags/business/business/community/price_rating.py +++ b/dags/business/business/community/price_rating.py @@ -8,6 +8,7 @@ from goojia_utils import ElasticSearchClient import pandas as pd import logging from elasticsearch import helpers +import sys from sklearn.mixture import GaussianMixture @@ -16,12 +17,13 @@ log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" + def gmm_train(total_rate, consumption_df, source, target): # 高斯混合模型 - gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]]) - consumption_df['cluster'] = gmm.predict(consumption_df[[source]]) - consumption_df['price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean') - consumption_df[target] = consumption_df['price_mean_cluster'].rank(method='dense').astype(int) + gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]]) + consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]]) + consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean') + consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int) consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1) return consumption_df @@ -46,12 +48,12 @@ def calc_price_rating(city=None): for city in cities: df = consumption_aim_df[consumption_aim_df['city'] == city].copy() - if len(df) < 10: + if len(df) < 5: continue - df = gmm_train(10, df, source, target) - if len(df) >= 100: - df = gmm_train(100, df, source, score_target) - df['price_score'] /= 10 + df = gmm_train(5, df, source, target) + if len(df) >= 5: + df = gmm_train(5, df, source, score_target) + df['price_score'] /= 5 # 批量更新 for index, item in df.iterrows(): @@ -82,4 +84,5 @@ def sync(city=None): calc_price_rating(city) -sync() +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) diff --git a/dags/business/business/community/relative_price_ranking.py b/dags/business/business/community/relative_price_ranking.py index dd408df..5cdf898 100644 --- a/dags/business/business/community/relative_price_ranking.py +++ b/dags/business/business/community/relative_price_ranking.py @@ -7,6 +7,8 @@ import pandas as pd from goojia_utils import ElasticSearchClient, commutils, dateutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor +import sys +from threading import Lock log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -14,9 +16,10 @@ es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" path = "/home/airflow/data/community" m_date = dateutils.iso_format() +mutex = Lock() -def find_around_seaweed(lng, lat): +def find_around_seaweed(city_now, lng, lat): """ 查找周边均价 """ @@ -27,6 +30,13 @@ def find_around_seaweed(lng, lat): "query": { "bool": { "filter": [ + { + "term": { + "city": { + "value": city_now + } + } + }, { "range": { "avg_price": { @@ -54,6 +64,31 @@ def build_doc(item): return doc_dict +find_count = 0 + + +def find_around_price_item(seaweed): + global find_count + find_count += 1 + if find_count % 1000 == 0: + log.info("已查找" + str(find_count)) + if 'lng' not in seaweed: + return + lng = commutils.format_lat_lng(seaweed['lng']) + lat = commutils.format_lat_lng(seaweed['lat']) + if pd.notna(lng): + city_now = seaweed['city'] + around_seaweed_df = find_around_seaweed(city_now, lng, lat) + mutex.acquire(timeout=60) + if len(around_seaweed_df) > 1: + around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True) + seaweed['relative_price_ranking'] = around_seaweed_df[around_seaweed_df['id'] == seaweed['_id']].iloc[0][ + 'relative_price_ranking'] + elif len(around_seaweed_df) == 1: + seaweed['relative_price_ranking'] = 0.5 + mutex.release() + + def sync(city=None): log.info("开始同步相对价格排名数据...") @@ -62,24 +97,17 @@ def sync(city=None): seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() seaweed_df = seaweed_df.reset_index(drop=True) - def find_around_price_item(index, item): - if index % 1000 == 0: - log.info("已处理" + str(index)) - if 'lng' not in item: - return - lng = commutils.format_lat_lng(item['lng']) - lat = commutils.format_lat_lng(item['lat']) - if pd.notna(lng): - around_seaweed_df = find_around_seaweed(lng, lat) - if len(around_seaweed_df) > 1: - around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True) - seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0] - elif len(around_seaweed_df) == 1: - seaweed_df.loc[index, 'relative_price_ranking'] = 0.5 + seaweed_list = list() + seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1) with ThreadPoolExecutor(max_workers=30) as pool: - for index, item in seaweed_df.iterrows(): - pool.submit(find_around_price_item, index, item) + for seaweed in seaweed_list: + pool.submit(find_around_price_item, seaweed) + + # for seaweed in seaweed_list: + # find_around_price_item(seaweed) + + seaweed_df = pd.DataFrame(seaweed_list) actions = list() count = 0 @@ -90,17 +118,15 @@ def sync(city=None): "_op_type": "update", "_index": index_name, "_id": item['_id'], - "doc": doc_info + "doc": doc_info, + "_retry_on_conflict": 5 } actions.append(action) if len(actions) == 5000: - try: - helpers.bulk(es206_client.elastic_client, actions) - count += len(actions) - log.info("更新:" + str(count)) - except Exception as e: - log.error(e) + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) actions.clear() if len(actions) > 0: try: @@ -112,4 +138,5 @@ def sync(city=None): log.info("完成同步相对价格排名数据...") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) diff --git a/dags/business/business/community/sell.py b/dags/business/business/community/sell.py index 408cf31..704b178 100644 --- a/dags/business/business/community/sell.py +++ b/dags/business/business/community/sell.py @@ -6,6 +6,8 @@ import pandas as pd from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor +import sys +from threading import Lock log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -14,6 +16,7 @@ es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" path = "/home/airflow/data/community" m_date = dateutils.iso_format() +mutex = Lock() def build_doc(item): @@ -49,12 +52,15 @@ def sync(city=None): names_str = "'" + names_str + "'" # 近三个月挂牌量 - # sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=93) - # seaweed_df.loc[index, 'three_months_sell_num'] = sample_num + sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=93) # 近半年挂牌量 - sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=186) - seaweed_df.loc[index, 'half_on_year_sell_num'] = sample_num + sample_half_year_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=186) + + mutex.acquire() + seaweed_df.loc[index, 'three_months_sell_num'] = sample_num + seaweed_df.loc[index, 'half_on_year_sell_num'] = sample_half_year_num + mutex.release() for index, item in seaweed_df.iterrows(): pool.submit(deal, index, item) @@ -84,4 +90,5 @@ def sync(city=None): log.info(f"完成同步出售信息至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/community/total_price.py b/dags/business/business/community/total_price.py index 3b6f867..254fec3 100644 --- a/dags/business/business/community/total_price.py +++ b/dags/business/business/community/total_price.py @@ -3,10 +3,9 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils from elasticsearch import helpers -from concurrent.futures import ThreadPoolExecutor -import re +import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -25,59 +24,20 @@ def build_doc(item): return doc_dict -# def avg_price_map_deal(x): -# avg_price_map[x['city'] + x['region'] + x['name']] = x['avg_price'] -# return x - - -def deal(seaweed_df, index, item): - if index % 1000 == 0: - log.info("已处理" + str(index)) - curr_city = item['city'] - curr_region = item['region'] - curr_name = item['name'] - names = [curr_name] - - if pd.notna(item['alias']) and '[]' != item['alias']: - alias_list = re.sub("\[|\]|'", '', item['alias']).split(',') - if len(alias_list) > 0: - for alias_item in alias_list: - names.append(alias_item) - - # 户均总价 - # if (curr_city + curr_region + curr_name) in avg_price_map: - # avg_price = avg_price_map[curr_city + curr_region + curr_name] - # if pd.notna(avg_price): - # 由于覆盖率较低,暂不过滤 - # min_avg_price = 0.8 * avg_price - # max_avg_rice = 1.2 * avg_price - total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366) - seaweed_df.loc[index, 'total_price'] = total_price - - def sync(city=None): log.info(f"开始同步户均总价信息至206 {index_name}") - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') - if pd.notna(city): - seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() - seaweed_df = seaweed_df.reset_index(drop=True) - # 加载库中均价数据 - # query_sql = "SELECT city, region, name, avg_price FROM community_summary " - # if pd.notna(city): - # query_sql += " WHERE city='" + city + "' " - # community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql) - - # community_summary_df.apply(lambda x: avg_price_map_deal(x), axis=1) - - with ThreadPoolExecutor(30) as pool: - for index, item in seaweed_df.iterrows(): - pool.submit(deal, seaweed_df, index, item) + query_sql = "SELECT city, region, name, avg_price, house_avg_area FROM community_summary WHERE avg_price > 0 and house_avg_area > 0 " + if pd.notna(city): + query_sql += " AND city='" + city + "' " + community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql) actions = list() count = 0 - for index, item in seaweed_df.iterrows(): + community_summary_df['total_price'] = community_summary_df['avg_price'] * community_summary_df['house_avg_area'] + community_summary_df['total_price'] = community_summary_df['total_price'].astype(int) + for index, item in community_summary_df.iterrows(): doc_info = build_doc(item) action = { @@ -106,4 +66,5 @@ def sync(city=None): log.info(f"完成同步户均总价信息至206 {index_name}") -sync() +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) diff --git a/dags/business/business/community/total_price_complete.py b/dags/business/business/community/total_price_complete.py index 3a94f63..a37c7b4 100644 --- a/dags/business/business/community/total_price_complete.py +++ b/dags/business/business/community/total_price_complete.py @@ -5,6 +5,7 @@ import logging import pandas as pd from goojia_utils import ElasticSearchClient, dateutils from elasticsearch import helpers +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -48,19 +49,15 @@ def save(community_factor_df): actions.clear() -def complete_by_area(community_df): - factor_df = pd.read_csv(path + '/小区户均总价面积因子已去极值.csv', sep='@', encoding='UTF-8') - community_factor_df = pd.merge(community_df, factor_df, how='left', on=['_id', 'city', 'region', 'name']) - community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor'] - return community_factor_df - - def complete_by_build_year_area(community_factor_df): build_year_area_df = pd.read_excel(path + '/全国各区各建筑年代户均面积已去极值.xlsx') build_year_area_dict = dict() for index, build_year_area in build_year_area_df.iterrows(): build_year_area_dict[ build_year_area['city'] + '|' + build_year_area['region'] + "|" + build_year_area['buildyear']] = build_year_area['factor'] + # 修复按城市跑批数据字段可能不存在问题 + if 'finish_building_date' not in community_factor_df.columns or 'total_price' not in community_factor_df.columns: + return False for index, item in community_factor_df[ (community_factor_df['total_price'].isna()) & (community_factor_df['finish_building_date'].notna())].iterrows(): finish_building_date = item['finish_building_date'] @@ -101,18 +98,22 @@ def complete_by_region_area(community_factor_df): community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor'] -def sync(): +def sync(city=None): log.info(f"开始补全户均总价信息至206 {index_name}") - community_df = es206_client.scan_by_sql_to_data_frame("select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null") + sql = "select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null " + if pd.notna(city): + sql += " and city='" + city + "'" + community_df = es206_client.scan_by_sql_to_data_frame(sql) - community_factor_df = complete_by_area(community_df) - complete_by_build_year_area(community_factor_df) - complete_by_region_area(community_factor_df) - - save(community_factor_df) + if len(community_df) > 0: + community_df['total_price'] = None + complete_by_build_year_area(community_df) + complete_by_region_area(community_df) + save(community_df) log.info(f"完成补全户均总价信息至206 {index_name}") -sync() +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) diff --git a/dags/business/business/community/trade.py b/dags/business/business/community/trade.py index ad2aa5d..d3a22d8 100644 --- a/dags/business/business/community/trade.py +++ b/dags/business/business/community/trade.py @@ -3,6 +3,7 @@ """ import logging import pandas as pd +import sys from goojia_utils import ElasticSearchClient, commutils, dateutils from elasticsearch import helpers @@ -24,14 +25,20 @@ def build_doc(item): def sync(city=None): log.info(f"开始同步成交信息至206 {index_name}") - seaweed_query = {'cat': 'district', 'status': 0} cities = None - if pd.notna(city): - seaweed_query['city'] = city - cities = [city] seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + if pd.notna(city): + cities = [city] + seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() + seaweed_df = seaweed_df.reset_index(drop=True) deal_price_df = commutils.get_deal_avg_price(cities, 3) + if len(deal_price_df) <= 0: + log.info(f"无成交,无需同步") + return + if 'match_city' not in deal_price_df.columns or 'match_region' not in deal_price_df.columns or 'match_name' not in deal_price_df.columns: + log.info(f"无成交,无需同步") + return deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']] deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name', 'avg_price': 'three_month_trade_avg_price'}) @@ -68,4 +75,5 @@ def sync(city=None): log.info(f"完成同步成交信息至206 {index_name}") -sync() +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index dade577..34e90ad 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -7,6 +7,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock +import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -86,4 +87,5 @@ def sync(city=None): log.info(f"完成同步交通至206 {index_name}") -sync() \ No newline at end of file +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/change_new_to_old_seaweed_dag.py b/dags/business/change_new_to_old_seaweed_dag.py new file mode 100644 index 0000000..14b7ae6 --- /dev/null +++ b/dags/business/change_new_to_old_seaweed_dag.py @@ -0,0 +1,17 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + "change_new_to_old_seaweed_dag", + # 每月1日1点运行 + schedule_interval="0 1 1 * *", + start_date=pendulum.datetime(2023, 2, 10, tz="Asia/Shanghai"), + catchup=True, + description="新房变二手房定时脚本" +) as dag: + t1 = BashOperator( + task_id='change_new_to_old_seaweed_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/change_new_to_old_seaweed.py', + ) + t1 \ No newline at end of file diff --git a/dags/business/community_summary_dag.py b/dags/business/community_summary_dag.py index 0e204ad..268ce91 100644 --- a/dags/business/community_summary_dag.py +++ b/dags/business/community_summary_dag.py @@ -10,84 +10,89 @@ with DAG( catchup=True, description="地址输出信息分类数据定时更新" ) as dag: - t1 = BashOperator( + comm = BashOperator( task_id='comm', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t2 = BashOperator( + price = BashOperator( task_id='price', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t3 = BashOperator( + sell = BashOperator( task_id='sell', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t4 = BashOperator( + avg_cost = BashOperator( task_id='avg_cost', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t5 = BashOperator( + trade = BashOperator( task_id='trade', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t6 = BashOperator( + total_price = BashOperator( task_id='total_price', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t6_1 = BashOperator( + total_price_complete = BashOperator( task_id='total_price_complete', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t7 = BashOperator( + traffic = BashOperator( task_id='traffic', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t7_1 = BashOperator( + business_district = BashOperator( task_id='business_district', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t8 = BashOperator( + medical = BashOperator( task_id='medical', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t9 = BashOperator( + life = BashOperator( task_id='life', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t10 = BashOperator( + relative_price_ranking = BashOperator( task_id='relative_price_ranking', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t10_1 = BashOperator( + estate_charge_complete = BashOperator( task_id='estate_charge_complete', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t10_2 = BashOperator( + estate_charge_ranking = BashOperator( task_id='estate_charge_ranking', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t11 = BashOperator( + consumption = BashOperator( task_id='consumption', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t12 = BashOperator( + person_rating = BashOperator( + task_id='person_rating', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/person_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', + ) + price_rating = BashOperator( task_id='price_rating', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t13 = BashOperator( + education = BashOperator( task_id='education', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t14 = BashOperator( + education2 = BashOperator( task_id='education2', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t15 = BashOperator( + extra = BashOperator( task_id='extra', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t16 = BashOperator( + other = BashOperator( task_id='other', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t10_1 >> t10_2 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16 + # 人均消费并行,值为空 + comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> [estate_charge_ranking, price_rating, person_rating] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other diff --git a/dags/jh/merge/merge.py b/dags/jh/merge/merge.py index 07ee54b..d3d84d3 100644 --- a/dags/jh/merge/merge.py +++ b/dags/jh/merge/merge.py @@ -4,8 +4,6 @@ # In[56]: -import os -import tarfile from goojia_utils import dateutils, ftputils, tarutils import datetime import calendar @@ -13,217 +11,7 @@ import requests import json import shutil - -# In[9]: - - -import ftplib import os -import socket -import stat -from os.path import isdir - -from numpy import unicode - -import paramiko - - -def _check_local(local): - if not os.path.exists(local): - try: - os.mkdir(local) - except IOError as err: - print(err) - - -class SftpClient: - def __init__(self): - self.client = None - self.sftp = None - pass - - def sftp_connect(self, username, password, host, port=22): - """ - 建立连接,获取sftp句柄 - :param username - :param password - :param host - :param port - :return: - """ - try: - self.client = paramiko.Transport((host, port)) - except Exception as error: - print(error) - else: - try: - self.client.connect(username=username, password=password) - except Exception as error: - print(error) - else: - self.sftp = paramiko.SFTPClient.from_transport(self.client) - - def get(self, remote, local): - """ - 从远程机器上下载文件(夹)到本地 - :param remote 远程路径 - :param local 本地路径 - """ - try: - result = self.sftp.stat(remote) - except IOError as err: - error = '[ERROR %s] %s: %s' % (err.errno, os.path.basename(os.path.normpath(remote)), err.strerror) - print(error) - else: - # 判断远程文件是否为目录 - if stat.S_ISDIR(result.st_mode): - dirname = os.path.basename(os.path.normpath(remote)) - local = os.path.join(local, dirname) - _check_local(local) - for file in self.sftp.listdir(remote): - sub_remote = os.path.join(remote, file) - sub_remote = sub_remote.replace('\\', '/') - self.get(sub_remote, local) - else: - # 拷贝文件 - if os.path.isdir(local): - local = os.path.join(local, os.path.basename(remote)) - try: - self.sftp.get(remote, local) - except IOError as err: - print(err) - else: - print('[get]', local, '<==', remote) - - # 断开连接 - def disconnect(self): - try: - self.client.close() - except Exception as error: - print(error) - error - - -class FtpClient: - - def __init__(self): - self.ftp = ftplib.FTP() - self.ftp.set_debuglevel(2) - pass - - def connect_ftp(self, host, port, timeout=60, user=None, passwd=None): - """ - 连接登录ftp - :param host: - :param port: - :param user: - :param passwd: - :return: - """ - try: - self.ftp.connect(host=host, port=port, timeout=timeout) - print("*******已经成功连接服务器FTP服务!!!") - except(socket.error, socket.gaierror) as e: - print("错误:无法访问服务器FTP服务!!!") - print(e) - exit() - try: - self.ftp.login(user=user, passwd=passwd) - except ftplib.error_perm as e: - print("登录失败,请检查用户名和密码,错误代码") - print(e) - exit() - - def cat_dir(self, filepath): - """ - 显示ftp指定目录下的文件 - :param filepath: - :return: - """ - return self.ftp.dir(filepath) - - def make_dir(self, filepath): - """ - 新建ftp下的目录,ftp目录下不存在此文件 - :param filepath: - :return: - """ - try: - filepath.encode("utf-8") - mkd = self.ftp.mkd(filepath) - return mkd - except ftplib.error_perm as e: - print("请查看目录下是否存在同名文件,异常为") - print(e) - - def remove_dir(self, dirname): - """ - 删除目录 - :param dirname - """ - try: - self.ftp.rmd(dirname) - except ftplib.error_perm as e: - print("异常:!!!目录已经被删除了") - print(e) - - def download_file(self, localpath, remotepath, filename): - """ - 从FTP指定目录上下载文件,并保存到本地指定路径 - :param localpath: linux上路径 - :param remotepath: 本地路径 - :param filename: 文件名 - """ - bufsize = 8192 # 设置缓冲区大小; - ulocalpath = unicode(localpath, "utf-8") - try: - self.ftp.cwd(remotepath) - filepath = ulocalpath + '/' - file_handler = open(filepath + filename, "wb").write # 以写模式打开本地文件,---需下载到本地的路径 - # 传一个回调函数给retrbinary() 它在每接收一个二进制数据时都会被调用 - self.ftp.retrbinary("RETR " % filename, file_handler, bufsize) # 接收服务器上文件并写入本地文件 - except ftplib.error_perm as e: - print("请检查路径是否正确,是否有权限读取,错误代码") - print(e) - - def upload(self, localpath): - """ - 上传至ftp - :param localpath: linux上路径 - """ - bufsize = 8192 - ulocalpath = unicode(localpath, "utf-8") - try: - file_handler = open(ulocalpath, "rb") - filename = os.path.split(ulocalpath)[-1] - try: - self.ftp.storbinary("stor " % filename, file_handler, bufsize) - except ftplib.error_perm as e: - print("请检查上传路径是否正确,是否连接FTP,---错误代码") - print(e) - except e: - print("请检查本地路径是否存在····") - - def lst_file(self, path): - """ - 获取指定目录下文件 - :param path - """ - s = [] - try: - for name in self.ftp.nlst(path): - name1 = unicode(name, "utf-8") - s.append(name1) - return s - except ftplib.all_errors as e: - print("请检查目录是否正确,错误代码时-----") - print(e) - - def close(self): - """ - 退出ftp - """ - self.ftp.quit() result = [] @@ -245,13 +33,18 @@ try: path = '/home/airflow/data/rh' if not os.path.exists(path): os.makedirs(path) + if not os.path.exists(path + "/" + date): + os.makedirs(path + "/" + date) - client = SftpClient() + client = ftputils.SftpClient() client.sftp_connect(username="ccb", password="ccb!@#", host="192.168.0.45", port=17495) - client.get(remote="/home/ccb_rh/output/" + date, local=path) + client.get(remote="/home/ccb_rh/output/" + date + '/000000_' + date + '_FJW.tar.gz', local=path + '/' + date) + client.get(remote="/home/ccb_rh/output/" + date + '/' + date + '.ok', local=path + '/' + date) client.disconnect() + + # In[80]: @@ -322,6 +115,29 @@ try: result.append('FJW_000000_' + cityid + '_BIGDATA_POSITION_FJW_ALL_' + date + '_0001.dat文件不存在') if 'FJW_000000_' + cityid + '_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('FJW_000000_' + cityid + '_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat文件不存在') + if 'FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_' + date + '_0001.dat文件不存在') + if 'FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat文件不存在') + if 'FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat文件不存在') + if 'FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在') + + if 'FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_' + date + '_0001.dat文件不存在') + if 'FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_' + date + '_0001.dat文件不存在') + if 'FJW_000000_000000_GUIDE_STATISTICS_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_000000_GUIDE_STATISTICS_ADD_' + date + '_0001.dat文件不存在') + if 'FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat文件不存在') + if 'FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat文件不存在') + + if 'FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_' + date + '_0001.dat文件不存在') + if len(result) == 0: result.append('融合检测完毕未发现异常文件') shutil.rmtree(path + "/" + date) diff --git a/dags/jh/para/估值模型测试城市模板.xlsx b/dags/jh/para/估值模型测试城市模板.xlsx new file mode 100644 index 0000000..9318a3c Binary files /dev/null and b/dags/jh/para/估值模型测试城市模板.xlsx differ diff --git a/dags/jh/price/no_price_community.py b/dags/jh/price/no_price_community.py new file mode 100644 index 0000000..49db8d7 --- /dev/null +++ b/dags/jh/price/no_price_community.py @@ -0,0 +1,63 @@ +import os, sys +import logging +import pandas as pd +import requests +from datetime import datetime +from goojia_utils import dateutils, oracle +from sqlalchemy import create_engine + +os.environ["NLS_LANG"] = "American_America.AL32UTF8" +os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7" +engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False) + +log = logging.getLogger(__name__) +path = "/home/airflow/data/jh/price" +if not os.path.exists(path): + os.makedirs(path) +year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_date(datetime.utcnow(), "%Y%m")) +log.info(f"测试月份:{year_month}") + +avm_df = pd.read_sql(f"select urbn_id,urbn_nm,gujia_bds_comm_num,gujiabdsvalmnycommnum from t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month} ", con=engine) +comm_df = pd.read_sql(f"select urbn_id,gujia_admndiv_nm,gujia_comm_id,comm_nm from base.t0831_bigdata_community_gj_h where gujia_bds_bds_st=0 ", con=engine) +price_df = pd.read_sql(f"select gujia_bds_id as gujia_comm_id,gujia_comm_basic_prc from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month} ", con=engine) +comm_df = comm_df.merge(avm_df, on="urbn_id", how="left") +comm_df = comm_df.merge(price_df, on="gujia_comm_id", how="left") +no_price_comm_df = comm_df[comm_df.gujia_comm_basic_prc.isna()][['urbn_id','urbn_nm','gujia_admndiv_nm','gujia_comm_id','comm_nm']] +avm_df.loc[:,'无均价数量'] = avm_df['gujia_bds_comm_num'] - avm_df['gujiabdsvalmnycommnum'] +avm_df.loc[:,'num'] = 60 - avm_df['gujiabdsvalmnycommnum'] +avm_df = avm_df[avm_df['无均价数量'] > 0] +avm_df = avm_df[['urbn_id','urbn_nm','num']] + +file_name = f"{year_month}无价格小区统计数据.xlsx" +aim_file_name = path + '/' + file_name +with pd.ExcelWriter(aim_file_name) as writer: + no_price_comm_df.to_excel(writer, sheet_name="无价格小区列表", columns=['urbn_id','urbn_nm','gujia_admndiv_nm','gujia_comm_id','comm_nm'], header=["城市id", "城市名称","区域","小区id","小区名称"], index=False) + avm_df.to_excel(writer, sheet_name="待补充数量", columns=['urbn_id','urbn_nm','num'], header=["城市id","城市名称","待补充数量"],index=False) + +log.info("无价格小区统计数据结束") +def up_to_nextcloud(): + """ + 上传统计信息至nextcloud + """ + from nextcloud import NextCloud + NEXTCLOUD_URL = "https://nextcloud.fangjia.com" + NEXTCLOUD_USERNAME = "shenjian@fangjia.com" + NEXTCLOUD_PASSWORD = "goojia123456" + nextcloud_file_name = '共享/建行/' + file_name + nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD) + nxc.upload_file(aim_file_name, nextcloud_file_name) + +def send_msg(): + infos = "【准确度测评】无价格小区统计数据生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情" + data = { + "msgtype": "text", + "text": { + "content": infos + } + } + # 发送消息至群 + requests.post( + url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736', + json=data) +up_to_nextcloud() +send_msg() \ No newline at end of file diff --git a/dags/jh/price/price_deviation.py b/dags/jh/price/price_deviation.py new file mode 100644 index 0000000..ed00a3b --- /dev/null +++ b/dags/jh/price/price_deviation.py @@ -0,0 +1,74 @@ +import os, sys +import logging +import pandas as pd +import requests +from datetime import datetime +from goojia_utils import dateutils, oracle +from sqlalchemy import create_engine + +os.environ["NLS_LANG"] = "American_America.AL32UTF8" +os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7" +engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False) + +log = logging.getLogger(__name__) +path = "/home/airflow/data/jh/price" +if not os.path.exists(path): + os.makedirs(path) +month_now = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now()), '%Y%m') +month_last = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-1), '%Y%m') +month_before = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-2), '%Y%m') + +type = sys.argv[1] if len(sys.argv) > 1 else '' +log.info("输出类型" + type) + +# 新增支持上一月与前月对比 +if type == month_last: + month_now = month_last + month_last = month_before +log.info(f"测试月份:{month_now}") + +price_df = pd.read_sql(f"select urbn_id,urbn_nm,gujia_admndiv_nm,gujia_bds_id as gujia_comm_id,comm_nm,gujia_bds_clmtd,gujia_comm_basic_prc from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={month_now} ", con=engine) +deal_df = pd.read_sql(f"select gujia_bds_mdl_id,gujia_comm_id,gujia_bds_trdtckt_prc from base.t0831_bigdata_deal_gj_h where to_char(txn_dt,'yyyyMM')='{month_last}' ", con=engine) +deal_price_df = deal_df.merge(price_df, on="gujia_comm_id", how="left") +deal_price_df["ppe"] = round((deal_price_df["gujia_comm_basic_prc"] - deal_price_df['gujia_bds_trdtckt_prc']) / deal_price_df['gujia_bds_trdtckt_prc'], 4) +deal_price_df = deal_price_df[abs(deal_price_df['ppe']) > 0.15] + +# third_df = pd.read_sql(f"select gujia_trdpt_avprc_id,gujia_bds_id as gujia_comm_id,gujia_dtsr_dsc,gujia_bds_comm_avprc from base.t0831_bigdata_third_party_price_gj_h where gujiabds_avprc_dt_val={month_last} ", con=engine) +# third_price_df = third_df.merge(price_df, on="gujia_comm_id", how="left") +# third_price_df["ppe"] = round((third_price_df["gujia_comm_basic_prc"] - third_price_df['gujia_bds_comm_avprc']) / third_price_df['gujia_bds_comm_avprc'], 4) +# third_price_df = third_price_df[abs(third_price_df['ppe']) > 0.15] + + +file_name = f"{month_now}测试偏差较大案例统计数据.xlsx" +aim_file_name = path + '/' + file_name +with pd.ExcelWriter(aim_file_name) as writer: + deal_price_df.to_excel(writer, sheet_name="成交", columns=['urbn_id','urbn_nm','gujia_bds_mdl_id','gujia_admndiv_nm','gujia_comm_id','comm_nm','gujia_bds_clmtd','gujia_comm_basic_prc','gujia_bds_trdtckt_prc','ppe'], header=["城市id", "城市名称","成交id","区域","小区id","小区名称",'计算类型','计算价格','成交价格','PPE'], index=False) + # third_price_df.to_excel(writer, sheet_name="第三方", columns=['urbn_id','urbn_nm','gujia_trdpt_avprc_id','gujia_admndiv_nm','gujia_comm_id','comm_nm','gujia_bds_clmtd','gujia_comm_basic_prc','gujia_bds_comm_avprc','PPE','gujia_dtsr_dsc'], header=["城市id", "城市名称","第三方id","区域","小区id","小区名称",'计算类型','计算价格','第三方价格','PPE','来源'],index=False) + +log.info("偏差较大案例统计数据结束") +def up_to_nextcloud(): + """ + 上传统计信息至nextcloud + """ + from nextcloud import NextCloud + NEXTCLOUD_URL = "https://nextcloud.fangjia.com" + NEXTCLOUD_USERNAME = "shenjian@fangjia.com" + NEXTCLOUD_PASSWORD = "goojia123456" + nextcloud_file_name = '共享/建行/' + file_name + nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD) + nxc.upload_file(aim_file_name, nextcloud_file_name) + +def send_msg(): + infos = "【准确度测评】测试偏差较大案例统计数据生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情" + data = { + "msgtype": "text", + "text": { + "content": infos + } + } + # 发送消息至群 + requests.post( + url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736', + json=data) +up_to_nextcloud() +send_msg() \ No newline at end of file diff --git a/dags/jh/price/price_evaluation.py b/dags/jh/price/price_evaluation.py new file mode 100644 index 0000000..6011509 --- /dev/null +++ b/dags/jh/price/price_evaluation.py @@ -0,0 +1,129 @@ +import os, sys +import logging +import pandas as pd +import requests +from datetime import datetime +from goojia_utils import dateutils, oracle +from sqlalchemy import create_engine + +os.environ["NLS_LANG"] = "American_America.AL32UTF8" +os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7" +engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False) + +important_cities =["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"] +city_order = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx") + + +def convert_calc_type(cale_type): + if cale_type == 1: + return "指导价" + elif cale_type == 2: + return "样本" + elif cale_type == 3: + return "神经网络" + + +fitment_code_map = {"0601": "毛坯", "0602": "简装", "0603": "精装"} + + +def convert_fitment_code(fitment_code): + if pd.isna(fitment_code): + return None + fitment_code = str(fitment_code) + if fitment_code in fitment_code_map: + return fitment_code_map[fitment_code] + return None + + +house_aspect_code_map = {"0301": "南北", "0302": "东南", "0303": "南", "0304": "东西", "0305": "东", "0306": "西南", "0307": "西", + "0308": "北", "0309": "东北", + "0310": "西北"} + + +def convert_house_aspect_code(house_aspect_code): + if pd.isna(house_aspect_code): + return None + house_aspect_code = str(house_aspect_code) + if house_aspect_code in house_aspect_code_map: + return house_aspect_code_map[house_aspect_code] + return None + +log = logging.getLogger(__name__) +path = "/home/airflow/data/jh/price" +if not os.path.exists(path): + os.makedirs(path) +year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_date(datetime.utcnow(), "%Y%m")) +log.info(f"测试月份:{year_month}") +avm_evaluation_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_bds_yrmo': 'year_month', 'gujia_bds_comm_num': 'community_num', 'gujiabdsvalmnycommnum': 'has_price_community_num', 'gujia_prc_cvr_rate': 'price_coverage', 'gujiabdsmnldpccommnum': 'artificial_calc_num', 'gujia_mnl_drc_prc_pct': 'artificial_calc_coverage', 'gujiabdssamp_comm_num': 'sample_calc_num', 'gujiaext_samp_clc_pct': 'sample_calc_coverage', 'gujiabdsnervetncomnum': 'ann_calc_num', 'gujianervethrunclcpct': 'ann_calc_coverage', 'gujiabds_tst_case_num': 'test_num', 'gujiabdsmdltstcommnum': 'deal_test_comm_num', 'gujiatst_comm_num_pct': 'test_comm_coverage', 'gujia_ppe15_cvr_rate': 'ppe15_coverage', 'gujia_rto_mdn': 'rate_median', 'gujia_rto_lv_san_ceft': 'coefficient', 'gujiabdsmdltstcasenum': 'deal_test_case_num', 'gujiamdltstcommnumpct': 'deal_test_comm_coverage', 'gujiamdltst_ppe15_pct': 'deal_ppe15_coverage', 'gujiamdlrtolvsan_ceft': 'deal_coefficient', 'gujia_mdl_md_bit_rto': 'deal_rate_median', 'gujiabdstrdpttcasenum': 'third_test_case_num', 'gujiabdstrdpttcommnum': 'third_test_comm_num', 'gujiatrdpttcommnumpct': 'third_test_comm_coverage', 'gujiatrdpttstppe15pct': 'third_ppe15_coverage', 'gujiatrdptrolvsanceft': 'third_coefficient', 'gujiatrdpt_md_bit_rto': 'third_rate_median', 'data_crt_tm': 'createtime', 'data_udt_tm': 'updatetime', 'gujia_src_co': 'sourcecode'} +avm_evaluation_item_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_rgon_id': 'distid', 'gujia_rgon_nm': 'distname', 'gujia_comm_id': 'commid', 'gujia_comm_nm': 'commname', 'gujiabds_clc_yrmo_val': 'year_month', 'gujia_bds_clc_avprc': 'calc_avg_price', 'gujia_bds_avprc': 'avg_price', 'gujia_bds_clc_tp': 'calc_type', 'gujia_bds_rlb': 'confidence', 'gujiabds_to_mkprc_mny': 'target_avg_price', 'mdl_dt': 'dealdate', 'gujiabds_valt_unitprc': 'unit_price', 'gujia_area': 'buildarea', 'gujia_dcrt_val': 'fitment', 'gujia_twrd_val': 'houseaspect', 'gujia_bds_floor_num': 'floornum', 'gujiabdstot_floor_num': 'floorcnt', 'gujia_data_tp': 'data_type', 'gujia_rto': 'rate', 'gujia_mns_rto': 'pe', 'data_crt_tm': 'createtime'} +avm_evaluation = pd.read_sql(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month}", con=engine) +avm_evaluation_item = pd.read_sql(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val={year_month}", con=engine) +avm_evaluation.rename(columns=avm_evaluation_column_map, inplace=True) +avm_evaluation_item.rename(columns=avm_evaluation_item_column_map, inplace=True) +community_base_price = pd.read_sql(f'''SELECT gujia_bds_id as commid,urbn_nm as cityname,gujia_admndiv_nm as distname,comm_nm as commname, + gujia_comm_basic_prc as comm_avg_price,gujia_bds_cfdc_dgr as confidence,gujia_bds_clmtd as calc_type from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month}''', + con=engine) +communities = pd.read_sql(f"select gujia_comm_id as COMMID from base.t0831_bigdata_community_gj_h", con=engine) + +community_base_price = community_base_price[community_base_price.commid.isin(communities["commid"])] +avm_evaluation_item["fitment"] = avm_evaluation_item["fitment"].map(convert_fitment_code) +avm_evaluation_item["houseaspect"] = avm_evaluation_item["houseaspect"].map(convert_house_aspect_code) +avm_evaluation_item["calc_type"] = avm_evaluation_item["calc_type"].map(convert_calc_type) +community_base_price["calc_type"] = community_base_price["calc_type"].map(convert_calc_type) +avm_evaluation_item["dealdate"] = avm_evaluation_item["dealdate"].apply(lambda x : x.strftime("%Y-%m-%d") if pd.notna(x) else None) +avm_evaluation_item_third = avm_evaluation_item[avm_evaluation_item.data_type=='第三方'] +avm_evaluation_item_deal = avm_evaluation_item[avm_evaluation_item.data_type=='成交'] +import_avm_evaluation_item_third = avm_evaluation_item_third[avm_evaluation_item_third.cityname.isin(important_cities)] +import_avm_evaluation_item_deal = avm_evaluation_item_deal[avm_evaluation_item_deal.cityname.isin(important_cities)] +import_avm_evaluation_item_deal.loc[:, "deal_total_price"] = import_avm_evaluation_item_deal["buildarea"] * import_avm_evaluation_item_deal["target_avg_price"] +import_avm_evaluation_item_deal.loc[:, "deal_total_price"] = import_avm_evaluation_item_deal["deal_total_price"].apply(lambda x : int(x)) +import_avm_evaluation_item_deal.loc[:, "total_price"] = import_avm_evaluation_item_deal["buildarea"] * import_avm_evaluation_item_deal["calc_avg_price"] +import_avm_evaluation_item_deal.loc[:, "total_price"] = import_avm_evaluation_item_deal["total_price"].apply(lambda x : int(x)) +import_community_base_price = community_base_price[community_base_price.cityname.isin(important_cities)] +normal_community_base_price = community_base_price[~(community_base_price.cityname.isin(important_cities))] +import_avm_evaluation = avm_evaluation[avm_evaluation.cityname.isin(important_cities)] +avm_evaluation = avm_evaluation.merge(city_order, on="cityname", how="left") +import_community_base_price = import_community_base_price.merge(city_order, on="cityname", how="left") +normal_community_base_price = normal_community_base_price.merge(city_order, on="cityname", how="left") +import_avm_evaluation_item_deal = import_avm_evaluation_item_deal.merge(city_order, on="cityname", how="left") +import_avm_evaluation_item_third = import_avm_evaluation_item_third.merge(city_order, on="cityname", how="left") +avm_evaluation.sort_values(by="cityrank", inplace=True) +import_community_base_price.sort_values(by="cityrank", inplace=True) +normal_community_base_price.sort_values(by="cityrank", inplace=True) +import_avm_evaluation_item_deal.sort_values(by="cityrank", inplace=True) +import_avm_evaluation_item_third.sort_values(by="cityrank", inplace=True) + +file_name = f"{year_month}估值模型测试.xlsx" +aim_file_name = path + '/' + file_name +with pd.ExcelWriter(aim_file_name) as writer: + avm_evaluation.to_excel(writer, sheet_name="92个城市估值概况", columns=["cityname","community_num","price_coverage","sample_calc_coverage","ann_calc_coverage","artificial_calc_coverage","deal_test_case_num","deal_ppe15_coverage","deal_test_comm_num","deal_test_comm_coverage","deal_rate_median","deal_coefficient","third_test_comm_num","third_test_comm_coverage","third_ppe15_coverage","third_rate_median","third_coefficient"], header=["城市名称","小区量","可输出均价小区占比","样本计算占比","神经网络计算占比","人工指导价占比","成交测试案例总量","成交测试PPE15占比","成交测试小区量","成交测试小区量占比","成交比率中位数","成交比率离散系数","第三方均价测试小区量","第三方均价测试小区量占比","第三方均价测试PPE15占比","第三方比率中位数","第三方比率离散系数"], index=False) + import_community_base_price.to_excel(writer, sheet_name="测试10个城市估值数据", columns=["cityname","distname","commname","comm_avg_price","confidence","calc_type"], header=["城市","区域","小区名","估值小区均价","置信度","估值方式"],index=False) + import_avm_evaluation_item_deal.to_excel(writer, sheet_name="10个城市成交测试案例", columns=['cityname','distname', 'commname', 'unit_price', 'confidence', 'calc_type','total_price', 'deal_total_price', 'target_avg_price','pe',"dealdate","fitment","houseaspect","floornum","floorcnt","buildarea"], header=["城市","区域","小区名","修正均价","置信度","估值方式","小区估价","对标成交价","对标均价","PPE","dealdate","fitment","houseaspect","floornum","floorcnt","buildarea"], index=False) + import_avm_evaluation_item_third.to_excel(writer, sheet_name="10个城市第三方均价测试", columns=["cityname","distname","commname","avg_price","confidence","calc_type","target_avg_price","pe"], header=["城市","区域","小区名","小区均价","置信度","估值方式","对标第三方均价","PPE"],index=False) +log.info("估值结果测试结束") +def up_to_nextcloud(): + """ + 上传统计信息至nextcloud + """ + from nextcloud import NextCloud + NEXTCLOUD_URL = "https://nextcloud.fangjia.com" + NEXTCLOUD_USERNAME = "shenjian@fangjia.com" + NEXTCLOUD_PASSWORD = "goojia123456" + nextcloud_file_name = '共享/建行/' + file_name + nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD) + nxc.upload_file(aim_file_name, nextcloud_file_name) + +def send_msg(): + infos = "【准确度测评】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情" + data = { + "msgtype": "text", + "text": { + "content": infos + } + } + # 发送消息至群 + requests.post( + url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736', + json=data) +up_to_nextcloud() +send_msg() \ No newline at end of file diff --git a/dags/jh/price/price_exception.py b/dags/jh/price/price_exception.py index f6634d6..89f4c60 100644 --- a/dags/jh/price/price_exception.py +++ b/dags/jh/price/price_exception.py @@ -7,9 +7,10 @@ import logging import os import sys import requests +from sqlalchemy import create_engine os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7" - +engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False) def simple_calc(result_df): """ 新增是否有样本列 @@ -25,8 +26,9 @@ def simple_calc(result_df): if len(actions) == 800: ins = "','".join(actions) ins = "'" + ins + "'" - simple_temp_df = oracle.scan_by_sql_to_data_frame("select distinct commid from bigdata_forsale_gj where " + - " publishtime> TO_DATE('" + month_before + "','YYYY-MM-DD') and commid in (" + ins + ")") + simple_temp_df = pd.read_sql( + "select distinct gujia_comm_id as commid from base.t0831_bigdata_forsale_gj_h where " + + " anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")", con=engine) simple_df = pd.concat([simple_df, simple_temp_df]) actions.clear() @@ -34,8 +36,9 @@ def simple_calc(result_df): if len(actions) > 0: ins = "','".join(actions) ins = "'" + ins + "'" - simple_temp_df = oracle.scan_by_sql_to_data_frame("select distinct commid from bigdata_forsale_gj where " + - " publishtime> TO_DATE('" + month_before + "','YYYY-MM-DD') and commid in (" + ins + ")") + simple_temp_df = pd.read_sql( + "select distinct gujia_comm_id as commid from base.t0831_bigdata_forsale_gj_h where " + + " anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")", con=engine) simple_df = pd.concat([simple_df, simple_temp_df]) actions.clear() simple_df['has_sample'] = 1 @@ -85,8 +88,8 @@ if type == month_last: month_now = month_last month_last = month_before -df = oracle.scan_by_sql_to_data_frame("SELECT id, cityname, distname, commid, commname, calc_type, confidence, avg_price, year_month FROM community_base_price_gj WHERE year_month IN('"+month_now+"', '"+month_last+"') " + - " ORDER BY cityname, distname, commid, commname, year_month") +df = pd.read_sql("SELECT gujia_bss_prc_id as id, urbn_nm as cityname, gujia_admndiv_nm as distname, gujia_bds_id as commid, comm_nm as commname, gujia_bds_clmtd as calc_type, gujia_bds_cfdc_dgr as confidence, gujia_comm_basic_prc as avg_price, gujiabds_clc_yrmo_val as year_month FROM t_app_p9pha_plda.avm_community_base_price_gj_h WHERE gujiabds_clc_yrmo_val IN('"+month_now+"', '"+month_last+"') " + + " ORDER BY urbn_nm, gujia_admndiv_nm, gujia_bds_id, comm_nm, gujia_bds_clmtd", con=engine) six_map = dict() five_map = dict() @@ -143,7 +146,7 @@ for index, item in city_count_df.iterrows(): city_count_df = city_count_df.rename(columns={'小区ID': '小区数量'}) city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度') -excel_writer.save() +#excel_writer.save() excel_writer.close() up_to_nextcloud() diff --git a/dags/jh/price_exception_dag.py b/dags/jh/price_exception_dag.py index 9ac94ac..68664ee 100644 --- a/dags/jh/price_exception_dag.py +++ b/dags/jh/price_exception_dag.py @@ -8,10 +8,23 @@ with DAG( schedule_interval="0 10 23 * *", start_date=pendulum.datetime(2022, 8, 23, tz="Asia/Shanghai"), catchup=True, - description="建行每月价格异常检测脚本,默认检测本月与上月差异, 若对比上月与前月价格,设置type=\"202210\"(前月日期即可) 若输出样本存在与否,设置type=sample" + description="建行每月价格异常检测脚本,默认检测本月与上月差异, 若对比上月与前月价格,设置year_month=\"202210\"(前月日期即可) 若输出样本存在与否,设置year_month=sample" ) as dag: t1 = BashOperator( - task_id='price_exception', - bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + task_id='price_evaluation', + bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_evaluation.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}', ) - t1 + t2 = BashOperator( + task_id='price_exception', + bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}', + ) + t3 = BashOperator( + task_id='no_price_community', + bash_command='python /opt/airflow/dags/repo/dags/jh/price/no_price_community.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}', + ) + t4 = BashOperator( + task_id='price_deviation', + bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_deviation.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}', + ) + + t1 >> t2 >> t3 >> t4 diff --git a/dags/stat/evaluate/district_sell_property.py b/dags/stat/evaluate/district_sell_property.py index 46c13d6..f28477a 100644 --- a/dags/stat/evaluate/district_sell_property.py +++ b/dags/stat/evaluate/district_sell_property.py @@ -2,7 +2,7 @@ 统计近三个月、近一年样本量及小区属性 """ import logging -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils, commutils import pandas as pd import datetime import sys @@ -24,37 +24,8 @@ seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0 .batch_size(10000))) seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'}) -def sell_stat(): - """ - 最近一年有样本小区占比 最近3月有样本小区占比 - """ - df = pd.read_excel(aim_file_name) - sell_seaweed_df = es206_client.scan_by_sql_to_data_frame( - "select city, region, district_name, p_date from latest_seawater where p_date > " + start_time + " and cat='sell' ") - # 小区别名升级为单独df - actions = list() - for index, item in seaweed_df.iterrows(): - if type(item['alias']) == list: - for name in item['alias']: - actions.append({'_id': item['_id'], 'city': item['city'], 'name': name, 'region': item['region']}) - alias_df = pd.DataFrame(actions) - - sell_seaweed_rename_df = sell_seaweed_df[['city', 'region', 'district_name']] - sell_seaweed_rename_df = sell_seaweed_rename_df.rename(columns={'district_name': 'name'}) - sell_seaweed_merge_df = pd.merge(sell_seaweed_rename_df, alias_df, how='left', on=['city', 'region', 'name']) - sell_seaweed_merge_df['count'] = sell_seaweed_merge_df[sell_seaweed_merge_df['_id'].notna()].groupby('_id')[ - '_id'].transform('size') - sell_seaweed_merge_df = sell_seaweed_merge_df[['city', 'region', '_id', 'count']] - sell_seaweed_merge_df = sell_seaweed_merge_df.drop_duplicates(subset=['city', 'region', '_id']) - sell_seaweed_ok_df = sell_seaweed_merge_df[sell_seaweed_merge_df['count'] > sell_num] - sell_seaweed_ok_df['city_seaweed_count'] = sell_seaweed_ok_df.groupby('city')['city'].transform('size') - city_seaweed_count_df = sell_seaweed_ok_df[['city', 'city_seaweed_count']].drop_duplicates( - subset=['city', 'city_seaweed_count']) - city_seaweed_count_df = city_seaweed_count_df.rename(columns={'city': '城市', 'city_seaweed_count': '最近3月有样本小区占比'}) - df = pd.merge(df, city_seaweed_count_df, how='left', on=['城市']) - df['最近3月有样本小区占比'] = round(df['最近3月有样本小区占比'] * 100 / df['小区量'], 2) - df.to_excel(aim_file_name, index=False) +def get_sell(df, daysBefore=365): cities = df[['城市']].drop_duplicates(keep='first')['城市'].values.tolist() city_one_year_count = list() for city in cities: @@ -79,8 +50,7 @@ def sell_stat(): names_str = "','".join(names) names_str = "'" + names_str + "'" try: - sql = "select count(*) from latest_seawater where city='" + city + "' and region='" + region + "' and cat='sell' and district_name in(" + names_str + ") and p_date > " + one_year_time - num = es206_client.search_by_sql(sql)['hits']['total']['value'] + num = commutils.get_sample(city, region, names_str, daysBefore=daysBefore) if num > sell_num: count += 1 total_num += num @@ -92,9 +62,25 @@ def sell_stat(): print(city + "已处理完, 近一年有样本小区数量" + str(count) + " 近一年的样本总量" + str(total_num)) result_df = pd.DataFrame(city_one_year_count) + return result_df + + +def sell_stat(): + """ + 最近一年有样本小区占比 最近3月有样本小区占比 + """ + df = pd.read_excel(aim_file_name) + result_df = get_sell(df, 365) result_df = result_df[['城市', '近一年有样本小区数量', '近一年的样本总量']] df = pd.merge(df, result_df, how='left', on=['城市']) - df['最近一年有样本小区占比'] = round(df['近一年有样本小区数量'] * 100 / df['小区量'], 2) + df['近一年有样本小区占比'] = round(df['近一年有样本小区数量'] * 100 / df['小区量'], 2) + + result_df = get_sell(df, 93) + result_df = result_df.rename(columns={'近一年有样本小区数量': '近三月有样本小区数量'}) + result_df = result_df[['城市', '近三月有样本小区数量']] + df = pd.merge(df, result_df, how='left', on=['城市']) + df['近三月有样本小区占比'] = round(df['近三月有样本小区数量'] * 100 / df['小区量'], 2) + df.to_excel(aim_file_name, index=False)