From 351bad8b57a8c99c51801fd171cbdcff7ed892ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 9 Jun 2025 14:36:20 +0800 Subject: [PATCH 01/16] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/fill_company_comm.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dags/company/company/fill_company_comm.py b/dags/company/company/fill_company_comm.py index 2a13f79..8d65df0 100644 --- a/dags/company/company/fill_company_comm.py +++ b/dags/company/company/fill_company_comm.py @@ -14,7 +14,7 @@ def sync(province): index="didata_company", body={"index": {"refresh_interval": "-1"}} ) - ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name from didata_company where province='{province}'") + ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name, committee from didata_company where province='{province}'") if len(ids) > 0: company_ids = ids['_id'].to_list() else: @@ -92,6 +92,9 @@ def sync(province): body={"index": {"refresh_interval": "5s"}} ) - +flag = False for province in areautils.get_all_province(): - sync(province) \ No newline at end of file + if '西藏自治区' == province: + flag = True + if flag: + sync(province) \ No newline at end of file From 6375437b7f43fd3938e73e3dbf280ec2782dfd51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 9 Jun 2025 14:46:03 +0800 Subject: [PATCH 02/16] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/sample/samples_notice.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dags/jh/sample/samples_notice.py b/dags/jh/sample/samples_notice.py index 0e0e98c..04824b3 100644 --- a/dags/jh/sample/samples_notice.py +++ b/dags/jh/sample/samples_notice.py @@ -13,7 +13,8 @@ last = dateutils.add_months(first_day, 0, -2, 0, 0) last_day = last.strftime('%Y%m%d') log.info(f"系统当前时间 : {now}") - +token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736' +dingutils.send_text('【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情', token) def proc(): # 区域维度,统计2个月内样本数据量 From 0d8c221dd324bbc44fe13506cbd389e2ca303e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 9 Jun 2025 14:49:24 +0800 Subject: [PATCH 03/16] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/sample/samples_notice.py | 35 ++++++++++++++------------------ 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/dags/jh/sample/samples_notice.py b/dags/jh/sample/samples_notice.py index 04824b3..736a477 100644 --- a/dags/jh/sample/samples_notice.py +++ b/dags/jh/sample/samples_notice.py @@ -13,8 +13,6 @@ last = dateutils.add_months(first_day, 0, -2, 0, 0) last_day = last.strftime('%Y%m%d') log.info(f"系统当前时间 : {now}") -token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736' -dingutils.send_text('【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情', token) def proc(): # 区域维度,统计2个月内样本数据量 @@ -101,10 +99,6 @@ def proc(): air_file_name = path + '/' + file_name return text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new - -text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new = proc() - - def up_to_nextcloud(): """ 上传统计信息至nextcloud @@ -117,17 +111,18 @@ def up_to_nextcloud(): nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD) nxc.upload_file(air_file_name, nextcloud_file_name) - -if text != '': - with pd.ExcelWriter(air_file_name) as writer: - if not df_region_res.empty: - df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city", "region", 'count'], index=False) - if not df_root_city_res.empty: - df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root', 'city', 'count'], - index=False) - if not df_root_res.empty: - df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root', count_old, count_new], - index=False) - up_to_nextcloud() - token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736' - dingutils.send_text(text, token) +if __name__ == '__main__': + text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new = proc() + if text != '': + with pd.ExcelWriter(air_file_name) as writer: + if not df_region_res.empty: + df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city", "region", 'count'], index=False) + if not df_root_city_res.empty: + df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root', 'city', 'count'], + index=False) + if not df_root_res.empty: + df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root', count_old, count_new], + index=False) + up_to_nextcloud() + token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736' + dingutils.send_text(text, token) From ac357b1d9e103cf13d45484283b803cc5aafddd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 9 Jun 2025 14:52:19 +0800 Subject: [PATCH 04/16] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/fill_company_comm.py | 13 +- dags/company/company/fill_company_dev.py | 6 +- dags/company/company/mark_right_dev.py | 234 +++++++++++----------- 3 files changed, 131 insertions(+), 122 deletions(-) diff --git a/dags/company/company/fill_company_comm.py b/dags/company/company/fill_company_comm.py index 8d65df0..4b04b91 100644 --- a/dags/company/company/fill_company_comm.py +++ b/dags/company/company/fill_company_comm.py @@ -92,9 +92,10 @@ def sync(province): body={"index": {"refresh_interval": "5s"}} ) -flag = False -for province in areautils.get_all_province(): - if '西藏自治区' == province: - flag = True - if flag: - sync(province) \ No newline at end of file +if __name__ == '__main__': + flag = False + for province in areautils.get_all_province(): + if '西藏自治区' == province: + flag = True + if flag: + sync(province) \ No newline at end of file diff --git a/dags/company/company/fill_company_dev.py b/dags/company/company/fill_company_dev.py index ad5490f..ff8ad08 100644 --- a/dags/company/company/fill_company_dev.py +++ b/dags/company/company/fill_company_dev.py @@ -65,4 +65,8 @@ def sync(province): DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( index="didata_company", body={"index": {"refresh_interval": "5s"}} - ) \ No newline at end of file + ) + +if __name__ == '__main__': + for province in areautils.get_all_province(): + sync(province) \ No newline at end of file diff --git a/dags/company/company/mark_right_dev.py b/dags/company/company/mark_right_dev.py index 22818ad..550cd8d 100644 --- a/dags/company/company/mark_right_dev.py +++ b/dags/company/company/mark_right_dev.py @@ -6,125 +6,129 @@ import re import logging log = logging.getLogger(__name__) -""" -对于没有围栏结果的,通过match_phrase匹配出来的开发区一定正确,如果有围栏出来的开发区,则企业经纬度不正确 -""" +def sync(): + """ + 对于没有围栏结果的,通过match_phrase匹配出来的开发区一定正确,如果有围栏出来的开发区,则企业经纬度不正确 + """ -dev_alias = {'福建永定工业园区': '永定区永定工业园', '福建漳州古雷港经济开发区': '古雷港经济开发区', '福建莆田湄洲湾北岸经济开发区': '湄洲湾北岸经济开发区', - '福建南平工业园区': '南平工业园区', '泉州台商投资区': '泉州市台商投资区', '福建洛江经济开发区': '洛江园区', '福建武平工业园区': '武平工业园', - '福建晋江经济开发区': '晋江市经济开发区', '漳州台商投资区': '龙池开发区', '莆田高新技术产业开发区': '莆田市高新技术', - '莆田高新技术产业开发区': '莆田市涵江区高新', '绥安工业开发区': '绥安工业园', '绥安工业开发区': '黄仓开发区', '绥安工业开发区': '黄仓工业园', - '绥安工业开发区': '旧镇工业园', '绥安工业开发区': '大南坂开发区', '绥安工业开发区': '大南坂工业开发区', '福建龙州工业园区': '龙州工业园', - '长汀经济开发区':'腾飞开发区', '福建福州金山工业园区': '金山投资区', '厦门象屿综合保税区': '厦门现代物流园', '福清融侨经济技术开发区': '融侨经济', - '福建洛江经济开发区': '万安开发区'} -dev_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame("select province, city, district, name, alias from development_zone where province='福建省'") -# 不要去重,有可能存在别名, 如福建云霄常山经济开发区 -#dev_df = dev_df[dev_df['name'] == '福建政和经济开发区'] -actions = list() -for dev in dev_df.itertuples(): - if dev.name in ['中国(福建)自由贸易试验区厦门片区', '中国(福建)自由贸易试验区福州片区', '中国(福建)自由贸易试验区平潭片区']: - # 中国(福建)自由贸易试验区 已包含,防止反复更新 - continue - if dev.name in ['政和经济开发区']: - continue + dev_alias = {'福建永定工业园区': '永定区永定工业园', '福建漳州古雷港经济开发区': '古雷港经济开发区', '福建莆田湄洲湾北岸经济开发区': '湄洲湾北岸经济开发区', + '福建南平工业园区': '南平工业园区', '泉州台商投资区': '泉州市台商投资区', '福建洛江经济开发区': '洛江园区', '福建武平工业园区': '武平工业园', + '福建晋江经济开发区': '晋江市经济开发区', '漳州台商投资区': '龙池开发区', '莆田高新技术产业开发区': '莆田市高新技术', + '莆田高新技术产业开发区': '莆田市涵江区高新', '绥安工业开发区': '绥安工业园', '绥安工业开发区': '黄仓开发区', '绥安工业开发区': '黄仓工业园', + '绥安工业开发区': '旧镇工业园', '绥安工业开发区': '大南坂开发区', '绥安工业开发区': '大南坂工业开发区', '福建龙州工业园区': '龙州工业园', + '长汀经济开发区':'腾飞开发区', '福建福州金山工业园区': '金山投资区', '厦门象屿综合保税区': '厦门现代物流园', '福清融侨经济技术开发区': '融侨经济', + '福建洛江经济开发区': '万安开发区'} + dev_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame("select province, city, district, name, alias from development_zone where province='福建省'") + # 不要去重,有可能存在别名, 如福建云霄常山经济开发区 + #dev_df = dev_df[dev_df['name'] == '福建政和经济开发区'] + actions = list() + for dev in dev_df.itertuples(): + if dev.name in ['中国(福建)自由贸易试验区厦门片区', '中国(福建)自由贸易试验区福州片区', '中国(福建)自由贸易试验区平潭片区']: + # 中国(福建)自由贸易试验区 已包含,防止反复更新 + continue + if dev.name in ['政和经济开发区']: + continue - count = 0 - names = list() - names.append(dev.name) - if str(dev.alias) != 'nan': - names.extend(dev.alias) - #东侨经济技术开发区 福建省宁德市东侨经济开发区塔山村塔兴路25号 - if '经济技术开发区' in dev.name: - names.append(re.sub('经济技术开发区', '经济开发区', dev.name)) - # 福州经济技术开发区 福州开发区君竹路83号科技发展中心大楼第四层R472室(自贸试验区内) - names.append(re.sub('经济技术开发区', '开发区', dev.name)) - # 长汀经济开发区 长汀县经济开发区工业新区河梁大山体 - if '经济开发区' in dev.name: - if pd.notna(dev.city) and pd.notna(dev.district): - district_short = areautils.get_district_short(dev.city, dev.district) - if pd.notna(district_short): - names.append(re.sub(district_short, dev.district, dev.name)) - # 漳州高新技术产业开发区 福建省漳州高新区九湖镇木棉村后壁山401-1号 - if '高新技术产业开发区' in dev.name: - names.append(re.sub('高新技术产业开发区', '高新区', dev.name)) - # 厦门火炬高技术产业开发区 厦门火炬高新区软件园三期溪西山尾路63号1303室-3单元之三 - if '高技术产业开发区' in dev.name: - names.append(re.sub('高技术产业开发区', '高新区', dev.name)) - #areautils.get_district_display() - # 福建诏安工业园区 福建省漳州市诏安工业园区丹诏大道15号 - if pd.notna(dev.city): - province_short = areautils.get_province(dev.city) - new_name = re.sub(province_short, '', dev.name) - if pd.notna(dev.city) and pd.notna(dev.district): - city_short = areautils.get_city_short(dev.city) - district_short = areautils.get_district_short(dev.city, dev.district) - if pd.notna(city_short) and city_short in new_name: - names.append(new_name) - if pd.notna(district_short) and district_short in new_name: - names.append(new_name) - # 福建省漳州市长泰县经济开发区林墩办事处帝君路 福建长泰经济开发区 - new_new_name = re.sub(district_short, district_short + '县', new_name) - names.append(new_new_name) - new_new_name = re.sub(district_short, district_short + '区', new_name) - names.append(new_new_name) + count = 0 + names = list() + names.append(dev.name) + if str(dev.alias) != 'nan': + names.extend(dev.alias) + #东侨经济技术开发区 福建省宁德市东侨经济开发区塔山村塔兴路25号 + if '经济技术开发区' in dev.name: + names.append(re.sub('经济技术开发区', '经济开发区', dev.name)) + # 福州经济技术开发区 福州开发区君竹路83号科技发展中心大楼第四层R472室(自贸试验区内) + names.append(re.sub('经济技术开发区', '开发区', dev.name)) + # 长汀经济开发区 长汀县经济开发区工业新区河梁大山体 + if '经济开发区' in dev.name: + if pd.notna(dev.city) and pd.notna(dev.district): + district_short = areautils.get_district_short(dev.city, dev.district) + if pd.notna(district_short): + names.append(re.sub(district_short, dev.district, dev.name)) + # 漳州高新技术产业开发区 福建省漳州高新区九湖镇木棉村后壁山401-1号 + if '高新技术产业开发区' in dev.name: + names.append(re.sub('高新技术产业开发区', '高新区', dev.name)) + # 厦门火炬高技术产业开发区 厦门火炬高新区软件园三期溪西山尾路63号1303室-3单元之三 + if '高技术产业开发区' in dev.name: + names.append(re.sub('高技术产业开发区', '高新区', dev.name)) + #areautils.get_district_display() + # 福建诏安工业园区 福建省漳州市诏安工业园区丹诏大道15号 + if pd.notna(dev.city): + province_short = areautils.get_province(dev.city) + new_name = re.sub(province_short, '', dev.name) + if pd.notna(dev.city) and pd.notna(dev.district): + city_short = areautils.get_city_short(dev.city) + district_short = areautils.get_district_short(dev.city, dev.district) + if pd.notna(city_short) and city_short in new_name: + names.append(new_name) + if pd.notna(district_short) and district_short in new_name: + names.append(new_name) + # 福建省漳州市长泰县经济开发区林墩办事处帝君路 福建长泰经济开发区 + new_new_name = re.sub(district_short, district_short + '县', new_name) + names.append(new_new_name) + new_new_name = re.sub(district_short, district_short + '区', new_name) + names.append(new_new_name) - for key in dev_alias: - if dev.name == key: - names.append(dev_alias[key]) - for name in names: - query = { - "query": { - "bool": { - "must_not": [ - { - "term": { - "dev": { - "value": dev.name + for key in dev_alias: + if dev.name == key: + names.append(dev_alias[key]) + for name in names: + query = { + "query": { + "bool": { + "must_not": [ + { + "term": { + "dev": { + "value": dev.name + } + } + }, + { + "term": { + "dev_status": { + "value": 0 + } + } } - } - }, - { - "term": { - "dev_status": { - "value": 0 + ], + "must": [ + { + "term": { + "province": { + "value": dev.province + } + } + }, + { + "match_phrase": { + "registered_address": name + } } - } + ] } - ], - "must": [ - { - "term": { - "province": { - "value": dev.province - } - } - }, - { - "match_phrase": { - "registered_address": name - } + } + } + count_data = DIDATA_DEV_ELASTIC_CLIENT.elastic_client.count(index='didata_company',body=query) + if count_data['count'] > 0: + for doc in DIDATA_DEV_ELASTIC_CLIENT.scan(index='didata_company',query=query): + action = { + "_op_type": "update", + "_index": "didata_company", + "_id": doc['_id'], + "doc": {'dev': dev.name, 'dev_status': 0} } - ] - } - } - } - count_data = DIDATA_DEV_ELASTIC_CLIENT.elastic_client.count(index='didata_company',body=query) - if count_data['count'] > 0: - for doc in DIDATA_DEV_ELASTIC_CLIENT.scan(index='didata_company',query=query): - action = { - "_op_type": "update", - "_index": "didata_company", - "_id": doc['_id'], - "doc": {'dev': dev.name, 'dev_status': 0} - } - actions.append(action) - if len(actions) == 5000: - helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions) - count += len(actions) - print(f"已更新 {dev.name} 开发区" + str(count) + "记录") - actions.clear() - if len(actions) > 0: - helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions) - count += len(actions) - print(f"已更新 {dev.name} 开发区" + str(count) + "记录") - actions.clear() \ No newline at end of file + actions.append(action) + if len(actions) == 5000: + helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions) + count += len(actions) + print(f"已更新 {dev.name} 开发区" + str(count) + "记录") + actions.clear() + if len(actions) > 0: + helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions) + count += len(actions) + print(f"已更新 {dev.name} 开发区" + str(count) + "记录") + actions.clear() + +if __name__ == '__main__': + sync() \ No newline at end of file From a09c8b2febf72939924c53ebae0953be0bc8b2b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 10 Jun 2025 09:06:10 +0800 Subject: [PATCH 05/16] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/fill_company_comm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/company/company/fill_company_comm.py b/dags/company/company/fill_company_comm.py index 4b04b91..0335a0e 100644 --- a/dags/company/company/fill_company_comm.py +++ b/dags/company/company/fill_company_comm.py @@ -95,7 +95,7 @@ def sync(province): if __name__ == '__main__': flag = False for province in areautils.get_all_province(): - if '西藏自治区' == province: + if '山东省' == province: flag = True if flag: sync(province) \ No newline at end of file From 43e9b5af1472ec0da701539a2fe9615760eec765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 10 Jun 2025 14:04:11 +0800 Subject: [PATCH 06/16] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/dag_office_rent_base_price.py | 9 +- dags/office/rent/base_price.py | 301 +++++++++++++++------- dags/office/sell/base_price.py | 19 +- dags/office/sell/fill_base_price.py | 22 +- 4 files changed, 234 insertions(+), 117 deletions(-) diff --git a/dags/office/dag_office_rent_base_price.py b/dags/office/dag_office_rent_base_price.py index dfef620..d78e609 100644 --- a/dags/office/dag_office_rent_base_price.py +++ b/dags/office/dag_office_rent_base_price.py @@ -1,9 +1,10 @@ """ 写字楼基准价计算 """ -from airflow.operators.bash import BashOperator +from airflow.operators.bash import BashOperator, PythonOperator import pendulum +from sell.base_price import calc from airflow import DAG @@ -14,9 +15,9 @@ with DAG( catchup=True, description="写字楼基准价计算" ) as dag: - t1 = BashOperator( + t1 = PythonOperator( task_id='base_price', - bash_command='python /opt/airflow/dags/repo/dags/office/rent/base_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}', + python_callable=calc, + provide_context=True ) t1 - diff --git a/dags/office/rent/base_price.py b/dags/office/rent/base_price.py index 47c95c3..6037757 100644 --- a/dags/office/rent/base_price.py +++ b/dags/office/rent/base_price.py @@ -10,7 +10,7 @@ # # 第五步:本身没有等级和类型的写字楼 # 最近10个有样本写字楼的基准价的平均值 -from goojia_utils import ElasticSearchClient, dateutils, MAIN_MONGO_CLIENT +from goojia_utils import ElasticSearchClient, dateutils, MAIN_MONGO_CLIENT, commutils import pandas as pd from datetime import datetime from pymongo import UpdateOne @@ -23,7 +23,103 @@ seaweed_df = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat') # 计算有样本写字楼的基准价 1. 取近一年,最近100个样本的中位值 -def calc(city, year_month): + +def calc(**kwargs): + """ + 写字楼基准价 + """ + # 从 Airflow 的 dag_run.conf 获取参数 + conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {} + city = conf.get('city', '') + year_month = conf.get('year_month', '') + district = conf.get('district', '') + office_name = conf.get('name', '') + + if pd.isna(office_name): + calcByCity(city, year_month) + else: + # 直接重算出最近一年的数据,保证一年有数据 + calcByOffice(city, district, office_name) + +def calcByOffice(city, district, office_name): + """ + 写字楼价格重算 + """ + query = {'cat': 'office', 'status': 0} + if pd.notna(city): + query['city'] = city + if pd.notna(district): + query['region'] = district + if pd.notna(city): + query['name'] = office_name + office = seaweed_df.find_one(query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'lng': 1, 'lat': 1, 'estate_type2': 1, 'bed_kind': 1}) + names = list() + names.append(office['name']) + if type(office['alias']) == list: + for alias_item in office['alias']: + names.append(alias_item) + names_str = "','".join(names) + names_str = "'" + names_str + "'" + + # 只查询该小区的样本 + year_time = dateutils.format_date(dateutils.datetime.datetime.now()) + one_year_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-365)) + sql = f"select * from latest_seawater where cat='office_rent' and city='{city}' and region='{district}' and district_name in ({names_str}) and p_date > {one_year_time}" + sample = es.scan_by_sql_to_data_frame(sql) + + location = { + "type": "Point", + "coordinates": [office['lng'] / math.pow(10, 10), office['lat'] / math.pow(10, 10)] + } + + date_range = pd.date_range(start=one_year_time, end=year_time, freq='M') + for date in date_range: + curr_date = int(dateutils.format_date(date, fm='%Y%m')) + # 筛选出当前月的样本 + curr_sample = sample[(sample['p_date']>=int(str(curr_date)+'01')) & (sample['p_date']<=int(str(curr_date)+'31'))] + update_office(city, pd.DataFrame(office), curr_sample, curr_date) + # 核查当月写字楼是否存在,不存在取最后6个月的平均值 + office_db = office_rent_stat_conn.find_one({'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date}) + if pd.isna(office_db): + avg_price_ret = office_rent_stat_conn.aggregate([ + { + "$match": { + "cat": "office", + "city": city, + "district": district, + "name": office_name, + "date": { "$gte": curr_date - 3, "$lte": curr_date + 3 } + } + }, + { + "$group": { + "_id": "$name", + "avg_price": { "$avg": "$avg_price" } + } + } + ]) + avg_prices = [doc["avg_price"] for doc in avg_price_ret] + # 计算平均值 + average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0 + if average_price > 0: + new_avg_price = average_price + else: + # 取样本的平均值 + new_avg_price = sample['avg_price'].mean() + if pd.notna(new_avg_price): + action = UpdateOne( + {'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date}, + {'$set': {'city': city, 'cat': 'office', 'date': curr_date, 'district': district, + 'name': office_name, + 'avg_price': new_avg_price, 'estate_type2': office['estate_type2'], + 'bed_kind': office['bed_kind'], + 'location': location, + 'calc_type': 5, 'updated_date': datetime.now()}}, + upsert=True) + office_rent_stat_conn.bulk_write([action]) + + +def calcByCity(city, year_month): """ 写字楼基准价 """ @@ -46,13 +142,12 @@ def calc(city, year_month): print("开始计算年" + str(year_month) + "写字楼基准价") sample = es.scan_by_sql_to_data_frame(sql) - query = {'cat': 'office', 'status': 0} - if pd.notna(city): - query['city'] = city - office = pd.DataFrame(list(seaweed_df.find(query).batch_size(10000))) - office = office[['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']] - office = office.explode('alias') + office = getOffice(city) + update_office(city, office, sample, year_month) + + +def update_office(city, office, sample, year_month): sample_new = sample sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( 'median') @@ -64,7 +159,6 @@ def calc(city, year_month): sample_new = sample_new[['city', 'region', 'district_name', 'day_avg_price']] sample_new = sample_new.rename(columns={'district_name': 'name'}) sample_new['alias'] = sample_new['name'] - # 使用主名与别名查找样本 merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left', on=['city', 'region', 'name']) @@ -88,7 +182,8 @@ def calc(city, year_month): estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None bed_kind = item['bed_kind'] if pd.notna(item['bed_kind']) else None action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, - {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, 'name': name, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind, 'location': location, 'calc_type': 1, 'updated_date': datetime.now()}}, @@ -100,13 +195,14 @@ def calc(city, year_month): if len(actions) > 0: office_rent_stat_conn.bulk_write(actions) actions.clear() - # 第二步 取周边3公里内同等级、同类型的最近10个有样本写字楼的基准价的平均值 - office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates(keep='first') + office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates( + keep='first') combined = pd.concat([office_info, merge_df]) diff_df = combined.drop_duplicates(subset=['city', 'region', 'name'], keep=False) actions = list() - for index, item in diff_df[(diff_df['estate_type2'].notna()) & (diff_df['bed_kind'].notna()) & (diff_df['lng'].notna()) ].iterrows(): + for index, item in diff_df[ + (diff_df['estate_type2'].notna()) & (diff_df['bed_kind'].notna()) & (diff_df['lng'].notna())].iterrows(): location = { "type": "Point", "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] @@ -159,7 +255,6 @@ def calc(city, year_month): if len(actions) > 0: office_rent_stat_conn.bulk_write(actions) actions.clear() - # 系数计算在Jupyter中 bed_kind_dict = {'甲级': 1.111888111888112, '丙级': 0.9143356643356645, '乙级': 1} radio_dict = {'创业园区': 0.7799999999999999, '商住两用': 1.0466666666666666, '标准写字楼': 1} @@ -210,13 +305,14 @@ def calc(city, year_month): count += 1 if count > 0: avg_price = round(sum_price / count, 2) - action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, - {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, - 'name': name, - 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None, - 'location': location, - 'calc_type': 3, 'updated_date': datetime.now()}}, - upsert=True) + action = UpdateOne( + {'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, + 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None, + 'location': location, + 'calc_type': 3, 'updated_date': datetime.now()}}, + upsert=True) actions.append(action) elif pd.notna(bed_kind): ret = office_rent_stat_conn.aggregate([ @@ -249,13 +345,14 @@ def calc(city, year_month): count += 1 if count > 0: avg_price = round(sum_price / count, 2) - action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, - {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, - 'name': name, - 'avg_price': avg_price, 'estate_type2': None, 'bed_kind': bed_kind, - 'location': location, - 'calc_type': 3, 'updated_date': datetime.now()}}, - upsert=True) + action = UpdateOne( + {'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, + 'avg_price': avg_price, 'estate_type2': None, 'bed_kind': bed_kind, + 'location': location, + 'calc_type': 3, 'updated_date': datetime.now()}}, + upsert=True) actions.append(action) if len(actions) == 1000: office_rent_stat_conn.bulk_write(actions) @@ -273,79 +370,85 @@ def calc(city, year_month): # 计算价格的写字楼 db_df = db_df.rename(columns={'district': 'region'}) # 使用 merge 操作并结合 indicator 参数 - merged = office_info.merge(db_df[['city', 'region', 'name']], on=['city', 'region', 'name'], how='left', - indicator=True) - - # 过滤出仅在 office_info 中存在的记录 - diff_df = merged[merged['_merge'] == 'left_only'].drop(columns=['_merge']) - - # 如果需要保持原始列顺序,可以重新排列列 - diff_df = diff_df[office_info.columns] - - actions.clear() - for index, item in diff_df.iterrows(): - try: - location = { - "type": "Point", - "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] - } - city = item['city'] - region = item['region'] - name = item['name'] - estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None - bed_kind = item['bed_kind'] - ret = office_rent_stat_conn.aggregate([ - { - '$geoNear': { - 'near': location, - 'distanceField': "distance", - 'maxDistance': 3000, - 'spherical': True - } - }, - { - '$match': { - 'date': year_month - } - }, - { - '$limit': 5 + if len(db_df) > 0: + merged = office_info.merge(db_df[['city', 'region', 'name']], on=['city', 'region', 'name'], how='left', + indicator=True) + # 过滤出仅在 office_info 中存在的记录 + diff_df = merged[merged['_merge'] == 'left_only'].drop(columns=['_merge']) + # 如果需要保持原始列顺序,可以重新排列列 + diff_df = diff_df[office_info.columns] + actions.clear() + for index, item in diff_df.iterrows(): + try: + location = { + "type": "Point", + "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] } - ]); - except Exception as e: - print("Error occurred:", str(e)) - print(item) - # 提取 avg_price 的值 - avg_prices = [doc["avg_price"] for doc in ret] - # 计算平均值 - average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0 - if average_price > 0: - avg_price = round(average_price, 2) - action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, - {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, - 'name': name, - 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None, - 'location': location, - 'calc_type': 4, 'updated_date': datetime.now()}}, - upsert=True) - actions.append(action) - if len(actions) == 1000: - office_rent_stat_conn.bulk_write(actions) - actions.clear() + city = item['city'] + region = item['region'] + name = item['name'] + estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None + bed_kind = item['bed_kind'] + ret = office_rent_stat_conn.aggregate([ + { + '$geoNear': { + 'near': location, + 'distanceField': "distance", + 'maxDistance': 3000, + 'spherical': True + } + }, + { + '$match': { + 'date': year_month + } + }, + { + '$limit': 5 + } + ]); + except Exception as e: + print("Error occurred:", str(e)) + print(item) + # 提取 avg_price 的值 + avg_prices = [doc["avg_price"] for doc in ret] + # 计算平均值 + average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0 + if average_price > 0: + avg_price = round(average_price, 2) + action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, + 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None, + 'location': location, + 'calc_type': 4, 'updated_date': datetime.now()}}, + upsert=True) + actions.append(action) + if len(actions) == 1000: + office_rent_stat_conn.bulk_write(actions) + actions.clear() if len(actions) > 0: office_rent_stat_conn.bulk_write(actions) actions.clear() -yearmonth = sys.argv[1] if len(sys.argv) > 1 else None -# for yearmonth in [ -# "202406", -# "202407", -# "202408", -# "202409", -# "202410", -# "202411", -# "202412", -# "202501" -# ]: -# calc(None, yearmonth) -calc(None, yearmonth) \ No newline at end of file + +def getOffice(city, district, name): + """ + 获取写字楼 + """ + query = {'cat': 'office', 'status': 0} + if pd.notna(city): + query['city'] = city + if pd.notna(district): + query['region'] = district + if pd.notna(city): + query['name'] = name + office = pd.DataFrame(list(seaweed_df.find(query).batch_size(10000))) + office = office[['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']] + office = office.explode('alias') + return office + + +if __name__ == "__main__": + # 测试脚本:直接运行时模拟 Airflow 的上下文 + calc(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}})) \ No newline at end of file diff --git a/dags/office/sell/base_price.py b/dags/office/sell/base_price.py index cf8c0bd..b455270 100644 --- a/dags/office/sell/base_price.py +++ b/dags/office/sell/base_price.py @@ -23,10 +23,18 @@ seaweed_df = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat') # 计算有样本写字楼的基准价 1. 取近一年,最近100个样本的中位值 -def calc(city, year_month): +def calc(**kwargs): """ 写字楼基准价 """ + # 从 Airflow 的 dag_run.conf 获取参数 + conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {} + city = conf.get('city', '') + year_month = conf.get('year_month', '') + district = conf.get('district', '') + office_name = conf.get('name', '') + + sql = "select * from latest_seawater where cat='office_sell' " if pd.notna(city): sql += "and city='" + city + "' " @@ -49,6 +57,10 @@ def calc(city, year_month): query = {'cat': 'office', 'status': 0} if pd.notna(city): query['city'] = city + if pd.notna(district): + query['region'] = district + if pd.notna(office_name): + query['name'] = office_name office = pd.DataFrame(list(seaweed_df.find(query).batch_size(10000))) office = office[['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']] office = office.explode('alias') @@ -353,5 +365,6 @@ def calc(city, year_month): print("已计算" + str(count) + "条数据") actions.clear() -yearmonth = sys.argv[1] if len(sys.argv) > 1 else None -calc(None, yearmonth) \ No newline at end of file +if __name__ == "__main__": + # 测试脚本:直接运行时模拟 Airflow 的上下文 + calc(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}})) \ No newline at end of file diff --git a/dags/office/sell/fill_base_price.py b/dags/office/sell/fill_base_price.py index d1c6b01..7245f8a 100644 --- a/dags/office/sell/fill_base_price.py +++ b/dags/office/sell/fill_base_price.py @@ -83,16 +83,16 @@ def calc(city, year_month): print("已处理{}条数据".format(count)) print("处理完毕") +if __name__ == '__main__': + yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' -yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' - -if yearmonth == '': - date = dateutils.add_months(datetime.today(), months=-1) - yearmonth = int(dateutils.format_date(date, '%Y%m')) -try: - if type(yearmonth) is str: - yearmonth = int(yearmonth) - calc(None, yearmonth) -except Exception as e: - print(e) + if yearmonth == '': + date = dateutils.add_months(datetime.today(), months=-1) + yearmonth = int(dateutils.format_date(date, '%Y%m')) + try: + if type(yearmonth) is str: + yearmonth = int(yearmonth) + calc(None, yearmonth) + except Exception as e: + print(e) From 24cbd8145423581a5c3f742a75a0e0ed0ee49073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 10 Jun 2025 14:41:41 +0800 Subject: [PATCH 07/16] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/dag_office_rent_base_price.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/office/dag_office_rent_base_price.py b/dags/office/dag_office_rent_base_price.py index d78e609..48e7783 100644 --- a/dags/office/dag_office_rent_base_price.py +++ b/dags/office/dag_office_rent_base_price.py @@ -1,7 +1,7 @@ """ 写字楼基准价计算 """ -from airflow.operators.bash import BashOperator, PythonOperator +from airflow.operators.python import PythonOperator import pendulum from sell.base_price import calc From 77cc4fb09659aaceeaebca7da0a9a62670d9eb84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 10 Jun 2025 14:46:52 +0800 Subject: [PATCH 08/16] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/dag_office_rent_base_price.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/office/dag_office_rent_base_price.py b/dags/office/dag_office_rent_base_price.py index 48e7783..2c4df53 100644 --- a/dags/office/dag_office_rent_base_price.py +++ b/dags/office/dag_office_rent_base_price.py @@ -4,7 +4,7 @@ from airflow.operators.python import PythonOperator import pendulum -from sell.base_price import calc +from office.rent.base_price import calc from airflow import DAG From cb938f81f02b4f678419aafff954071b0f3faa20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 10 Jun 2025 15:12:44 +0800 Subject: [PATCH 09/16] =?UTF-8?q?=E8=BF=98=E5=8E=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/sell/base_price.py | 28 +++++++++++++--------------- dags/office/sell/fill_price.py | 1 - 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/dags/office/sell/base_price.py b/dags/office/sell/base_price.py index b455270..f354259 100644 --- a/dags/office/sell/base_price.py +++ b/dags/office/sell/base_price.py @@ -23,18 +23,7 @@ seaweed_df = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat') # 计算有样本写字楼的基准价 1. 取近一年,最近100个样本的中位值 -def calc(**kwargs): - """ - 写字楼基准价 - """ - # 从 Airflow 的 dag_run.conf 获取参数 - conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {} - city = conf.get('city', '') - year_month = conf.get('year_month', '') - district = conf.get('district', '') - office_name = conf.get('name', '') - - +def calc(city, year_month): sql = "select * from latest_seawater where cat='office_sell' " if pd.notna(city): sql += "and city='" + city + "' " @@ -365,6 +354,15 @@ def calc(**kwargs): print("已计算" + str(count) + "条数据") actions.clear() -if __name__ == "__main__": - # 测试脚本:直接运行时模拟 Airflow 的上下文 - calc(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}})) \ No newline at end of file +if __name__ == '__main__': + yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' + + if yearmonth == '': + date = dateutils.add_months(datetime.today(), months=-1) + yearmonth = int(dateutils.format_date(date, '%Y%m')) + try: + if type(yearmonth) is str: + yearmonth = int(yearmonth) + calc(None, yearmonth) + except Exception as e: + print(e) \ No newline at end of file diff --git a/dags/office/sell/fill_price.py b/dags/office/sell/fill_price.py index 1187d07..e6484ed 100644 --- a/dags/office/sell/fill_price.py +++ b/dags/office/sell/fill_price.py @@ -13,7 +13,6 @@ es_client = ElasticSearchClient(host='192.168.0.206') office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat') office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat') - def calc(city, year_month): print(f"开始处理{city}, {year_month}") query = {'date': year_month, 'cat': 'office'} From df71f89b07c5580b80f45b01d9ad738369d67bac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 10 Jun 2025 15:31:57 +0800 Subject: [PATCH 10/16] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=86=99=E5=AD=97?= =?UTF-8?q?=E6=A5=BC=E5=87=BA=E5=94=AE=E4=BB=B7=E6=A0=BC=E9=87=8D=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/dag_office_sell_base_price.py | 17 +- dags/office/rent/base_price.py | 4 +- dags/office/sell/base_price.py | 322 ++++++++++++++-------- 3 files changed, 216 insertions(+), 127 deletions(-) diff --git a/dags/office/dag_office_sell_base_price.py b/dags/office/dag_office_sell_base_price.py index 81550cb..82e3ef9 100644 --- a/dags/office/dag_office_sell_base_price.py +++ b/dags/office/dag_office_sell_base_price.py @@ -2,6 +2,8 @@ 写字楼基准价计算 """ from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator +from office.sell.base_price import calc import pendulum @@ -14,13 +16,14 @@ with DAG( catchup=True, description="写字楼出售基准价计算" ) as dag: - t1 = BashOperator( - task_id='sell_base_price', - bash_command='python /opt/airflow/dags/repo/dags/office/sell/base_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}', - ) - t2 = BashOperator( - task_id='fill_base_price', - bash_command='python /opt/airflow/dags/repo/dags/office/sell/fill_base_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}', + t1 = PythonOperator( + task_id='base_price', + python_callable=calc, + provide_context=True ) + # t2 = BashOperator( + # task_id='fill_base_price', + # bash_command='python /opt/airflow/dags/repo/dags/office/sell/fill_base_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}', + # ) t1 >> t2 diff --git a/dags/office/rent/base_price.py b/dags/office/rent/base_price.py index 6037757..1af5161 100644 --- a/dags/office/rent/base_price.py +++ b/dags/office/rent/base_price.py @@ -102,10 +102,10 @@ def calcByOffice(city, district, office_name): # 计算平均值 average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0 if average_price > 0: - new_avg_price = average_price + new_avg_price = round(average_price, 2) else: # 取样本的平均值 - new_avg_price = sample['avg_price'].mean() + new_avg_price = round(sample['avg_price'].mean(), 2) if pd.notna(new_avg_price): action = UpdateOne( {'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date}, diff --git a/dags/office/sell/base_price.py b/dags/office/sell/base_price.py index f354259..fff8628 100644 --- a/dags/office/sell/base_price.py +++ b/dags/office/sell/base_price.py @@ -10,7 +10,7 @@ # # 第五步:本身没有等级和类型的写字楼 # 最近10个有样本写字楼的基准价的平均值 -from goojia_utils import ElasticSearchClient, dateutils, MAIN_MONGO_CLIENT +from goojia_utils import ElasticSearchClient, dateutils, MAIN_MONGO_CLIENT, commutils import pandas as pd from datetime import datetime from pymongo import UpdateOne @@ -23,7 +23,106 @@ seaweed_df = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat') # 计算有样本写字楼的基准价 1. 取近一年,最近100个样本的中位值 -def calc(city, year_month): + +def calc(**kwargs): + """ + 写字楼基准价 + """ + # 从 Airflow 的 dag_run.conf 获取参数 + conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {} + city = conf.get('city', '') + year_month = conf.get('year_month', '') + district = conf.get('district', '') + office_name = conf.get('name', '') + + if pd.isna(office_name): + calcByCity(city, year_month) + else: + # 直接重算出最近一年的数据,保证一年有数据 + calcByOffice(city, district, office_name) + +def calcByOffice(city, district, office_name): + """ + 写字楼价格重算 + """ + query = {'cat': 'office', 'status': 0} + if pd.notna(city): + query['city'] = city + if pd.notna(district): + query['region'] = district + if pd.notna(city): + query['name'] = office_name + office = seaweed_df.find_one(query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'lng': 1, 'lat': 1, 'estate_type2': 1, 'bed_kind': 1}) + names = list() + names.append(office['name']) + if type(office['alias']) == list: + for alias_item in office['alias']: + names.append(alias_item) + names_str = "','".join(names) + names_str = "'" + names_str + "'" + + # 只查询该小区的样本 + year_time = dateutils.format_date(dateutils.datetime.datetime.now()) + one_year_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-365)) + sql = f"select * from latest_seawater where cat='office_sell' and city='{city}' and region='{district}' and district_name in ({names_str}) and p_date > {one_year_time}" + sample = es.scan_by_sql_to_data_frame(sql) + + location = { + "type": "Point", + "coordinates": [office['lng'] / math.pow(10, 10), office['lat'] / math.pow(10, 10)] + } + + date_range = pd.date_range(start=one_year_time, end=year_time, freq='M') + for date in date_range: + curr_date = int(dateutils.format_date(date, fm='%Y%m')) + # 筛选出当前月的样本 + curr_sample = sample[(sample['p_date']>=int(str(curr_date)+'01')) & (sample['p_date']<=int(str(curr_date)+'31'))] + update_office(city, pd.DataFrame(office), curr_sample, curr_date) + # 核查当月写字楼是否存在,不存在取最后6个月的平均值 + office_db = office_sell_stat_conn.find_one({'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date}) + if pd.isna(office_db): + avg_price_ret = office_sell_stat_conn.aggregate([ + { + "$match": { + "cat": "office", + "city": city, + "district": district, + "name": office_name, + "date": { "$gte": curr_date - 3, "$lte": curr_date + 3 } + } + }, + { + "$group": { + "_id": "$name", + "avg_price": { "$avg": "$avg_price" } + } + } + ]) + avg_prices = [doc["avg_price"] for doc in avg_price_ret] + # 计算平均值 + average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0 + if average_price > 0: + new_avg_price = int(average_price) + else: + # 取样本的平均值 + new_avg_price = int(sample['avg_price'].mean()) + if pd.notna(new_avg_price): + action = UpdateOne( + {'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date}, + {'$set': {'city': city, 'cat': 'office', 'date': curr_date, 'district': district, + 'name': office_name, + 'avg_price': new_avg_price, 'estate_type2': office['estate_type2'], + 'bed_kind': office['bed_kind'], + 'location': location, + 'calc_type': 5, 'updated_date': datetime.now()}}, + upsert=True) + office_sell_stat_conn.bulk_write([action]) + + +def calcByCity(city, year_month): + """ + 写字楼基准价 + """ sql = "select * from latest_seawater where cat='office_sell' " if pd.notna(city): sql += "and city='" + city + "' " @@ -43,17 +142,12 @@ def calc(city, year_month): print("开始计算年" + str(year_month) + "写字楼基准价") sample = es.scan_by_sql_to_data_frame(sql) - query = {'cat': 'office', 'status': 0} - if pd.notna(city): - query['city'] = city - if pd.notna(district): - query['region'] = district - if pd.notna(office_name): - query['name'] = office_name - office = pd.DataFrame(list(seaweed_df.find(query).batch_size(10000))) - office = office[['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']] - office = office.explode('alias') + office = getOffice(city) + update_office(city, office, sample, year_month) + + +def update_office(city, office, sample, year_month): sample_new = sample sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( 'median') @@ -65,7 +159,6 @@ def calc(city, year_month): sample_new = sample_new[['city', 'region', 'district_name', 'day_avg_price']] sample_new = sample_new.rename(columns={'district_name': 'name'}) sample_new['alias'] = sample_new['name'] - # 使用主名与别名查找样本 merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left', on=['city', 'region', 'name']) @@ -77,7 +170,6 @@ def calc(city, year_month): # 第一步:计算有样本写字楼的基准价 print("有样本数量:" + str(len(merge_df))) actions = list() - count = 0 for index, item in merge_df.iterrows(): city = item['city'] region = item['region'] @@ -90,7 +182,8 @@ def calc(city, year_month): estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None bed_kind = item['bed_kind'] if pd.notna(item['bed_kind']) else None action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, - {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, 'name': name, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind, 'location': location, 'calc_type': 1, 'updated_date': datetime.now()}}, @@ -98,21 +191,18 @@ def calc(city, year_month): actions.append(action) if len(actions) == 1000: office_sell_stat_conn.bulk_write(actions) - count += len(actions) - print("已计算" + str(count) + "条数据") actions.clear() if len(actions) > 0: office_sell_stat_conn.bulk_write(actions) - count += len(actions) - print("已计算" + str(count) + "条数据") actions.clear() - # 第二步 取周边3公里内同等级、同类型的最近10个有样本写字楼的基准价的平均值 - office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates(keep='first') + office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates( + keep='first') combined = pd.concat([office_info, merge_df]) diff_df = combined.drop_duplicates(subset=['city', 'region', 'name'], keep=False) actions = list() - for index, item in diff_df[(diff_df['estate_type2'].notna()) & (diff_df['bed_kind'].notna()) & (diff_df['lng'].notna()) ].iterrows(): + for index, item in diff_df[ + (diff_df['estate_type2'].notna()) & (diff_df['bed_kind'].notna()) & (diff_df['lng'].notna())].iterrows(): location = { "type": "Point", "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] @@ -161,15 +251,10 @@ def calc(city, year_month): actions.append(action) if len(actions) == 1000: office_sell_stat_conn.bulk_write(actions) - count += len(actions) - print("已计算" + str(count) + "条数据") actions.clear() if len(actions) > 0: office_sell_stat_conn.bulk_write(actions) - count += len(actions) - print("已计算" + str(count) + "条数据") actions.clear() - # 系数计算在Jupyter中 bed_kind_dict = {'甲级': 1.111888111888112, '丙级': 0.9143356643356645, '乙级': 1} radio_dict = {'创业园区': 0.7799999999999999, '商住两用': 1.0466666666666666, '标准写字楼': 1} @@ -220,13 +305,14 @@ def calc(city, year_month): count += 1 if count > 0: avg_price = int(sum_price / count) - action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, - {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, - 'name': name, - 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None, - 'location': location, - 'calc_type': 3, 'updated_date': datetime.now()}}, - upsert=True) + action = UpdateOne( + {'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, + 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None, + 'location': location, + 'calc_type': 3, 'updated_date': datetime.now()}}, + upsert=True) actions.append(action) elif pd.notna(bed_kind): ret = office_sell_stat_conn.aggregate([ @@ -259,23 +345,20 @@ def calc(city, year_month): count += 1 if count > 0: avg_price = int(sum_price / count) - action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, - {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, - 'name': name, - 'avg_price': avg_price, 'estate_type2': None, 'bed_kind': bed_kind, - 'location': location, - 'calc_type': 3, 'updated_date': datetime.now()}}, - upsert=True) + action = UpdateOne( + {'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, + 'avg_price': avg_price, 'estate_type2': None, 'bed_kind': bed_kind, + 'location': location, + 'calc_type': 3, 'updated_date': datetime.now()}}, + upsert=True) actions.append(action) if len(actions) == 1000: office_sell_stat_conn.bulk_write(actions) - count += len(actions) - print("已计算" + str(count) + "条数据") actions.clear() if len(actions) > 0: office_sell_stat_conn.bulk_write(actions) - count += len(actions) - print("已计算" + str(count) + "条数据") actions.clear() # 第五步 query = {'cat': 'office', 'date': year_month, 'calc_type': {'$in': [1, 2, 3]}} @@ -287,82 +370,85 @@ def calc(city, year_month): # 计算价格的写字楼 db_df = db_df.rename(columns={'district': 'region'}) # 使用 merge 操作并结合 indicator 参数 - merged = office_info.merge(db_df[['city', 'region', 'name']], on=['city', 'region', 'name'], how='left', - indicator=True) - - # 过滤出仅在 office_info 中存在的记录 - diff_df = merged[merged['_merge'] == 'left_only'].drop(columns=['_merge']) - - # 如果需要保持原始列顺序,可以重新排列列 - diff_df = diff_df[office_info.columns] - - actions.clear() - for index, item in diff_df.iterrows(): - try: - location = { - "type": "Point", - "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] - } - city = item['city'] - region = item['region'] - name = item['name'] - estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None - bed_kind = item['bed_kind'] - ret = office_sell_stat_conn.aggregate([ - { - '$geoNear': { - 'near': location, - 'distanceField': "distance", - 'maxDistance': 3000, - 'spherical': True - } - }, - { - '$match': { - 'date': year_month - } - }, - { - '$limit': 5 + if len(db_df) > 0: + merged = office_info.merge(db_df[['city', 'region', 'name']], on=['city', 'region', 'name'], how='left', + indicator=True) + # 过滤出仅在 office_info 中存在的记录 + diff_df = merged[merged['_merge'] == 'left_only'].drop(columns=['_merge']) + # 如果需要保持原始列顺序,可以重新排列列 + diff_df = diff_df[office_info.columns] + actions.clear() + for index, item in diff_df.iterrows(): + try: + location = { + "type": "Point", + "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] } - ]); - except Exception as e: - print("Error occurred:", str(e)) - print(item) - # 提取 avg_price 的值 - avg_prices = [doc["avg_price"] for doc in ret] - # 计算平均值 - average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0 - if average_price > 0: - avg_price = int(average_price) - action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, - {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, - 'name': name, - 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None, - 'location': location, - 'calc_type': 4, 'updated_date': datetime.now()}}, - upsert=True) - actions.append(action) - if len(actions) == 1000: - office_sell_stat_conn.bulk_write(actions) - count += len(actions) - print("已计算" + str(count) + "条数据") - actions.clear() + city = item['city'] + region = item['region'] + name = item['name'] + estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None + bed_kind = item['bed_kind'] + ret = office_sell_stat_conn.aggregate([ + { + '$geoNear': { + 'near': location, + 'distanceField': "distance", + 'maxDistance': 3000, + 'spherical': True + } + }, + { + '$match': { + 'date': year_month + } + }, + { + '$limit': 5 + } + ]); + except Exception as e: + print("Error occurred:", str(e)) + print(item) + # 提取 avg_price 的值 + avg_prices = [doc["avg_price"] for doc in ret] + # 计算平均值 + average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0 + if average_price > 0: + avg_price = int(average_price) + action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, + {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, + 'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None, + 'location': location, + 'calc_type': 4, 'updated_date': datetime.now()}}, + upsert=True) + actions.append(action) + if len(actions) == 1000: + office_sell_stat_conn.bulk_write(actions) + actions.clear() if len(actions) > 0: office_sell_stat_conn.bulk_write(actions) - count += len(actions) - print("已计算" + str(count) + "条数据") actions.clear() -if __name__ == '__main__': - yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' - if yearmonth == '': - date = dateutils.add_months(datetime.today(), months=-1) - yearmonth = int(dateutils.format_date(date, '%Y%m')) - try: - if type(yearmonth) is str: - yearmonth = int(yearmonth) - calc(None, yearmonth) - except Exception as e: - print(e) \ No newline at end of file +def getOffice(city, district, name): + """ + 获取写字楼 + """ + query = {'cat': 'office', 'status': 0} + if pd.notna(city): + query['city'] = city + if pd.notna(district): + query['region'] = district + if pd.notna(city): + query['name'] = name + office = pd.DataFrame(list(seaweed_df.find(query).batch_size(10000))) + office = office[['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']] + office = office.explode('alias') + return office + + +if __name__ == "__main__": + # 测试脚本:直接运行时模拟 Airflow 的上下文 + calc(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}})) \ No newline at end of file From 96c795be4a557d320f7ea2055437d02a17adbb56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 10 Jun 2025 16:27:26 +0800 Subject: [PATCH 11/16] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=86=99=E5=AD=97?= =?UTF-8?q?=E6=A5=BC=E5=87=BA=E5=94=AE=E4=BB=B7=E6=A0=BC=E9=87=8D=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/rent/base_price.py | 2 +- dags/office/sell/fill_base_price.py | 49 ++++++++++++++++++----------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/dags/office/rent/base_price.py b/dags/office/rent/base_price.py index 1af5161..a2f4e6d 100644 --- a/dags/office/rent/base_price.py +++ b/dags/office/rent/base_price.py @@ -35,7 +35,7 @@ def calc(**kwargs): district = conf.get('district', '') office_name = conf.get('name', '') - if pd.isna(office_name): + if office_name == '': calcByCity(city, year_month) else: # 直接重算出最近一年的数据,保证一年有数据 diff --git a/dags/office/sell/fill_base_price.py b/dags/office/sell/fill_base_price.py index 7245f8a..bfd334d 100644 --- a/dags/office/sell/fill_base_price.py +++ b/dags/office/sell/fill_base_price.py @@ -11,18 +11,40 @@ office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat') -def calc(city, year_month): - query = {'date': year_month, 'cat': 'office'} - if pd.notna(city): +def calc(**kwargs): + """ + 写字楼基准价 + """ + # 从 Airflow 的 dag_run.conf 获取参数 + conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {} + city = conf.get('city', '') + year_month = conf.get('year_month', '') + district = conf.get('district', '') + office_name = conf.get('name', '') + + query = {'cat': 'office'} + if year_month != '': + query['date'] = year_month + if city != '': query['city'] = city + if district != '' and office_name != '': + query['district'] = district + query['name'] = office_name office_sell = pd.DataFrame(list(office_sell_stat_conn.find(query))) office_rent = pd.DataFrame(list(office_rent_stat_conn.find(query))) if len(office_rent) <= 0: return + if year_month != '': + calc_by_yearmonth(office_sell, office_rent, year_month) + else: + # 对于单个重算的不进行补充,已经自己补充过了 + pass + +def calc_by_yearmonth(office_sell, office_rent, year_month): office_sell_df = office_sell[['city', 'district', 'name', 'avg_price']] office_sell_df = office_sell_df.rename(columns={'avg_price': 'sell_price'}) - office_rent_df = office_rent[['city', 'district', 'name', 'avg_price', 'estate_type2', 'bed_kind','location']] + office_rent_df = office_rent[['city', 'district', 'name', 'avg_price', 'estate_type2', 'bed_kind', 'location']] office_rent_df = office_rent_df.rename(columns={'avg_price': 'rent_price'}) # 每月动态计算系数,因为不同月份会有差异 merge_df = pd.merge(office_sell_df, office_rent_df, how='left', on=['city', 'district', 'name']) @@ -43,13 +65,13 @@ def calc(city, year_month): 填充存在租金,不存在售价的数据 """ if item['city'] + "_" + item['district'] in district_radio.keys(): - item['sell_price'] = round(district_radio[item['city'] + "_" + item['district']] * item['rent_price'] * 30 * 12, 0) + item['sell_price'] = round( + district_radio[item['city'] + "_" + item['district']] * item['rent_price'] * 30 * 12, 0) elif item['city'] in city_radio.keys(): item['sell_price'] = round(city_radio[item['city']] * item['rent_price'] * 30 * 12, 0) return item if len(merge2_df) == 0: - print(city + "无数据补充") return merge2_df = threads.process_dataframe_multithread(merge2_df, deal, 10) merge2_df = merge2_df[merge2_df['sell_price'].notna()] @@ -83,16 +105,7 @@ def calc(city, year_month): print("已处理{}条数据".format(count)) print("处理完毕") -if __name__ == '__main__': - yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' - - if yearmonth == '': - date = dateutils.add_months(datetime.today(), months=-1) - yearmonth = int(dateutils.format_date(date, '%Y%m')) - try: - if type(yearmonth) is str: - yearmonth = int(yearmonth) - calc(None, yearmonth) - except Exception as e: - print(e) +if __name__ == "__main__": + # 测试脚本:直接运行时模拟 Airflow 的上下文 + calc(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}})) From 6ce61b63366b32f50159c86b8a3270e1c6a5465f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 10 Jun 2025 16:33:31 +0800 Subject: [PATCH 12/16] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=86=99=E5=AD=97?= =?UTF-8?q?=E6=A5=BC=E5=87=BA=E5=94=AE=E4=BB=B7=E6=A0=BC=E9=87=8D=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/dag_office_sell_base_price.py | 10 ++++++---- dags/office/sell/fill_base_price.py | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dags/office/dag_office_sell_base_price.py b/dags/office/dag_office_sell_base_price.py index 82e3ef9..4465e2a 100644 --- a/dags/office/dag_office_sell_base_price.py +++ b/dags/office/dag_office_sell_base_price.py @@ -4,6 +4,7 @@ from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator from office.sell.base_price import calc +from office.sell.fill_base_price import fill_base_price import pendulum @@ -21,9 +22,10 @@ with DAG( python_callable=calc, provide_context=True ) - # t2 = BashOperator( - # task_id='fill_base_price', - # bash_command='python /opt/airflow/dags/repo/dags/office/sell/fill_base_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}', - # ) + t2 = PythonOperator( + task_id='base_price', + python_callable=fill_base_price, + provide_context=True + ) t1 >> t2 diff --git a/dags/office/sell/fill_base_price.py b/dags/office/sell/fill_base_price.py index bfd334d..7cfc1e1 100644 --- a/dags/office/sell/fill_base_price.py +++ b/dags/office/sell/fill_base_price.py @@ -11,7 +11,7 @@ office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat') -def calc(**kwargs): +def fill_base_price(**kwargs): """ 写字楼基准价 """ @@ -107,5 +107,5 @@ def calc_by_yearmonth(office_sell, office_rent, year_month): if __name__ == "__main__": # 测试脚本:直接运行时模拟 Airflow 的上下文 - calc(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}})) + fill_base_price(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}})) From 92ce23a11db2b5d516c76ab9f32f8f7a9544c9f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 10 Jun 2025 16:38:08 +0800 Subject: [PATCH 13/16] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=86=99=E5=AD=97?= =?UTF-8?q?=E6=A5=BC=E5=87=BA=E5=94=AE=E4=BB=B7=E6=A0=BC=E9=87=8D=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/dag_office_sell_base_price.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dags/office/dag_office_sell_base_price.py b/dags/office/dag_office_sell_base_price.py index 4465e2a..8030ad1 100644 --- a/dags/office/dag_office_sell_base_price.py +++ b/dags/office/dag_office_sell_base_price.py @@ -18,12 +18,12 @@ with DAG( description="写字楼出售基准价计算" ) as dag: t1 = PythonOperator( - task_id='base_price', + task_id='calc', python_callable=calc, provide_context=True ) t2 = PythonOperator( - task_id='base_price', + task_id='fill_base_price', python_callable=fill_base_price, provide_context=True ) From a6e60819ad1d923ce4cb2e6a5eab9047a1f9ea86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 12 Jun 2025 13:17:16 +0800 Subject: [PATCH 14/16] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=87=8D=E7=AE=97BUG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/rent/base_price.py | 48 +++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/dags/office/rent/base_price.py b/dags/office/rent/base_price.py index a2f4e6d..2ec5b2b 100644 --- a/dags/office/rent/base_price.py +++ b/dags/office/rent/base_price.py @@ -76,7 +76,10 @@ def calcByOffice(city, district, office_name): for date in date_range: curr_date = int(dateutils.format_date(date, fm='%Y%m')) # 筛选出当前月的样本 - curr_sample = sample[(sample['p_date']>=int(str(curr_date)+'01')) & (sample['p_date']<=int(str(curr_date)+'31'))] + if len(sample) > 0: + curr_sample = sample[(sample['p_date']>=int(str(curr_date)+'01')) & (sample['p_date']<=int(str(curr_date)+'31'))] + else: + curr_sample = pd.DataFrame(columns=['city', 'region', 'district_name', 'avg_price']) update_office(city, pd.DataFrame(office), curr_sample, curr_date) # 核查当月写字楼是否存在,不存在取最后6个月的平均值 office_db = office_rent_stat_conn.find_one({'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date}) @@ -103,7 +106,7 @@ def calcByOffice(city, district, office_name): average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0 if average_price > 0: new_avg_price = round(average_price, 2) - else: + elif len(sample) > 0: # 取样本的平均值 new_avg_price = round(sample['avg_price'].mean(), 2) if pd.notna(new_avg_price): @@ -149,24 +152,27 @@ def calcByCity(city, year_month): def update_office(city, office, sample, year_month): sample_new = sample - sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( - 'median') - sample_new['sample_count'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( - 'count') - sample_new = sample_new[sample_new['sample_count'] >= 5] - sample_new = sample_new[['city', 'region', 'district_name', 'sample_avg_price']].dropna() - sample_new['day_avg_price'] = sample_new['sample_avg_price'] / 30 - sample_new = sample_new[['city', 'region', 'district_name', 'day_avg_price']] - sample_new = sample_new.rename(columns={'district_name': 'name'}) - sample_new['alias'] = sample_new['name'] - # 使用主名与别名查找样本 - merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left', - on=['city', 'region', 'name']) - merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left', - on=['city', 'region', 'alias']) - merge_df = pd.concat([merge_df1, merge_df2]) - merge_df = merge_df[merge_df['day_avg_price'].notna()] - merge_df = merge_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first') + if len(sample_new) > 0: + sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( + 'median') + sample_new['sample_count'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( + 'count') + sample_new = sample_new[sample_new['sample_count'] >= 5] + sample_new = sample_new[['city', 'region', 'district_name', 'sample_avg_price']].dropna() + sample_new['day_avg_price'] = sample_new['sample_avg_price'] / 30 + sample_new = sample_new[['city', 'region', 'district_name', 'day_avg_price']] + sample_new = sample_new.rename(columns={'district_name': 'name'}) + sample_new['alias'] = sample_new['name'] + # 使用主名与别名查找样本 + merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left', + on=['city', 'region', 'name']) + merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left', + on=['city', 'region', 'alias']) + merge_df = pd.concat([merge_df1, merge_df2]) + merge_df = merge_df[merge_df['day_avg_price'].notna()] + merge_df = merge_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first') + else: + merge_df = pd.DataFrame() # 第一步:计算有样本写字楼的基准价 print("有样本数量:" + str(len(merge_df))) actions = list() @@ -451,4 +457,4 @@ def getOffice(city, district, name): if __name__ == "__main__": # 测试脚本:直接运行时模拟 Airflow 的上下文 - calc(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}})) \ No newline at end of file + calc(dag_run=type('DagRun', (), {'conf': {'city': '福州', 'district': '晋安', 'name': '筑源大厦'}})) \ No newline at end of file From bc7f275d0637f88b7672daf1b93399256022e0a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 12 Jun 2025 13:17:16 +0800 Subject: [PATCH 15/16] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=87=8D=E7=AE=97BUG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/rent/base_price.py | 49 +++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/dags/office/rent/base_price.py b/dags/office/rent/base_price.py index a2f4e6d..4a4d258 100644 --- a/dags/office/rent/base_price.py +++ b/dags/office/rent/base_price.py @@ -76,7 +76,10 @@ def calcByOffice(city, district, office_name): for date in date_range: curr_date = int(dateutils.format_date(date, fm='%Y%m')) # 筛选出当前月的样本 - curr_sample = sample[(sample['p_date']>=int(str(curr_date)+'01')) & (sample['p_date']<=int(str(curr_date)+'31'))] + if len(sample) > 0: + curr_sample = sample[(sample['p_date']>=int(str(curr_date)+'01')) & (sample['p_date']<=int(str(curr_date)+'31'))] + else: + curr_sample = pd.DataFrame(columns=['city', 'region', 'district_name', 'avg_price']) update_office(city, pd.DataFrame(office), curr_sample, curr_date) # 核查当月写字楼是否存在,不存在取最后6个月的平均值 office_db = office_rent_stat_conn.find_one({'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date}) @@ -101,9 +104,10 @@ def calcByOffice(city, district, office_name): avg_prices = [doc["avg_price"] for doc in avg_price_ret] # 计算平均值 average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0 + new_avg_price = None if average_price > 0: new_avg_price = round(average_price, 2) - else: + elif len(sample) > 0: # 取样本的平均值 new_avg_price = round(sample['avg_price'].mean(), 2) if pd.notna(new_avg_price): @@ -149,24 +153,27 @@ def calcByCity(city, year_month): def update_office(city, office, sample, year_month): sample_new = sample - sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( - 'median') - sample_new['sample_count'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( - 'count') - sample_new = sample_new[sample_new['sample_count'] >= 5] - sample_new = sample_new[['city', 'region', 'district_name', 'sample_avg_price']].dropna() - sample_new['day_avg_price'] = sample_new['sample_avg_price'] / 30 - sample_new = sample_new[['city', 'region', 'district_name', 'day_avg_price']] - sample_new = sample_new.rename(columns={'district_name': 'name'}) - sample_new['alias'] = sample_new['name'] - # 使用主名与别名查找样本 - merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left', - on=['city', 'region', 'name']) - merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left', - on=['city', 'region', 'alias']) - merge_df = pd.concat([merge_df1, merge_df2]) - merge_df = merge_df[merge_df['day_avg_price'].notna()] - merge_df = merge_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first') + if len(sample_new) > 0: + sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( + 'median') + sample_new['sample_count'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( + 'count') + sample_new = sample_new[sample_new['sample_count'] >= 5] + sample_new = sample_new[['city', 'region', 'district_name', 'sample_avg_price']].dropna() + sample_new['day_avg_price'] = sample_new['sample_avg_price'] / 30 + sample_new = sample_new[['city', 'region', 'district_name', 'day_avg_price']] + sample_new = sample_new.rename(columns={'district_name': 'name'}) + sample_new['alias'] = sample_new['name'] + # 使用主名与别名查找样本 + merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left', + on=['city', 'region', 'name']) + merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left', + on=['city', 'region', 'alias']) + merge_df = pd.concat([merge_df1, merge_df2]) + merge_df = merge_df[merge_df['day_avg_price'].notna()] + merge_df = merge_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first') + else: + merge_df = pd.DataFrame() # 第一步:计算有样本写字楼的基准价 print("有样本数量:" + str(len(merge_df))) actions = list() @@ -451,4 +458,4 @@ def getOffice(city, district, name): if __name__ == "__main__": # 测试脚本:直接运行时模拟 Airflow 的上下文 - calc(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}})) \ No newline at end of file + calc(dag_run=type('DagRun', (), {'conf': {'city': '福州', 'district': '晋安', 'name': '筑源大厦'}})) \ No newline at end of file From b6502d0051f175106b61bbb2e54d1bda7b207016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 16 Jun 2025 13:32:02 +0800 Subject: [PATCH 16/16] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=87=8D=E7=AE=97BUG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/rent/base_price.py | 27 ++++++++++++++++++++------- dags/office/sell/base_price.py | 23 ++++++++++++++++++----- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/dags/office/rent/base_price.py b/dags/office/rent/base_price.py index 4a4d258..1f84650 100644 --- a/dags/office/rent/base_price.py +++ b/dags/office/rent/base_price.py @@ -52,10 +52,14 @@ def calcByOffice(city, district, office_name): query['region'] = district if pd.notna(city): query['name'] = office_name - office = seaweed_df.find_one(query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'lng': 1, 'lat': 1, 'estate_type2': 1, 'bed_kind': 1}) + office = seaweed_df.find_one(query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'lng': 1, 'lat': 1, 'estate_type2': 1, 'bed_kind': 1, '_id': 0}) + if 'estate_type2' not in office: + office['estate_type2'] = None + if 'bed_kind' not in office: + office['bed_kind'] = None names = list() names.append(office['name']) - if type(office['alias']) == list: + if ('alias' in office) and (type(office['alias']) == list): for alias_item in office['alias']: names.append(alias_item) names_str = "','".join(names) @@ -80,7 +84,7 @@ def calcByOffice(city, district, office_name): curr_sample = sample[(sample['p_date']>=int(str(curr_date)+'01')) & (sample['p_date']<=int(str(curr_date)+'31'))] else: curr_sample = pd.DataFrame(columns=['city', 'region', 'district_name', 'avg_price']) - update_office(city, pd.DataFrame(office), curr_sample, curr_date) + update_office(city, pd.DataFrame([office]), curr_sample, curr_date) # 核查当月写字楼是否存在,不存在取最后6个月的平均值 office_db = office_rent_stat_conn.find_one({'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date}) if pd.isna(office_db): @@ -152,7 +156,13 @@ def calcByCity(city, year_month): def update_office(city, office, sample, year_month): + if 'bed_kind' not in office.columns: + office['bed_kind'] = None + if 'estate_type2' not in office.columns: + office['estate_type2'] = None sample_new = sample + if 'alias' not in sample_new.columns: + sample_new['alias'] = None if len(sample_new) > 0: sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( 'median') @@ -167,9 +177,12 @@ def update_office(city, office, sample, year_month): # 使用主名与别名查找样本 merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left', on=['city', 'region', 'name']) - merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left', - on=['city', 'region', 'alias']) - merge_df = pd.concat([merge_df1, merge_df2]) + if len(sample_new['alias'].unique()) > 0: + merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left', + on=['city', 'region', 'alias']) + merge_df = pd.concat([merge_df1, merge_df2]) + else: + merge_df = merge_df1.copy() merge_df = merge_df[merge_df['day_avg_price'].notna()] merge_df = merge_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first') else: @@ -458,4 +471,4 @@ def getOffice(city, district, name): if __name__ == "__main__": # 测试脚本:直接运行时模拟 Airflow 的上下文 - calc(dag_run=type('DagRun', (), {'conf': {'city': '福州', 'district': '晋安', 'name': '筑源大厦'}})) \ No newline at end of file + calc(dag_run=type('DagRun', (), {'conf': {'city': '北京', 'district': '朝阳', 'name': '中核华建大厦'}})) \ No newline at end of file diff --git a/dags/office/sell/base_price.py b/dags/office/sell/base_price.py index fff8628..d2c7718 100644 --- a/dags/office/sell/base_price.py +++ b/dags/office/sell/base_price.py @@ -53,9 +53,13 @@ def calcByOffice(city, district, office_name): if pd.notna(city): query['name'] = office_name office = seaweed_df.find_one(query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'lng': 1, 'lat': 1, 'estate_type2': 1, 'bed_kind': 1}) + if 'estate_type2' not in office: + office['estate_type2'] = None + if 'bed_kind' not in office: + office['bed_kind'] = None names = list() names.append(office['name']) - if type(office['alias']) == list: + if ('alias' in office) and (type(office['alias']) == list): for alias_item in office['alias']: names.append(alias_item) names_str = "','".join(names) @@ -77,7 +81,7 @@ def calcByOffice(city, district, office_name): curr_date = int(dateutils.format_date(date, fm='%Y%m')) # 筛选出当前月的样本 curr_sample = sample[(sample['p_date']>=int(str(curr_date)+'01')) & (sample['p_date']<=int(str(curr_date)+'31'))] - update_office(city, pd.DataFrame(office), curr_sample, curr_date) + update_office(city, pd.DataFrame([office]), curr_sample, curr_date) # 核查当月写字楼是否存在,不存在取最后6个月的平均值 office_db = office_sell_stat_conn.find_one({'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date}) if pd.isna(office_db): @@ -148,7 +152,13 @@ def calcByCity(city, year_month): def update_office(city, office, sample, year_month): + if 'bed_kind' not in office.columns: + office['bed_kind'] = None + if 'estate_type2' not in office.columns: + office['estate_type2'] = None sample_new = sample + if 'alias' not in sample_new.columns: + sample_new['alias'] = None sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( 'median') sample_new['sample_count'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform( @@ -162,9 +172,12 @@ def update_office(city, office, sample, year_month): # 使用主名与别名查找样本 merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left', on=['city', 'region', 'name']) - merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left', - on=['city', 'region', 'alias']) - merge_df = pd.concat([merge_df1, merge_df2]) + if len(sample_new['alias'].unique()) > 0: + merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left', + on=['city', 'region', 'alias']) + merge_df = pd.concat([merge_df1, merge_df2]) + else: + merge_df = merge_df1.copy() merge_df = merge_df[merge_df['day_avg_price'].notna()] merge_df = merge_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first') # 第一步:计算有样本写字楼的基准价