Merge remote-tracking branch 'origin/master'
This commit is contained in:
@@ -14,7 +14,7 @@ def sync(province):
|
||||
index="didata_company",
|
||||
body={"index": {"refresh_interval": "-1"}}
|
||||
)
|
||||
ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name from didata_company where province='{province}'")
|
||||
ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name, committee from didata_company where province='{province}'")
|
||||
if len(ids) > 0:
|
||||
company_ids = ids['_id'].to_list()
|
||||
else:
|
||||
@@ -92,6 +92,10 @@ def sync(province):
|
||||
body={"index": {"refresh_interval": "5s"}}
|
||||
)
|
||||
|
||||
|
||||
for province in areautils.get_all_province():
|
||||
sync(province)
|
||||
if __name__ == '__main__':
|
||||
flag = False
|
||||
for province in areautils.get_all_province():
|
||||
if '山东省' == province:
|
||||
flag = True
|
||||
if flag:
|
||||
sync(province)
|
||||
@@ -65,4 +65,8 @@ def sync(province):
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
|
||||
index="didata_company",
|
||||
body={"index": {"refresh_interval": "5s"}}
|
||||
)
|
||||
)
|
||||
|
||||
if __name__ == '__main__':
|
||||
for province in areautils.get_all_province():
|
||||
sync(province)
|
||||
@@ -6,125 +6,129 @@ import re
|
||||
import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
"""
|
||||
对于没有围栏结果的,通过match_phrase匹配出来的开发区一定正确,如果有围栏出来的开发区,则企业经纬度不正确
|
||||
"""
|
||||
def sync():
|
||||
"""
|
||||
对于没有围栏结果的,通过match_phrase匹配出来的开发区一定正确,如果有围栏出来的开发区,则企业经纬度不正确
|
||||
"""
|
||||
|
||||
dev_alias = {'福建永定工业园区': '永定区永定工业园', '福建漳州古雷港经济开发区': '古雷港经济开发区', '福建莆田湄洲湾北岸经济开发区': '湄洲湾北岸经济开发区',
|
||||
'福建南平工业园区': '南平工业园区', '泉州台商投资区': '泉州市台商投资区', '福建洛江经济开发区': '洛江园区', '福建武平工业园区': '武平工业园',
|
||||
'福建晋江经济开发区': '晋江市经济开发区', '漳州台商投资区': '龙池开发区', '莆田高新技术产业开发区': '莆田市高新技术',
|
||||
'莆田高新技术产业开发区': '莆田市涵江区高新', '绥安工业开发区': '绥安工业园', '绥安工业开发区': '黄仓开发区', '绥安工业开发区': '黄仓工业园',
|
||||
'绥安工业开发区': '旧镇工业园', '绥安工业开发区': '大南坂开发区', '绥安工业开发区': '大南坂工业开发区', '福建龙州工业园区': '龙州工业园',
|
||||
'长汀经济开发区':'腾飞开发区', '福建福州金山工业园区': '金山投资区', '厦门象屿综合保税区': '厦门现代物流园', '福清融侨经济技术开发区': '融侨经济',
|
||||
'福建洛江经济开发区': '万安开发区'}
|
||||
dev_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame("select province, city, district, name, alias from development_zone where province='福建省'")
|
||||
# 不要去重,有可能存在别名, 如福建云霄常山经济开发区
|
||||
#dev_df = dev_df[dev_df['name'] == '福建政和经济开发区']
|
||||
actions = list()
|
||||
for dev in dev_df.itertuples():
|
||||
if dev.name in ['中国(福建)自由贸易试验区厦门片区', '中国(福建)自由贸易试验区福州片区', '中国(福建)自由贸易试验区平潭片区']:
|
||||
# 中国(福建)自由贸易试验区 已包含,防止反复更新
|
||||
continue
|
||||
if dev.name in ['政和经济开发区']:
|
||||
continue
|
||||
dev_alias = {'福建永定工业园区': '永定区永定工业园', '福建漳州古雷港经济开发区': '古雷港经济开发区', '福建莆田湄洲湾北岸经济开发区': '湄洲湾北岸经济开发区',
|
||||
'福建南平工业园区': '南平工业园区', '泉州台商投资区': '泉州市台商投资区', '福建洛江经济开发区': '洛江园区', '福建武平工业园区': '武平工业园',
|
||||
'福建晋江经济开发区': '晋江市经济开发区', '漳州台商投资区': '龙池开发区', '莆田高新技术产业开发区': '莆田市高新技术',
|
||||
'莆田高新技术产业开发区': '莆田市涵江区高新', '绥安工业开发区': '绥安工业园', '绥安工业开发区': '黄仓开发区', '绥安工业开发区': '黄仓工业园',
|
||||
'绥安工业开发区': '旧镇工业园', '绥安工业开发区': '大南坂开发区', '绥安工业开发区': '大南坂工业开发区', '福建龙州工业园区': '龙州工业园',
|
||||
'长汀经济开发区':'腾飞开发区', '福建福州金山工业园区': '金山投资区', '厦门象屿综合保税区': '厦门现代物流园', '福清融侨经济技术开发区': '融侨经济',
|
||||
'福建洛江经济开发区': '万安开发区'}
|
||||
dev_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame("select province, city, district, name, alias from development_zone where province='福建省'")
|
||||
# 不要去重,有可能存在别名, 如福建云霄常山经济开发区
|
||||
#dev_df = dev_df[dev_df['name'] == '福建政和经济开发区']
|
||||
actions = list()
|
||||
for dev in dev_df.itertuples():
|
||||
if dev.name in ['中国(福建)自由贸易试验区厦门片区', '中国(福建)自由贸易试验区福州片区', '中国(福建)自由贸易试验区平潭片区']:
|
||||
# 中国(福建)自由贸易试验区 已包含,防止反复更新
|
||||
continue
|
||||
if dev.name in ['政和经济开发区']:
|
||||
continue
|
||||
|
||||
count = 0
|
||||
names = list()
|
||||
names.append(dev.name)
|
||||
if str(dev.alias) != 'nan':
|
||||
names.extend(dev.alias)
|
||||
#东侨经济技术开发区 福建省宁德市东侨经济开发区塔山村塔兴路25号
|
||||
if '经济技术开发区' in dev.name:
|
||||
names.append(re.sub('经济技术开发区', '经济开发区', dev.name))
|
||||
# 福州经济技术开发区 福州开发区君竹路83号科技发展中心大楼第四层R472室(自贸试验区内)
|
||||
names.append(re.sub('经济技术开发区', '开发区', dev.name))
|
||||
# 长汀经济开发区 长汀县经济开发区工业新区河梁大山体
|
||||
if '经济开发区' in dev.name:
|
||||
if pd.notna(dev.city) and pd.notna(dev.district):
|
||||
district_short = areautils.get_district_short(dev.city, dev.district)
|
||||
if pd.notna(district_short):
|
||||
names.append(re.sub(district_short, dev.district, dev.name))
|
||||
# 漳州高新技术产业开发区 福建省漳州高新区九湖镇木棉村后壁山401-1号
|
||||
if '高新技术产业开发区' in dev.name:
|
||||
names.append(re.sub('高新技术产业开发区', '高新区', dev.name))
|
||||
# 厦门火炬高技术产业开发区 厦门火炬高新区软件园三期溪西山尾路63号1303室-3单元之三
|
||||
if '高技术产业开发区' in dev.name:
|
||||
names.append(re.sub('高技术产业开发区', '高新区', dev.name))
|
||||
#areautils.get_district_display()
|
||||
# 福建诏安工业园区 福建省漳州市诏安工业园区丹诏大道15号
|
||||
if pd.notna(dev.city):
|
||||
province_short = areautils.get_province(dev.city)
|
||||
new_name = re.sub(province_short, '', dev.name)
|
||||
if pd.notna(dev.city) and pd.notna(dev.district):
|
||||
city_short = areautils.get_city_short(dev.city)
|
||||
district_short = areautils.get_district_short(dev.city, dev.district)
|
||||
if pd.notna(city_short) and city_short in new_name:
|
||||
names.append(new_name)
|
||||
if pd.notna(district_short) and district_short in new_name:
|
||||
names.append(new_name)
|
||||
# 福建省漳州市长泰县经济开发区林墩办事处帝君路 福建长泰经济开发区
|
||||
new_new_name = re.sub(district_short, district_short + '县', new_name)
|
||||
names.append(new_new_name)
|
||||
new_new_name = re.sub(district_short, district_short + '区', new_name)
|
||||
names.append(new_new_name)
|
||||
count = 0
|
||||
names = list()
|
||||
names.append(dev.name)
|
||||
if str(dev.alias) != 'nan':
|
||||
names.extend(dev.alias)
|
||||
#东侨经济技术开发区 福建省宁德市东侨经济开发区塔山村塔兴路25号
|
||||
if '经济技术开发区' in dev.name:
|
||||
names.append(re.sub('经济技术开发区', '经济开发区', dev.name))
|
||||
# 福州经济技术开发区 福州开发区君竹路83号科技发展中心大楼第四层R472室(自贸试验区内)
|
||||
names.append(re.sub('经济技术开发区', '开发区', dev.name))
|
||||
# 长汀经济开发区 长汀县经济开发区工业新区河梁大山体
|
||||
if '经济开发区' in dev.name:
|
||||
if pd.notna(dev.city) and pd.notna(dev.district):
|
||||
district_short = areautils.get_district_short(dev.city, dev.district)
|
||||
if pd.notna(district_short):
|
||||
names.append(re.sub(district_short, dev.district, dev.name))
|
||||
# 漳州高新技术产业开发区 福建省漳州高新区九湖镇木棉村后壁山401-1号
|
||||
if '高新技术产业开发区' in dev.name:
|
||||
names.append(re.sub('高新技术产业开发区', '高新区', dev.name))
|
||||
# 厦门火炬高技术产业开发区 厦门火炬高新区软件园三期溪西山尾路63号1303室-3单元之三
|
||||
if '高技术产业开发区' in dev.name:
|
||||
names.append(re.sub('高技术产业开发区', '高新区', dev.name))
|
||||
#areautils.get_district_display()
|
||||
# 福建诏安工业园区 福建省漳州市诏安工业园区丹诏大道15号
|
||||
if pd.notna(dev.city):
|
||||
province_short = areautils.get_province(dev.city)
|
||||
new_name = re.sub(province_short, '', dev.name)
|
||||
if pd.notna(dev.city) and pd.notna(dev.district):
|
||||
city_short = areautils.get_city_short(dev.city)
|
||||
district_short = areautils.get_district_short(dev.city, dev.district)
|
||||
if pd.notna(city_short) and city_short in new_name:
|
||||
names.append(new_name)
|
||||
if pd.notna(district_short) and district_short in new_name:
|
||||
names.append(new_name)
|
||||
# 福建省漳州市长泰县经济开发区林墩办事处帝君路 福建长泰经济开发区
|
||||
new_new_name = re.sub(district_short, district_short + '县', new_name)
|
||||
names.append(new_new_name)
|
||||
new_new_name = re.sub(district_short, district_short + '区', new_name)
|
||||
names.append(new_new_name)
|
||||
|
||||
for key in dev_alias:
|
||||
if dev.name == key:
|
||||
names.append(dev_alias[key])
|
||||
for name in names:
|
||||
query = {
|
||||
"query": {
|
||||
"bool": {
|
||||
"must_not": [
|
||||
{
|
||||
"term": {
|
||||
"dev": {
|
||||
"value": dev.name
|
||||
for key in dev_alias:
|
||||
if dev.name == key:
|
||||
names.append(dev_alias[key])
|
||||
for name in names:
|
||||
query = {
|
||||
"query": {
|
||||
"bool": {
|
||||
"must_not": [
|
||||
{
|
||||
"term": {
|
||||
"dev": {
|
||||
"value": dev.name
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"term": {
|
||||
"dev_status": {
|
||||
"value": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"term": {
|
||||
"dev_status": {
|
||||
"value": 0
|
||||
],
|
||||
"must": [
|
||||
{
|
||||
"term": {
|
||||
"province": {
|
||||
"value": dev.province
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"match_phrase": {
|
||||
"registered_address": name
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"must": [
|
||||
{
|
||||
"term": {
|
||||
"province": {
|
||||
"value": dev.province
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"match_phrase": {
|
||||
"registered_address": name
|
||||
}
|
||||
}
|
||||
}
|
||||
count_data = DIDATA_DEV_ELASTIC_CLIENT.elastic_client.count(index='didata_company',body=query)
|
||||
if count_data['count'] > 0:
|
||||
for doc in DIDATA_DEV_ELASTIC_CLIENT.scan(index='didata_company',query=query):
|
||||
action = {
|
||||
"_op_type": "update",
|
||||
"_index": "didata_company",
|
||||
"_id": doc['_id'],
|
||||
"doc": {'dev': dev.name, 'dev_status': 0}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
count_data = DIDATA_DEV_ELASTIC_CLIENT.elastic_client.count(index='didata_company',body=query)
|
||||
if count_data['count'] > 0:
|
||||
for doc in DIDATA_DEV_ELASTIC_CLIENT.scan(index='didata_company',query=query):
|
||||
action = {
|
||||
"_op_type": "update",
|
||||
"_index": "didata_company",
|
||||
"_id": doc['_id'],
|
||||
"doc": {'dev': dev.name, 'dev_status': 0}
|
||||
}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions)
|
||||
count += len(actions)
|
||||
print(f"已更新 {dev.name} 开发区" + str(count) + "记录")
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions)
|
||||
count += len(actions)
|
||||
print(f"已更新 {dev.name} 开发区" + str(count) + "记录")
|
||||
actions.clear()
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions)
|
||||
count += len(actions)
|
||||
print(f"已更新 {dev.name} 开发区" + str(count) + "记录")
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions)
|
||||
count += len(actions)
|
||||
print(f"已更新 {dev.name} 开发区" + str(count) + "记录")
|
||||
actions.clear()
|
||||
|
||||
if __name__ == '__main__':
|
||||
sync()
|
||||
@@ -14,7 +14,6 @@ last_day = last.strftime('%Y%m%d')
|
||||
|
||||
log.info(f"系统当前时间 : {now}")
|
||||
|
||||
|
||||
def proc():
|
||||
# 区域维度,统计2个月内样本数据量
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
@@ -100,10 +99,6 @@ def proc():
|
||||
air_file_name = path + '/' + file_name
|
||||
return text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new
|
||||
|
||||
|
||||
text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new = proc()
|
||||
|
||||
|
||||
def up_to_nextcloud():
|
||||
"""
|
||||
上传统计信息至nextcloud
|
||||
@@ -116,17 +111,18 @@ def up_to_nextcloud():
|
||||
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD)
|
||||
nxc.upload_file(air_file_name, nextcloud_file_name)
|
||||
|
||||
|
||||
if text != '':
|
||||
with pd.ExcelWriter(air_file_name) as writer:
|
||||
if not df_region_res.empty:
|
||||
df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city", "region", 'count'], index=False)
|
||||
if not df_root_city_res.empty:
|
||||
df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root', 'city', 'count'],
|
||||
index=False)
|
||||
if not df_root_res.empty:
|
||||
df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root', count_old, count_new],
|
||||
index=False)
|
||||
up_to_nextcloud()
|
||||
token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736'
|
||||
dingutils.send_text(text, token)
|
||||
if __name__ == '__main__':
|
||||
text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new = proc()
|
||||
if text != '':
|
||||
with pd.ExcelWriter(air_file_name) as writer:
|
||||
if not df_region_res.empty:
|
||||
df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city", "region", 'count'], index=False)
|
||||
if not df_root_city_res.empty:
|
||||
df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root', 'city', 'count'],
|
||||
index=False)
|
||||
if not df_root_res.empty:
|
||||
df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root', count_old, count_new],
|
||||
index=False)
|
||||
up_to_nextcloud()
|
||||
token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736'
|
||||
dingutils.send_text(text, token)
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
"""
|
||||
写字楼基准价计算
|
||||
"""
|
||||
from airflow.operators.bash import BashOperator
|
||||
from airflow.operators.python import PythonOperator
|
||||
|
||||
import pendulum
|
||||
from office.rent.base_price import calc
|
||||
|
||||
from airflow import DAG
|
||||
|
||||
@@ -14,9 +15,9 @@ with DAG(
|
||||
catchup=True,
|
||||
description="写字楼基准价计算"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
t1 = PythonOperator(
|
||||
task_id='base_price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/office/rent/base_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}',
|
||||
python_callable=calc,
|
||||
provide_context=True
|
||||
)
|
||||
t1
|
||||
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
写字楼基准价计算
|
||||
"""
|
||||
from airflow.operators.bash import BashOperator
|
||||
from airflow.operators.python import PythonOperator
|
||||
from office.sell.base_price import calc
|
||||
from office.sell.fill_base_price import fill_base_price
|
||||
|
||||
import pendulum
|
||||
|
||||
@@ -14,13 +17,15 @@ with DAG(
|
||||
catchup=True,
|
||||
description="写字楼出售基准价计算"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='sell_base_price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/office/sell/base_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}',
|
||||
t1 = PythonOperator(
|
||||
task_id='calc',
|
||||
python_callable=calc,
|
||||
provide_context=True
|
||||
)
|
||||
t2 = BashOperator(
|
||||
t2 = PythonOperator(
|
||||
task_id='fill_base_price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/office/sell/fill_base_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}',
|
||||
python_callable=fill_base_price,
|
||||
provide_context=True
|
||||
)
|
||||
t1 >> t2
|
||||
|
||||
|
||||
+240
-117
@@ -10,7 +10,7 @@
|
||||
#
|
||||
# 第五步:本身没有等级和类型的写字楼
|
||||
# 最近10个有样本写字楼的基准价的平均值
|
||||
from goojia_utils import ElasticSearchClient, dateutils, MAIN_MONGO_CLIENT
|
||||
from goojia_utils import ElasticSearchClient, dateutils, MAIN_MONGO_CLIENT, commutils
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
from pymongo import UpdateOne
|
||||
@@ -23,7 +23,111 @@ seaweed_df = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat')
|
||||
|
||||
# 计算有样本写字楼的基准价 1. 取近一年,最近100个样本的中位值
|
||||
def calc(city, year_month):
|
||||
|
||||
def calc(**kwargs):
|
||||
"""
|
||||
写字楼基准价
|
||||
"""
|
||||
# 从 Airflow 的 dag_run.conf 获取参数
|
||||
conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
|
||||
city = conf.get('city', '')
|
||||
year_month = conf.get('year_month', '')
|
||||
district = conf.get('district', '')
|
||||
office_name = conf.get('name', '')
|
||||
|
||||
if office_name == '':
|
||||
calcByCity(city, year_month)
|
||||
else:
|
||||
# 直接重算出最近一年的数据,保证一年有数据
|
||||
calcByOffice(city, district, office_name)
|
||||
|
||||
def calcByOffice(city, district, office_name):
|
||||
"""
|
||||
写字楼价格重算
|
||||
"""
|
||||
query = {'cat': 'office', 'status': 0}
|
||||
if pd.notna(city):
|
||||
query['city'] = city
|
||||
if pd.notna(district):
|
||||
query['region'] = district
|
||||
if pd.notna(city):
|
||||
query['name'] = office_name
|
||||
office = seaweed_df.find_one(query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'lng': 1, 'lat': 1, 'estate_type2': 1, 'bed_kind': 1, '_id': 0})
|
||||
if 'estate_type2' not in office:
|
||||
office['estate_type2'] = None
|
||||
if 'bed_kind' not in office:
|
||||
office['bed_kind'] = None
|
||||
names = list()
|
||||
names.append(office['name'])
|
||||
if ('alias' in office) and (type(office['alias']) == list):
|
||||
for alias_item in office['alias']:
|
||||
names.append(alias_item)
|
||||
names_str = "','".join(names)
|
||||
names_str = "'" + names_str + "'"
|
||||
|
||||
# 只查询该小区的样本
|
||||
year_time = dateutils.format_date(dateutils.datetime.datetime.now())
|
||||
one_year_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-365))
|
||||
sql = f"select * from latest_seawater where cat='office_rent' and city='{city}' and region='{district}' and district_name in ({names_str}) and p_date > {one_year_time}"
|
||||
sample = es.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
location = {
|
||||
"type": "Point",
|
||||
"coordinates": [office['lng'] / math.pow(10, 10), office['lat'] / math.pow(10, 10)]
|
||||
}
|
||||
|
||||
date_range = pd.date_range(start=one_year_time, end=year_time, freq='M')
|
||||
for date in date_range:
|
||||
curr_date = int(dateutils.format_date(date, fm='%Y%m'))
|
||||
# 筛选出当前月的样本
|
||||
if len(sample) > 0:
|
||||
curr_sample = sample[(sample['p_date']>=int(str(curr_date)+'01')) & (sample['p_date']<=int(str(curr_date)+'31'))]
|
||||
else:
|
||||
curr_sample = pd.DataFrame(columns=['city', 'region', 'district_name', 'avg_price'])
|
||||
update_office(city, pd.DataFrame([office]), curr_sample, curr_date)
|
||||
# 核查当月写字楼是否存在,不存在取最后6个月的平均值
|
||||
office_db = office_rent_stat_conn.find_one({'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date})
|
||||
if pd.isna(office_db):
|
||||
avg_price_ret = office_rent_stat_conn.aggregate([
|
||||
{
|
||||
"$match": {
|
||||
"cat": "office",
|
||||
"city": city,
|
||||
"district": district,
|
||||
"name": office_name,
|
||||
"date": { "$gte": curr_date - 3, "$lte": curr_date + 3 }
|
||||
}
|
||||
},
|
||||
{
|
||||
"$group": {
|
||||
"_id": "$name",
|
||||
"avg_price": { "$avg": "$avg_price" }
|
||||
}
|
||||
}
|
||||
])
|
||||
avg_prices = [doc["avg_price"] for doc in avg_price_ret]
|
||||
# 计算平均值
|
||||
average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0
|
||||
new_avg_price = None
|
||||
if average_price > 0:
|
||||
new_avg_price = round(average_price, 2)
|
||||
elif len(sample) > 0:
|
||||
# 取样本的平均值
|
||||
new_avg_price = round(sample['avg_price'].mean(), 2)
|
||||
if pd.notna(new_avg_price):
|
||||
action = UpdateOne(
|
||||
{'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': curr_date, 'district': district,
|
||||
'name': office_name,
|
||||
'avg_price': new_avg_price, 'estate_type2': office['estate_type2'],
|
||||
'bed_kind': office['bed_kind'],
|
||||
'location': location,
|
||||
'calc_type': 5, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
office_rent_stat_conn.bulk_write([action])
|
||||
|
||||
|
||||
def calcByCity(city, year_month):
|
||||
"""
|
||||
写字楼基准价
|
||||
"""
|
||||
@@ -46,33 +150,43 @@ def calc(city, year_month):
|
||||
print("开始计算年" + str(year_month) + "写字楼基准价")
|
||||
|
||||
sample = es.scan_by_sql_to_data_frame(sql)
|
||||
query = {'cat': 'office', 'status': 0}
|
||||
if pd.notna(city):
|
||||
query['city'] = city
|
||||
office = pd.DataFrame(list(seaweed_df.find(query).batch_size(10000)))
|
||||
office = office[['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']]
|
||||
office = office.explode('alias')
|
||||
office = getOffice(city)
|
||||
|
||||
update_office(city, office, sample, year_month)
|
||||
|
||||
|
||||
def update_office(city, office, sample, year_month):
|
||||
if 'bed_kind' not in office.columns:
|
||||
office['bed_kind'] = None
|
||||
if 'estate_type2' not in office.columns:
|
||||
office['estate_type2'] = None
|
||||
sample_new = sample
|
||||
sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform(
|
||||
'median')
|
||||
sample_new['sample_count'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform(
|
||||
'count')
|
||||
sample_new = sample_new[sample_new['sample_count'] >= 5]
|
||||
sample_new = sample_new[['city', 'region', 'district_name', 'sample_avg_price']].dropna()
|
||||
sample_new['day_avg_price'] = sample_new['sample_avg_price'] / 30
|
||||
sample_new = sample_new[['city', 'region', 'district_name', 'day_avg_price']]
|
||||
sample_new = sample_new.rename(columns={'district_name': 'name'})
|
||||
sample_new['alias'] = sample_new['name']
|
||||
|
||||
# 使用主名与别名查找样本
|
||||
merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left',
|
||||
on=['city', 'region', 'name'])
|
||||
merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left',
|
||||
on=['city', 'region', 'alias'])
|
||||
merge_df = pd.concat([merge_df1, merge_df2])
|
||||
merge_df = merge_df[merge_df['day_avg_price'].notna()]
|
||||
merge_df = merge_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first')
|
||||
if 'alias' not in sample_new.columns:
|
||||
sample_new['alias'] = None
|
||||
if len(sample_new) > 0:
|
||||
sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform(
|
||||
'median')
|
||||
sample_new['sample_count'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform(
|
||||
'count')
|
||||
sample_new = sample_new[sample_new['sample_count'] >= 5]
|
||||
sample_new = sample_new[['city', 'region', 'district_name', 'sample_avg_price']].dropna()
|
||||
sample_new['day_avg_price'] = sample_new['sample_avg_price'] / 30
|
||||
sample_new = sample_new[['city', 'region', 'district_name', 'day_avg_price']]
|
||||
sample_new = sample_new.rename(columns={'district_name': 'name'})
|
||||
sample_new['alias'] = sample_new['name']
|
||||
# 使用主名与别名查找样本
|
||||
merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left',
|
||||
on=['city', 'region', 'name'])
|
||||
if len(sample_new['alias'].unique()) > 0:
|
||||
merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left',
|
||||
on=['city', 'region', 'alias'])
|
||||
merge_df = pd.concat([merge_df1, merge_df2])
|
||||
else:
|
||||
merge_df = merge_df1.copy()
|
||||
merge_df = merge_df[merge_df['day_avg_price'].notna()]
|
||||
merge_df = merge_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first')
|
||||
else:
|
||||
merge_df = pd.DataFrame()
|
||||
# 第一步:计算有样本写字楼的基准价
|
||||
print("有样本数量:" + str(len(merge_df)))
|
||||
actions = list()
|
||||
@@ -88,7 +202,8 @@ def calc(city, year_month):
|
||||
estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None
|
||||
bed_kind = item['bed_kind'] if pd.notna(item['bed_kind']) else None
|
||||
action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, 'name': name,
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind,
|
||||
'location': location,
|
||||
'calc_type': 1, 'updated_date': datetime.now()}},
|
||||
@@ -100,13 +215,14 @@ def calc(city, year_month):
|
||||
if len(actions) > 0:
|
||||
office_rent_stat_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
|
||||
# 第二步 取周边3公里内同等级、同类型的最近10个有样本写字楼的基准价的平均值
|
||||
office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates(keep='first')
|
||||
office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates(
|
||||
keep='first')
|
||||
combined = pd.concat([office_info, merge_df])
|
||||
diff_df = combined.drop_duplicates(subset=['city', 'region', 'name'], keep=False)
|
||||
actions = list()
|
||||
for index, item in diff_df[(diff_df['estate_type2'].notna()) & (diff_df['bed_kind'].notna()) & (diff_df['lng'].notna()) ].iterrows():
|
||||
for index, item in diff_df[
|
||||
(diff_df['estate_type2'].notna()) & (diff_df['bed_kind'].notna()) & (diff_df['lng'].notna())].iterrows():
|
||||
location = {
|
||||
"type": "Point",
|
||||
"coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)]
|
||||
@@ -159,7 +275,6 @@ def calc(city, year_month):
|
||||
if len(actions) > 0:
|
||||
office_rent_stat_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
|
||||
# 系数计算在Jupyter中
|
||||
bed_kind_dict = {'甲级': 1.111888111888112, '丙级': 0.9143356643356645, '乙级': 1}
|
||||
radio_dict = {'创业园区': 0.7799999999999999, '商住两用': 1.0466666666666666, '标准写字楼': 1}
|
||||
@@ -210,13 +325,14 @@ def calc(city, year_month):
|
||||
count += 1
|
||||
if count > 0:
|
||||
avg_price = round(sum_price / count, 2)
|
||||
action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None,
|
||||
'location': location,
|
||||
'calc_type': 3, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
action = UpdateOne(
|
||||
{'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None,
|
||||
'location': location,
|
||||
'calc_type': 3, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
elif pd.notna(bed_kind):
|
||||
ret = office_rent_stat_conn.aggregate([
|
||||
@@ -249,13 +365,14 @@ def calc(city, year_month):
|
||||
count += 1
|
||||
if count > 0:
|
||||
avg_price = round(sum_price / count, 2)
|
||||
action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': avg_price, 'estate_type2': None, 'bed_kind': bed_kind,
|
||||
'location': location,
|
||||
'calc_type': 3, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
action = UpdateOne(
|
||||
{'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': avg_price, 'estate_type2': None, 'bed_kind': bed_kind,
|
||||
'location': location,
|
||||
'calc_type': 3, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
if len(actions) == 1000:
|
||||
office_rent_stat_conn.bulk_write(actions)
|
||||
@@ -273,79 +390,85 @@ def calc(city, year_month):
|
||||
# 计算价格的写字楼
|
||||
db_df = db_df.rename(columns={'district': 'region'})
|
||||
# 使用 merge 操作并结合 indicator 参数
|
||||
merged = office_info.merge(db_df[['city', 'region', 'name']], on=['city', 'region', 'name'], how='left',
|
||||
indicator=True)
|
||||
|
||||
# 过滤出仅在 office_info 中存在的记录
|
||||
diff_df = merged[merged['_merge'] == 'left_only'].drop(columns=['_merge'])
|
||||
|
||||
# 如果需要保持原始列顺序,可以重新排列列
|
||||
diff_df = diff_df[office_info.columns]
|
||||
|
||||
actions.clear()
|
||||
for index, item in diff_df.iterrows():
|
||||
try:
|
||||
location = {
|
||||
"type": "Point",
|
||||
"coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)]
|
||||
}
|
||||
city = item['city']
|
||||
region = item['region']
|
||||
name = item['name']
|
||||
estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None
|
||||
bed_kind = item['bed_kind']
|
||||
ret = office_rent_stat_conn.aggregate([
|
||||
{
|
||||
'$geoNear': {
|
||||
'near': location,
|
||||
'distanceField': "distance",
|
||||
'maxDistance': 3000,
|
||||
'spherical': True
|
||||
}
|
||||
},
|
||||
{
|
||||
'$match': {
|
||||
'date': year_month
|
||||
}
|
||||
},
|
||||
{
|
||||
'$limit': 5
|
||||
if len(db_df) > 0:
|
||||
merged = office_info.merge(db_df[['city', 'region', 'name']], on=['city', 'region', 'name'], how='left',
|
||||
indicator=True)
|
||||
# 过滤出仅在 office_info 中存在的记录
|
||||
diff_df = merged[merged['_merge'] == 'left_only'].drop(columns=['_merge'])
|
||||
# 如果需要保持原始列顺序,可以重新排列列
|
||||
diff_df = diff_df[office_info.columns]
|
||||
actions.clear()
|
||||
for index, item in diff_df.iterrows():
|
||||
try:
|
||||
location = {
|
||||
"type": "Point",
|
||||
"coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)]
|
||||
}
|
||||
]);
|
||||
except Exception as e:
|
||||
print("Error occurred:", str(e))
|
||||
print(item)
|
||||
# 提取 avg_price 的值
|
||||
avg_prices = [doc["avg_price"] for doc in ret]
|
||||
# 计算平均值
|
||||
average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0
|
||||
if average_price > 0:
|
||||
avg_price = round(average_price, 2)
|
||||
action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None,
|
||||
'location': location,
|
||||
'calc_type': 4, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
if len(actions) == 1000:
|
||||
office_rent_stat_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
city = item['city']
|
||||
region = item['region']
|
||||
name = item['name']
|
||||
estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None
|
||||
bed_kind = item['bed_kind']
|
||||
ret = office_rent_stat_conn.aggregate([
|
||||
{
|
||||
'$geoNear': {
|
||||
'near': location,
|
||||
'distanceField': "distance",
|
||||
'maxDistance': 3000,
|
||||
'spherical': True
|
||||
}
|
||||
},
|
||||
{
|
||||
'$match': {
|
||||
'date': year_month
|
||||
}
|
||||
},
|
||||
{
|
||||
'$limit': 5
|
||||
}
|
||||
]);
|
||||
except Exception as e:
|
||||
print("Error occurred:", str(e))
|
||||
print(item)
|
||||
# 提取 avg_price 的值
|
||||
avg_prices = [doc["avg_price"] for doc in ret]
|
||||
# 计算平均值
|
||||
average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0
|
||||
if average_price > 0:
|
||||
avg_price = round(average_price, 2)
|
||||
action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None,
|
||||
'location': location,
|
||||
'calc_type': 4, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
if len(actions) == 1000:
|
||||
office_rent_stat_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
office_rent_stat_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
|
||||
yearmonth = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
# for yearmonth in [
|
||||
# "202406",
|
||||
# "202407",
|
||||
# "202408",
|
||||
# "202409",
|
||||
# "202410",
|
||||
# "202411",
|
||||
# "202412",
|
||||
# "202501"
|
||||
# ]:
|
||||
# calc(None, yearmonth)
|
||||
calc(None, yearmonth)
|
||||
|
||||
def getOffice(city, district, name):
|
||||
"""
|
||||
获取写字楼
|
||||
"""
|
||||
query = {'cat': 'office', 'status': 0}
|
||||
if pd.notna(city):
|
||||
query['city'] = city
|
||||
if pd.notna(district):
|
||||
query['region'] = district
|
||||
if pd.notna(city):
|
||||
query['name'] = name
|
||||
office = pd.DataFrame(list(seaweed_df.find(query).batch_size(10000)))
|
||||
office = office[['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']]
|
||||
office = office.explode('alias')
|
||||
return office
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 测试脚本:直接运行时模拟 Airflow 的上下文
|
||||
calc(dag_run=type('DagRun', (), {'conf': {'city': '北京', 'district': '朝阳', 'name': '中核华建大厦'}}))
|
||||
+218
-108
@@ -10,7 +10,7 @@
|
||||
#
|
||||
# 第五步:本身没有等级和类型的写字楼
|
||||
# 最近10个有样本写字楼的基准价的平均值
|
||||
from goojia_utils import ElasticSearchClient, dateutils, MAIN_MONGO_CLIENT
|
||||
from goojia_utils import ElasticSearchClient, dateutils, MAIN_MONGO_CLIENT, commutils
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
from pymongo import UpdateOne
|
||||
@@ -23,7 +23,107 @@ seaweed_df = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat')
|
||||
|
||||
# 计算有样本写字楼的基准价 1. 取近一年,最近100个样本的中位值
|
||||
def calc(city, year_month):
|
||||
|
||||
def calc(**kwargs):
|
||||
"""
|
||||
写字楼基准价
|
||||
"""
|
||||
# 从 Airflow 的 dag_run.conf 获取参数
|
||||
conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
|
||||
city = conf.get('city', '')
|
||||
year_month = conf.get('year_month', '')
|
||||
district = conf.get('district', '')
|
||||
office_name = conf.get('name', '')
|
||||
|
||||
if pd.isna(office_name):
|
||||
calcByCity(city, year_month)
|
||||
else:
|
||||
# 直接重算出最近一年的数据,保证一年有数据
|
||||
calcByOffice(city, district, office_name)
|
||||
|
||||
def calcByOffice(city, district, office_name):
|
||||
"""
|
||||
写字楼价格重算
|
||||
"""
|
||||
query = {'cat': 'office', 'status': 0}
|
||||
if pd.notna(city):
|
||||
query['city'] = city
|
||||
if pd.notna(district):
|
||||
query['region'] = district
|
||||
if pd.notna(city):
|
||||
query['name'] = office_name
|
||||
office = seaweed_df.find_one(query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'lng': 1, 'lat': 1, 'estate_type2': 1, 'bed_kind': 1})
|
||||
if 'estate_type2' not in office:
|
||||
office['estate_type2'] = None
|
||||
if 'bed_kind' not in office:
|
||||
office['bed_kind'] = None
|
||||
names = list()
|
||||
names.append(office['name'])
|
||||
if ('alias' in office) and (type(office['alias']) == list):
|
||||
for alias_item in office['alias']:
|
||||
names.append(alias_item)
|
||||
names_str = "','".join(names)
|
||||
names_str = "'" + names_str + "'"
|
||||
|
||||
# 只查询该小区的样本
|
||||
year_time = dateutils.format_date(dateutils.datetime.datetime.now())
|
||||
one_year_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-365))
|
||||
sql = f"select * from latest_seawater where cat='office_sell' and city='{city}' and region='{district}' and district_name in ({names_str}) and p_date > {one_year_time}"
|
||||
sample = es.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
location = {
|
||||
"type": "Point",
|
||||
"coordinates": [office['lng'] / math.pow(10, 10), office['lat'] / math.pow(10, 10)]
|
||||
}
|
||||
|
||||
date_range = pd.date_range(start=one_year_time, end=year_time, freq='M')
|
||||
for date in date_range:
|
||||
curr_date = int(dateutils.format_date(date, fm='%Y%m'))
|
||||
# 筛选出当前月的样本
|
||||
curr_sample = sample[(sample['p_date']>=int(str(curr_date)+'01')) & (sample['p_date']<=int(str(curr_date)+'31'))]
|
||||
update_office(city, pd.DataFrame([office]), curr_sample, curr_date)
|
||||
# 核查当月写字楼是否存在,不存在取最后6个月的平均值
|
||||
office_db = office_sell_stat_conn.find_one({'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date})
|
||||
if pd.isna(office_db):
|
||||
avg_price_ret = office_sell_stat_conn.aggregate([
|
||||
{
|
||||
"$match": {
|
||||
"cat": "office",
|
||||
"city": city,
|
||||
"district": district,
|
||||
"name": office_name,
|
||||
"date": { "$gte": curr_date - 3, "$lte": curr_date + 3 }
|
||||
}
|
||||
},
|
||||
{
|
||||
"$group": {
|
||||
"_id": "$name",
|
||||
"avg_price": { "$avg": "$avg_price" }
|
||||
}
|
||||
}
|
||||
])
|
||||
avg_prices = [doc["avg_price"] for doc in avg_price_ret]
|
||||
# 计算平均值
|
||||
average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0
|
||||
if average_price > 0:
|
||||
new_avg_price = int(average_price)
|
||||
else:
|
||||
# 取样本的平均值
|
||||
new_avg_price = int(sample['avg_price'].mean())
|
||||
if pd.notna(new_avg_price):
|
||||
action = UpdateOne(
|
||||
{'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': curr_date, 'district': district,
|
||||
'name': office_name,
|
||||
'avg_price': new_avg_price, 'estate_type2': office['estate_type2'],
|
||||
'bed_kind': office['bed_kind'],
|
||||
'location': location,
|
||||
'calc_type': 5, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
office_sell_stat_conn.bulk_write([action])
|
||||
|
||||
|
||||
def calcByCity(city, year_month):
|
||||
"""
|
||||
写字楼基准价
|
||||
"""
|
||||
@@ -46,14 +146,19 @@ def calc(city, year_month):
|
||||
print("开始计算年" + str(year_month) + "写字楼基准价")
|
||||
|
||||
sample = es.scan_by_sql_to_data_frame(sql)
|
||||
query = {'cat': 'office', 'status': 0}
|
||||
if pd.notna(city):
|
||||
query['city'] = city
|
||||
office = pd.DataFrame(list(seaweed_df.find(query).batch_size(10000)))
|
||||
office = office[['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']]
|
||||
office = office.explode('alias')
|
||||
office = getOffice(city)
|
||||
|
||||
update_office(city, office, sample, year_month)
|
||||
|
||||
|
||||
def update_office(city, office, sample, year_month):
|
||||
if 'bed_kind' not in office.columns:
|
||||
office['bed_kind'] = None
|
||||
if 'estate_type2' not in office.columns:
|
||||
office['estate_type2'] = None
|
||||
sample_new = sample
|
||||
if 'alias' not in sample_new.columns:
|
||||
sample_new['alias'] = None
|
||||
sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform(
|
||||
'median')
|
||||
sample_new['sample_count'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform(
|
||||
@@ -64,19 +169,20 @@ def calc(city, year_month):
|
||||
sample_new = sample_new[['city', 'region', 'district_name', 'day_avg_price']]
|
||||
sample_new = sample_new.rename(columns={'district_name': 'name'})
|
||||
sample_new['alias'] = sample_new['name']
|
||||
|
||||
# 使用主名与别名查找样本
|
||||
merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left',
|
||||
on=['city', 'region', 'name'])
|
||||
merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left',
|
||||
on=['city', 'region', 'alias'])
|
||||
merge_df = pd.concat([merge_df1, merge_df2])
|
||||
if len(sample_new['alias'].unique()) > 0:
|
||||
merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left',
|
||||
on=['city', 'region', 'alias'])
|
||||
merge_df = pd.concat([merge_df1, merge_df2])
|
||||
else:
|
||||
merge_df = merge_df1.copy()
|
||||
merge_df = merge_df[merge_df['day_avg_price'].notna()]
|
||||
merge_df = merge_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first')
|
||||
# 第一步:计算有样本写字楼的基准价
|
||||
print("有样本数量:" + str(len(merge_df)))
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in merge_df.iterrows():
|
||||
city = item['city']
|
||||
region = item['region']
|
||||
@@ -89,7 +195,8 @@ def calc(city, year_month):
|
||||
estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None
|
||||
bed_kind = item['bed_kind'] if pd.notna(item['bed_kind']) else None
|
||||
action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, 'name': name,
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind,
|
||||
'location': location,
|
||||
'calc_type': 1, 'updated_date': datetime.now()}},
|
||||
@@ -97,21 +204,18 @@ def calc(city, year_month):
|
||||
actions.append(action)
|
||||
if len(actions) == 1000:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已计算" + str(count) + "条数据")
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已计算" + str(count) + "条数据")
|
||||
actions.clear()
|
||||
|
||||
# 第二步 取周边3公里内同等级、同类型的最近10个有样本写字楼的基准价的平均值
|
||||
office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates(keep='first')
|
||||
office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates(
|
||||
keep='first')
|
||||
combined = pd.concat([office_info, merge_df])
|
||||
diff_df = combined.drop_duplicates(subset=['city', 'region', 'name'], keep=False)
|
||||
actions = list()
|
||||
for index, item in diff_df[(diff_df['estate_type2'].notna()) & (diff_df['bed_kind'].notna()) & (diff_df['lng'].notna()) ].iterrows():
|
||||
for index, item in diff_df[
|
||||
(diff_df['estate_type2'].notna()) & (diff_df['bed_kind'].notna()) & (diff_df['lng'].notna())].iterrows():
|
||||
location = {
|
||||
"type": "Point",
|
||||
"coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)]
|
||||
@@ -160,15 +264,10 @@ def calc(city, year_month):
|
||||
actions.append(action)
|
||||
if len(actions) == 1000:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已计算" + str(count) + "条数据")
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已计算" + str(count) + "条数据")
|
||||
actions.clear()
|
||||
|
||||
# 系数计算在Jupyter中
|
||||
bed_kind_dict = {'甲级': 1.111888111888112, '丙级': 0.9143356643356645, '乙级': 1}
|
||||
radio_dict = {'创业园区': 0.7799999999999999, '商住两用': 1.0466666666666666, '标准写字楼': 1}
|
||||
@@ -219,13 +318,14 @@ def calc(city, year_month):
|
||||
count += 1
|
||||
if count > 0:
|
||||
avg_price = int(sum_price / count)
|
||||
action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None,
|
||||
'location': location,
|
||||
'calc_type': 3, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
action = UpdateOne(
|
||||
{'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None,
|
||||
'location': location,
|
||||
'calc_type': 3, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
elif pd.notna(bed_kind):
|
||||
ret = office_sell_stat_conn.aggregate([
|
||||
@@ -258,23 +358,20 @@ def calc(city, year_month):
|
||||
count += 1
|
||||
if count > 0:
|
||||
avg_price = int(sum_price / count)
|
||||
action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': avg_price, 'estate_type2': None, 'bed_kind': bed_kind,
|
||||
'location': location,
|
||||
'calc_type': 3, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
action = UpdateOne(
|
||||
{'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': avg_price, 'estate_type2': None, 'bed_kind': bed_kind,
|
||||
'location': location,
|
||||
'calc_type': 3, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
if len(actions) == 1000:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已计算" + str(count) + "条数据")
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已计算" + str(count) + "条数据")
|
||||
actions.clear()
|
||||
# 第五步
|
||||
query = {'cat': 'office', 'date': year_month, 'calc_type': {'$in': [1, 2, 3]}}
|
||||
@@ -286,72 +383,85 @@ def calc(city, year_month):
|
||||
# 计算价格的写字楼
|
||||
db_df = db_df.rename(columns={'district': 'region'})
|
||||
# 使用 merge 操作并结合 indicator 参数
|
||||
merged = office_info.merge(db_df[['city', 'region', 'name']], on=['city', 'region', 'name'], how='left',
|
||||
indicator=True)
|
||||
|
||||
# 过滤出仅在 office_info 中存在的记录
|
||||
diff_df = merged[merged['_merge'] == 'left_only'].drop(columns=['_merge'])
|
||||
|
||||
# 如果需要保持原始列顺序,可以重新排列列
|
||||
diff_df = diff_df[office_info.columns]
|
||||
|
||||
actions.clear()
|
||||
for index, item in diff_df.iterrows():
|
||||
try:
|
||||
location = {
|
||||
"type": "Point",
|
||||
"coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)]
|
||||
}
|
||||
city = item['city']
|
||||
region = item['region']
|
||||
name = item['name']
|
||||
estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None
|
||||
bed_kind = item['bed_kind']
|
||||
ret = office_sell_stat_conn.aggregate([
|
||||
{
|
||||
'$geoNear': {
|
||||
'near': location,
|
||||
'distanceField': "distance",
|
||||
'maxDistance': 3000,
|
||||
'spherical': True
|
||||
}
|
||||
},
|
||||
{
|
||||
'$match': {
|
||||
'date': year_month
|
||||
}
|
||||
},
|
||||
{
|
||||
'$limit': 5
|
||||
if len(db_df) > 0:
|
||||
merged = office_info.merge(db_df[['city', 'region', 'name']], on=['city', 'region', 'name'], how='left',
|
||||
indicator=True)
|
||||
# 过滤出仅在 office_info 中存在的记录
|
||||
diff_df = merged[merged['_merge'] == 'left_only'].drop(columns=['_merge'])
|
||||
# 如果需要保持原始列顺序,可以重新排列列
|
||||
diff_df = diff_df[office_info.columns]
|
||||
actions.clear()
|
||||
for index, item in diff_df.iterrows():
|
||||
try:
|
||||
location = {
|
||||
"type": "Point",
|
||||
"coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)]
|
||||
}
|
||||
]);
|
||||
except Exception as e:
|
||||
print("Error occurred:", str(e))
|
||||
print(item)
|
||||
# 提取 avg_price 的值
|
||||
avg_prices = [doc["avg_price"] for doc in ret]
|
||||
# 计算平均值
|
||||
average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0
|
||||
if average_price > 0:
|
||||
avg_price = int(average_price)
|
||||
action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None,
|
||||
'location': location,
|
||||
'calc_type': 4, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
if len(actions) == 1000:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已计算" + str(count) + "条数据")
|
||||
actions.clear()
|
||||
city = item['city']
|
||||
region = item['region']
|
||||
name = item['name']
|
||||
estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None
|
||||
bed_kind = item['bed_kind']
|
||||
ret = office_sell_stat_conn.aggregate([
|
||||
{
|
||||
'$geoNear': {
|
||||
'near': location,
|
||||
'distanceField': "distance",
|
||||
'maxDistance': 3000,
|
||||
'spherical': True
|
||||
}
|
||||
},
|
||||
{
|
||||
'$match': {
|
||||
'date': year_month
|
||||
}
|
||||
},
|
||||
{
|
||||
'$limit': 5
|
||||
}
|
||||
]);
|
||||
except Exception as e:
|
||||
print("Error occurred:", str(e))
|
||||
print(item)
|
||||
# 提取 avg_price 的值
|
||||
avg_prices = [doc["avg_price"] for doc in ret]
|
||||
# 计算平均值
|
||||
average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0
|
||||
if average_price > 0:
|
||||
avg_price = int(average_price)
|
||||
action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None,
|
||||
'location': location,
|
||||
'calc_type': 4, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
if len(actions) == 1000:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已计算" + str(count) + "条数据")
|
||||
actions.clear()
|
||||
|
||||
yearmonth = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
calc(None, yearmonth)
|
||||
|
||||
def getOffice(city, district, name):
|
||||
"""
|
||||
获取写字楼
|
||||
"""
|
||||
query = {'cat': 'office', 'status': 0}
|
||||
if pd.notna(city):
|
||||
query['city'] = city
|
||||
if pd.notna(district):
|
||||
query['region'] = district
|
||||
if pd.notna(city):
|
||||
query['name'] = name
|
||||
office = pd.DataFrame(list(seaweed_df.find(query).batch_size(10000)))
|
||||
office = office[['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']]
|
||||
office = office.explode('alias')
|
||||
return office
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 测试脚本:直接运行时模拟 Airflow 的上下文
|
||||
calc(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}}))
|
||||
@@ -11,18 +11,40 @@ office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office
|
||||
office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat')
|
||||
|
||||
|
||||
def calc(city, year_month):
|
||||
query = {'date': year_month, 'cat': 'office'}
|
||||
if pd.notna(city):
|
||||
def fill_base_price(**kwargs):
|
||||
"""
|
||||
写字楼基准价
|
||||
"""
|
||||
# 从 Airflow 的 dag_run.conf 获取参数
|
||||
conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
|
||||
city = conf.get('city', '')
|
||||
year_month = conf.get('year_month', '')
|
||||
district = conf.get('district', '')
|
||||
office_name = conf.get('name', '')
|
||||
|
||||
query = {'cat': 'office'}
|
||||
if year_month != '':
|
||||
query['date'] = year_month
|
||||
if city != '':
|
||||
query['city'] = city
|
||||
if district != '' and office_name != '':
|
||||
query['district'] = district
|
||||
query['name'] = office_name
|
||||
office_sell = pd.DataFrame(list(office_sell_stat_conn.find(query)))
|
||||
office_rent = pd.DataFrame(list(office_rent_stat_conn.find(query)))
|
||||
|
||||
if len(office_rent) <= 0:
|
||||
return
|
||||
if year_month != '':
|
||||
calc_by_yearmonth(office_sell, office_rent, year_month)
|
||||
else:
|
||||
# 对于单个重算的不进行补充,已经自己补充过了
|
||||
pass
|
||||
|
||||
def calc_by_yearmonth(office_sell, office_rent, year_month):
|
||||
office_sell_df = office_sell[['city', 'district', 'name', 'avg_price']]
|
||||
office_sell_df = office_sell_df.rename(columns={'avg_price': 'sell_price'})
|
||||
office_rent_df = office_rent[['city', 'district', 'name', 'avg_price', 'estate_type2', 'bed_kind','location']]
|
||||
office_rent_df = office_rent[['city', 'district', 'name', 'avg_price', 'estate_type2', 'bed_kind', 'location']]
|
||||
office_rent_df = office_rent_df.rename(columns={'avg_price': 'rent_price'})
|
||||
# 每月动态计算系数,因为不同月份会有差异
|
||||
merge_df = pd.merge(office_sell_df, office_rent_df, how='left', on=['city', 'district', 'name'])
|
||||
@@ -43,13 +65,13 @@ def calc(city, year_month):
|
||||
填充存在租金,不存在售价的数据
|
||||
"""
|
||||
if item['city'] + "_" + item['district'] in district_radio.keys():
|
||||
item['sell_price'] = round(district_radio[item['city'] + "_" + item['district']] * item['rent_price'] * 30 * 12, 0)
|
||||
item['sell_price'] = round(
|
||||
district_radio[item['city'] + "_" + item['district']] * item['rent_price'] * 30 * 12, 0)
|
||||
elif item['city'] in city_radio.keys():
|
||||
item['sell_price'] = round(city_radio[item['city']] * item['rent_price'] * 30 * 12, 0)
|
||||
return item
|
||||
|
||||
if len(merge2_df) == 0:
|
||||
print(city + "无数据补充")
|
||||
return
|
||||
merge2_df = threads.process_dataframe_multithread(merge2_df, deal, 10)
|
||||
merge2_df = merge2_df[merge2_df['sell_price'].notna()]
|
||||
@@ -83,16 +105,7 @@ def calc(city, year_month):
|
||||
print("已处理{}条数据".format(count))
|
||||
print("处理完毕")
|
||||
|
||||
|
||||
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
|
||||
if yearmonth == '':
|
||||
date = dateutils.add_months(datetime.today(), months=-1)
|
||||
yearmonth = int(dateutils.format_date(date, '%Y%m'))
|
||||
try:
|
||||
if type(yearmonth) is str:
|
||||
yearmonth = int(yearmonth)
|
||||
calc(None, yearmonth)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
if __name__ == "__main__":
|
||||
# 测试脚本:直接运行时模拟 Airflow 的上下文
|
||||
fill_base_price(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}}))
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ es_client = ElasticSearchClient(host='192.168.0.206')
|
||||
office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat')
|
||||
office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat')
|
||||
|
||||
|
||||
def calc(city, year_month):
|
||||
print(f"开始处理{city}, {year_month}")
|
||||
query = {'date': year_month, 'cat': 'office'}
|
||||
|
||||
Reference in New Issue
Block a user