Merge branch 'master' of http://gitlab.fangjia.com/dev0/airflow
This commit is contained in:
@@ -7,4 +7,5 @@ pip install -r requirements.txt -i https://repo.fangjia.com/repository/pypi-publ
|
||||
|
||||
- 具体业务逻辑放入xxx/business中,如poi/business
|
||||
- 添加.airflowignore xxx/business/*, 如poi/business/*
|
||||
- dag放在xxx目录下即可,命名为xx_dag.py
|
||||
- dag放在xxx目录下即可,命名为dag_xx.py,之前命名为xx_dag导致xx与报名相同而加入airflowignore后dag不显示问题,
|
||||
如merge_dag会不显示,而dag_merge即可,怀疑是jh/merge/*的bug
|
||||
@@ -1,6 +1,9 @@
|
||||
poi/business/.*
|
||||
poi/business/*
|
||||
stat/evaluate/*
|
||||
stat/community/*
|
||||
mt/supply/*
|
||||
business/business/*
|
||||
plugin/**
|
||||
jh/merge/*
|
||||
jh/para/*
|
||||
jh/price/*
|
||||
db/*
|
||||
@@ -1,27 +1,32 @@
|
||||
from goojia_utils import REDIS_CLIENT
|
||||
import json
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
"""
|
||||
高德key每日定时生成
|
||||
"""
|
||||
#amap_key = {"4e9592208336433ee1b7aa4d1d434418", "10bcb421d2ee4e51ed9268d6dca58ad1"}
|
||||
|
||||
log.info("测试生成key")
|
||||
|
||||
#老板新注册key,限制15万
|
||||
amap_key = {"6e7b0b0539ff51a714678cd69183fe32"}
|
||||
|
||||
service = {'v3/geocode/geo': 150000,
|
||||
'v3/geocode/regeo': 150000,
|
||||
'v3/place/text': 150000,
|
||||
'v5/place/text': 150000,
|
||||
'v3/place/around': 150000,
|
||||
'v5/place/around': 150000,
|
||||
'v3/place/detail': 150000,
|
||||
'v5/place/detail': 150000}
|
||||
|
||||
for service_key in service.keys():
|
||||
val = dict()
|
||||
for key in amap_key:
|
||||
val[key] = service[service_key]
|
||||
|
||||
REDIS_CLIENT.hset("amapKey", service_key, json.dumps(val))
|
||||
# amap_key = {"6e7b0b0539ff51a714678cd69183fe32"}
|
||||
#
|
||||
# service = {'v3/geocode/geo': 150000,
|
||||
# 'v3/geocode/regeo': 150000,
|
||||
# 'v3/place/text': 150000,
|
||||
# 'v5/place/text': 150000,
|
||||
# 'v3/place/around': 150000,
|
||||
# 'v5/place/around': 150000,
|
||||
# 'v3/place/detail': 150000,
|
||||
# 'v5/place/detail': 150000}
|
||||
#
|
||||
# for service_key in service.keys():
|
||||
# val = dict()
|
||||
# for key in amap_key:
|
||||
# val[key] = service[service_key]
|
||||
#
|
||||
# REDIS_CLIENT.hset("amapKey", service_key, json.dumps(val))
|
||||
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
新房变二手房定时脚本
|
||||
创建年代、入住/交付时间三年后,或者有样本就变成二手房
|
||||
"""
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
|
||||
from pymongo import UpdateOne
|
||||
import pandas as pd
|
||||
import logging
|
||||
import datetime
|
||||
import time
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do():
|
||||
log.info("开始执行新房变二手房脚本")
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'is_new': 1},
|
||||
{'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'open_date': 1, 'live_date': 1,
|
||||
'create_year': 1}).batch_size(100000)))
|
||||
if len(df) <= 0:
|
||||
log.info("执行新房变二手房脚本完毕,未发现新房")
|
||||
return False
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
def deal(x):
|
||||
global count
|
||||
is_old = False
|
||||
time_now = int(time.mktime(time.strptime(datetime.datetime.now().strftime('%Y-%m-%d'), '%Y-%m-%d')))
|
||||
if 'open_date' in x and pd.notna(x['open_date']):
|
||||
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(x['open_date'], years=3), '%Y-%m-%d'), '%Y-%m-%d')))
|
||||
if time_old < time_now:
|
||||
is_old = True
|
||||
if is_old is False and 'live_date' in x and pd.notna(x['live_date']):
|
||||
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['live_date']), years=3), '%Y-%m-%d'), '%Y-%m-%d')))
|
||||
if time_old < time_now:
|
||||
is_old = True
|
||||
if is_old is False and 'create_year' in x and pd.notna(x['create_year']):
|
||||
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['create_year'], '%Y-%m'), years=3), '%Y-%m-%d'), '%Y-%m-%d')))
|
||||
if time_old < time_now:
|
||||
is_old = True
|
||||
# 查询样本库
|
||||
names = [x['name']]
|
||||
if type(x['alias']) == list:
|
||||
for alias_item in x['alias']:
|
||||
names.append(alias_item)
|
||||
names_str = "','".join(names)
|
||||
names_str = "'" + names_str + "'"
|
||||
if is_old is False:
|
||||
nums = commutils.get_sample(x['city'], x['region'], names_str)
|
||||
if nums > 0:
|
||||
is_old = True
|
||||
if is_old:
|
||||
actions.append(UpdateOne({'_id': x['_id']}, {'$set': {'is_new': 0, 'm_date': datetime.datetime.now()}}))
|
||||
if len(actions) == 5000:
|
||||
seaweed_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
|
||||
df.apply(lambda x: deal(x), axis=1)
|
||||
if len(actions) > 0:
|
||||
seaweed_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info(f"执行新房变二手房脚本完毕,共更新{count}")
|
||||
|
||||
|
||||
do()
|
||||
@@ -3,45 +3,70 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
import numpy as np
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
from threading import Lock
|
||||
|
||||
mutex = Lock()
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
m_date = dateutils.iso_format()
|
||||
path = "/home/airflow/data/community"
|
||||
file_path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||
|
||||
avg_cost_min_max_df = pd.read_excel(file_path + '/全国各区域人均消费中位值.xlsx')
|
||||
avg_cost_city_region_dict = dict()
|
||||
|
||||
|
||||
def find_around_price(lng, lat):
|
||||
# 加载各区域人均消费最大最小值
|
||||
def deal_avg_cost(x):
|
||||
city_now = x['city']
|
||||
region_now = x['region']
|
||||
if pd.notna(city_now) and pd.notna(region_now):
|
||||
avg_cost_city_region_dict[city_now + region_now] = {'min': x['3'], 'max': x['97']}
|
||||
elif pd.notna(city_now):
|
||||
avg_cost_city_region_dict[city_now] = {'min': x['3'], 'max': x['97']}
|
||||
|
||||
|
||||
avg_cost_min_max_df.apply(lambda x: deal_avg_cost(x), axis=1)
|
||||
|
||||
|
||||
def find_around_price(seaweed):
|
||||
"""
|
||||
查找周边均价
|
||||
返回取值为10~300,为所有有价格的1-99百分位
|
||||
"""
|
||||
if pd.isna(lng):
|
||||
return None
|
||||
lng = commutils.format_lat_lng(seaweed['lng'])
|
||||
lat = commutils.format_lat_lng(seaweed['lat'])
|
||||
city_short = seaweed['city']
|
||||
region_short = seaweed['region']
|
||||
city_display = areautils.get_city_display(city_short)
|
||||
|
||||
filter_query = [{"terms": {"tag": ["美食;中餐厅", "美食;外国餐厅"]}}]
|
||||
if pd.notna(city_display):
|
||||
filter_query.append({"term": {"city": {"value": city_display}}})
|
||||
|
||||
min_now = 10
|
||||
max_now = 300
|
||||
if (city_short+region_short) in avg_cost_city_region_dict:
|
||||
min_now = avg_cost_city_region_dict[city_short + region_short]['min']
|
||||
max_now = avg_cost_city_region_dict[city_short + region_short]['max']
|
||||
elif city_short in avg_cost_city_region_dict:
|
||||
min_now = avg_cost_city_region_dict[city_short]['min']
|
||||
max_now = avg_cost_city_region_dict[city_short]['max']
|
||||
|
||||
filter_query.append({"range": {"price": {"gte": min_now, "lte": max_now}}})
|
||||
filter_query.append({"geo_distance": {"distance": 1000, "location": [lng, lat]}})
|
||||
|
||||
ret_es = es206_client.elastic_client.search({
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"term": {
|
||||
"status": {
|
||||
"value": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"filter": [
|
||||
{
|
||||
"geo_distance": {
|
||||
"distance": 1000,
|
||||
"location": [lng, lat]
|
||||
}
|
||||
}
|
||||
]
|
||||
"filter": filter_query
|
||||
}
|
||||
},
|
||||
"size": 0,
|
||||
@@ -58,69 +83,51 @@ def find_around_price(lng, lat):
|
||||
|
||||
def build_doc(item):
|
||||
doc_dict = dict()
|
||||
if 'avg_cost' in item and pd.notna(item['avg_cost']):
|
||||
doc_dict['avg_cost'] = item['avg_cost']
|
||||
if 'avg_cost' in item:
|
||||
if pd.notna(item['avg_cost']) or item['avg_cost'] > 0:
|
||||
doc_dict['avg_cost'] = int(item['avg_cost'])
|
||||
else:
|
||||
doc_dict['avg_cost'] = None
|
||||
return doc_dict
|
||||
|
||||
|
||||
def update_status(new_df):
|
||||
actions = list()
|
||||
find_count = 0
|
||||
|
||||
def deal(item):
|
||||
action_doc = {
|
||||
"_op_type": "update",
|
||||
"_index": "bd_poi",
|
||||
"_id": item['_id'],
|
||||
"doc": {'status': 0}
|
||||
}
|
||||
actions.append(action_doc)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
return item
|
||||
|
||||
new_df.apply(lambda x: deal(x), axis=1)
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
def find_around_price_item(seaweed):
|
||||
global find_count
|
||||
find_count += 1
|
||||
if find_count % 1000 == 0:
|
||||
log.info("已查找" + str(find_count))
|
||||
if 'lng' not in seaweed:
|
||||
return
|
||||
price = find_around_price(seaweed)
|
||||
mutex.acquire(timeout=60)
|
||||
if pd.notna(price):
|
||||
seaweed['avg_cost'] = int(price)
|
||||
else:
|
||||
seaweed['avg_cost'] = None
|
||||
mutex.release()
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
log.info("开始同步消费数据...")
|
||||
query_sql = "select id, city, tag, type, price from bd_poi where price > 0 and tag in('美食;中餐厅', '美食;外国餐厅') "
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
query_sql += " and city='" + areautils.get_city_display(city) + "'"
|
||||
df = es206_client.scan_by_sql_to_data_frame(query_sql)
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
# 按百分位5-95筛选数据
|
||||
cities = df['city'].unique()
|
||||
for city in cities:
|
||||
df.loc[df['city'] == city, 'minPrice5'] = np.percentile(a=df[df['city'] == city]['price'], q=5, axis=0)
|
||||
df.loc[df['city'] == city, 'minPrice95'] = np.percentile(a=df[df['city'] == city]['price'], q=95, axis=0)
|
||||
seaweed_list = list()
|
||||
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
|
||||
|
||||
new_df = df[(df['price'] >= df['minPrice5']) & (df['price'] <= df['minPrice95'])]
|
||||
new_df['new_mean'] = new_df.groupby(by=['city'])['price'].transform(np.mean)
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
for seaweed in seaweed_list:
|
||||
pool.submit(find_around_price_item, seaweed)
|
||||
|
||||
# 按最新的百分位更新库中价格状态
|
||||
update_status(new_df)
|
||||
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
|
||||
def find_around_price_item(index, item):
|
||||
if 'lng' not in item:
|
||||
return
|
||||
lng = commutils.format_lat_lng(item['lng'])
|
||||
lat = commutils.format_lat_lng(item['lat'])
|
||||
price = find_around_price(lng, lat)
|
||||
seaweed_df.loc[index, 'avg_cost'] = price
|
||||
|
||||
with ThreadPoolExecutor(max_workers=50) as pool:
|
||||
for index, item in seaweed_df.iterrows():
|
||||
pool.submit(find_around_price_item, index, item)
|
||||
# for seaweed in seaweed_list:
|
||||
# find_around_price_item(seaweed)
|
||||
|
||||
seaweed_df = pd.DataFrame(seaweed_list)
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in seaweed_df.iterrows():
|
||||
@@ -147,4 +154,5 @@ def sync(city=None):
|
||||
log.info("完成同步消费数据...")
|
||||
|
||||
|
||||
sync()
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
|
||||
@@ -3,13 +3,15 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database="tmp")
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -88,10 +90,10 @@ def deal(actions, item):
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步商圈至206 {index_name}")
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
@@ -104,4 +106,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步商圈至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
@@ -1,16 +1,17 @@
|
||||
"""
|
||||
同步小区基本信息至ES206
|
||||
"""
|
||||
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
import os
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
m_date = dateutils.iso_format()
|
||||
@@ -47,6 +48,8 @@ def build_doc(item):
|
||||
estate_charge = commutils.format_property_fee(item['estate_charge'])
|
||||
if pd.notna(estate_charge):
|
||||
doc_dict['estate_charge'] = float(estate_charge)
|
||||
# 表示该物业费已经从库中补充
|
||||
doc_dict['estate_charge_flag'] = 0
|
||||
|
||||
finish_building_date = commutils.format_finish_building_date(item['finish_building_date'])
|
||||
if pd.notna(finish_building_date):
|
||||
@@ -83,8 +86,11 @@ def deal_comm(community_summary_df, seaweed_df):
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步小区基本属性至206 {index_name}...")
|
||||
seaweed_query = {'status': 0, 'cat': 'district'}
|
||||
ck_sql = "ALTER TABLE tmp.community_summary DELETE WHERE 1=1 "
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
ck_sql += " AND city='" + city + "'"
|
||||
tmp_ck.execute(ck_sql)
|
||||
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1,
|
||||
@@ -93,8 +99,20 @@ def sync(city=None):
|
||||
'estate_charge': 1, 'finish_building_date': 1})
|
||||
.batch_size(100000)))
|
||||
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}")
|
||||
sql = f"select id from {index_name} "
|
||||
|
||||
if pd.notna(city):
|
||||
sql += f" where city='{city}'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
||||
# 清除别名中逗号
|
||||
def dealAlias(x):
|
||||
if isinstance(x, list) and len(x) > 0:
|
||||
for index, element in enumerate(x):
|
||||
if "'" in element:
|
||||
x[index] = str(element).replace("'", "")
|
||||
return x
|
||||
seaweed_df['alias'] = seaweed_df['alias'].apply(lambda x: dealAlias(x))
|
||||
|
||||
if len(community_summary_df) > 0:
|
||||
community_summary_df['flag'] = 1
|
||||
@@ -105,6 +123,11 @@ def sync(city=None):
|
||||
seaweed_existed_df = pd.DataFrame()
|
||||
seaweed_not_existed_df = seaweed_df
|
||||
|
||||
# 写入csv文件,防止小区变动导致的后续过程更新报错,
|
||||
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||
# 写入中间表clickhouse,解决多worker节点问题
|
||||
tmp_ck.to_sql_plus(seaweed_df, 'community_summary')
|
||||
|
||||
actions = list()
|
||||
# 对于存在的更新
|
||||
count = 0
|
||||
@@ -124,6 +147,11 @@ def sync(city=None):
|
||||
count += len(actions)
|
||||
log.info("更新:" + str(count))
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
log.info("更新:" + str(count))
|
||||
actions.clear()
|
||||
|
||||
# 对于不存在的插入
|
||||
count = 0
|
||||
@@ -145,13 +173,13 @@ def sync(city=None):
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
log.info("插入:" + str(count))
|
||||
actions.clear()
|
||||
|
||||
# 写入csv文件,防止小区变动导致的后续过程更新报错
|
||||
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||
|
||||
deal_comm(community_summary_df, seaweed_df)
|
||||
log.info(f"完成同步小区基本属性至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
|
||||
@@ -1,55 +1,71 @@
|
||||
"""
|
||||
月消费潜力、评级计算
|
||||
"""
|
||||
from goojia_utils import ElasticSearchClient
|
||||
from goojia_utils import ElasticSearchClient, dateutils, nextcloudutils
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import logging
|
||||
import os
|
||||
from elasticsearch import helpers
|
||||
import matplotlib.pyplot as plt
|
||||
import sys
|
||||
|
||||
from sklearn.mixture import GaussianMixture
|
||||
from sklearn.preprocessing import QuantileTransformer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
index_name = "community_summary"
|
||||
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||
save_path_prefix = "/home/airflow/data/community"
|
||||
|
||||
save_png_path = save_path_prefix + "/consumption_rating"
|
||||
if not os.path.exists(save_png_path):
|
||||
os.makedirs(save_png_path)
|
||||
|
||||
nextcloud_aim_file_path = '共享/财力评级/财力评级图'
|
||||
|
||||
reasonable_df = pd.read_excel(path + '/财力评级分布情况20221215.xlsx')
|
||||
reasonable_dict = reasonable_df.set_index('city')['是否合理'].to_dict()
|
||||
|
||||
factor_df = pd.read_excel(path + '/财力评级极值设置.xlsx')
|
||||
factor_dict = dict()
|
||||
for index, item in factor_df.iterrows():
|
||||
if pd.isna(item['min_total_price']):
|
||||
continue
|
||||
factor_dict[item['city']] = {'min_total_price': item['min_total_price'], 'max_total_price': item['max_total_price'], 'min_estate_charge': item['min_estate_charge'], 'max_estate_charge': item['max_estate_charge'], \
|
||||
'min_avg_cost': item['min_avg_cost'], 'max_avg_cost': item['max_avg_cost']}
|
||||
|
||||
def calc_consumption_potential(city):
|
||||
"""
|
||||
计算消费潜力
|
||||
"""
|
||||
sql = "select id, city, region, name, total_price, estate_charge, relative_price_ranking, avg_cost from community_summary"
|
||||
if pd.notna(city):
|
||||
sql += " where city='" + city + "'"
|
||||
sql = "select id, city, region, name, total_price, estate_charge, estate_charge_fix, relative_price_ranking, avg_cost from community_summary where city='" + city + "'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
# 户均总价计算时候改成万
|
||||
### 极值设置
|
||||
if city in factor_dict:
|
||||
community_summary_df.loc[community_summary_df['total_price'] < factor_dict[city]['min_total_price'], 'total_price'] = factor_dict[city]['min_total_price']
|
||||
community_summary_df.loc[community_summary_df['total_price'] > factor_dict[city]['max_total_price'], 'total_price'] = factor_dict[city]['max_total_price']
|
||||
community_summary_df.loc[community_summary_df['estate_charge'] < factor_dict[city]['min_estate_charge'], 'estate_charge'] = factor_dict[city]['min_estate_charge']
|
||||
community_summary_df.loc[community_summary_df['estate_charge'] > factor_dict[city]['max_estate_charge'], 'estate_charge'] = factor_dict[city]['max_estate_charge']
|
||||
community_summary_df.loc[community_summary_df['avg_cost'] < factor_dict[city]['min_avg_cost'], 'avg_cost'] = factor_dict[city]['min_avg_cost']
|
||||
community_summary_df.loc[community_summary_df['avg_cost'] > factor_dict[city]['max_avg_cost'], 'avg_cost'] = factor_dict[city]['max_avg_cost']
|
||||
columns = community_summary_df.columns
|
||||
if 'total_price' not in columns or 'estate_charge' not in columns or 'relative_price_ranking' not in columns or 'avg_cost' not in columns:
|
||||
return []
|
||||
community_summary_df = community_summary_df.dropna(how='any').copy()
|
||||
if len(community_summary_df) == 0:
|
||||
log.warning(f"{city}无法评级,缺少因子")
|
||||
return []
|
||||
|
||||
community_summary_df['total_price'] = community_summary_df['total_price'] / 10000
|
||||
# 户均总价计算时候改成万
|
||||
community_summary_df['total_price'] = round(community_summary_df['total_price'] / 10000, 0)
|
||||
|
||||
community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform(
|
||||
np.mean)
|
||||
community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.std)
|
||||
|
||||
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(
|
||||
# 噪音词处理
|
||||
community_summary_df['avg_cost_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform(
|
||||
community_summary_df[['avg_cost']])
|
||||
|
||||
# 解决某些城市无修正物业费问题
|
||||
if 'estate_charge_fix' not in community_summary_df.columns:
|
||||
community_summary_df['estate_charge_fix'] = None
|
||||
# 修正的物业费入模
|
||||
community_summary_df['estate_charge_fix'].fillna(community_summary_df['estate_charge'], inplace=True)
|
||||
community_summary_df['estate_charge_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform(
|
||||
community_summary_df[['estate_charge_fix']])
|
||||
|
||||
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge_fixed'].transform(
|
||||
np.mean)
|
||||
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(
|
||||
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge_fixed'].transform(
|
||||
np.std)
|
||||
|
||||
community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])[
|
||||
@@ -57,22 +73,29 @@ def calc_consumption_potential(city):
|
||||
community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])[
|
||||
'relative_price_ranking'].transform(np.std)
|
||||
|
||||
community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.mean)
|
||||
community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.std)
|
||||
community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost_fixed'].transform(np.mean)
|
||||
community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost_fixed'].transform(np.std)
|
||||
|
||||
community_summary_df['total_price_score'] = 286.0134196 * (
|
||||
community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df[
|
||||
'total_price_std']
|
||||
# 和田等小城市均值可能为0
|
||||
if community_summary_df['total_price_std'].unique()[0] != 0:
|
||||
community_summary_df['total_price_score'] = 286.0134196 * (community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df['total_price_std']
|
||||
else:
|
||||
community_summary_df['total_price_score'] = 0
|
||||
|
||||
community_summary_df['estate_charge_score'] = 139.1374524 * (
|
||||
community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std']
|
||||
if community_summary_df['estate_charge_std'].unique()[0] != 0:
|
||||
community_summary_df['estate_charge_score'] = 139.1374524 * (community_summary_df['estate_charge_fixed'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std']
|
||||
else:
|
||||
community_summary_df['estate_charge_score'] = 0
|
||||
|
||||
community_summary_df['relative_price_ranking_score'] = 21.8427926 * (
|
||||
community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / community_summary_df['relative_price_ranking_std']
|
||||
if community_summary_df['relative_price_ranking_std'].unique()[0] != 0:
|
||||
community_summary_df['relative_price_ranking_score'] = 21.8427926 * (community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / community_summary_df['relative_price_ranking_std']
|
||||
else:
|
||||
community_summary_df['relative_price_ranking_score'] = 0
|
||||
|
||||
community_summary_df['avg_cost_score'] = 67.0195871700 * (
|
||||
community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df[
|
||||
'avg_cost_std']
|
||||
if community_summary_df['avg_cost_std'].unique()[0] != 0:
|
||||
community_summary_df['avg_cost_score'] = 67.0195871700 * (community_summary_df['avg_cost_fixed'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std']
|
||||
else:
|
||||
community_summary_df['avg_cost_score'] = 0
|
||||
|
||||
community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df[
|
||||
'estate_charge_score'] + community_summary_df['relative_price_ranking_score'] + community_summary_df['avg_cost_score'] + community_summary_df['avg_cost_score'] + 4099.978248
|
||||
@@ -81,10 +104,10 @@ def calc_consumption_potential(city):
|
||||
|
||||
def gmm_train(total_rate, consumption_df, source, target):
|
||||
# 高斯混合模型
|
||||
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
|
||||
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense').astype(int)
|
||||
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
||||
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||
return consumption_df
|
||||
|
||||
@@ -93,47 +116,153 @@ def calc_consumption_rating(community_summary_df):
|
||||
"""
|
||||
计算消费等级
|
||||
"""
|
||||
consumption_aim_df = community_summary_df[['_id', 'city', 'consumption_potential']]
|
||||
consumption_aim_df = consumption_aim_df.dropna(subset=['consumption_potential'])
|
||||
cities = consumption_aim_df['city'].unique()
|
||||
source = "consumption_potential"
|
||||
df = community_summary_df[['_id', 'city', 'consumption_potential']]
|
||||
df = df.dropna(subset=['consumption_potential'])
|
||||
source = "normal_consumption_potential"
|
||||
target = "consumption_rating"
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
for city in cities:
|
||||
df = consumption_aim_df[consumption_aim_df['city'] == city].copy()
|
||||
if len(df) < 10:
|
||||
continue
|
||||
df = gmm_train(10, df, source, target)
|
||||
# 批量更新
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
row['consumption_rating'] = item['consumption_rating']
|
||||
row['consumption_potential'] = item['consumption_potential']
|
||||
if len(df) < 5:
|
||||
# 对于样本低于5的直接排名
|
||||
df[target] = df['consumption_potential'].rank(method='dense', ascending=False).astype(int)
|
||||
else:
|
||||
# 极值设置
|
||||
df = filter_min_max(df)
|
||||
df = gmm_train(5, df, source, target)
|
||||
# 批量更新
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
row['consumption_rating'] = item['consumption_rating']
|
||||
row['consumption_potential'] = item['consumption_potential']
|
||||
row['m_date'] = dateutils.iso_format()
|
||||
|
||||
action = {'_op_type': 'update', # 操作 index update create delete
|
||||
'_index': index_name, # index
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': row,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级中: 当前已评级{}".format(count))
|
||||
action = {'_op_type': 'update', # 操作 index update create delete
|
||||
'_index': index_name, # index
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': row,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级中: 当前已评级{}".format(count))
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级完成,共更新: {}".format(count))
|
||||
return pd.merge(community_summary_df, df, how='left', on=['_id', 'city', 'consumption_potential'])
|
||||
|
||||
|
||||
def sync_by_city(city_now):
|
||||
log.info(f"{city_now}开始评级....")
|
||||
community_summary_df = calc_consumption_potential(city_now)
|
||||
if len(community_summary_df) == 0:
|
||||
log.warning(f"{city_now}无法评级,缺少因子")
|
||||
return
|
||||
community_summary_df = calc_consumption_rating(community_summary_df)
|
||||
if len(community_summary_df) == 0:
|
||||
log.warning(f"{city_now}无法评级,数量低于5")
|
||||
return
|
||||
save_png(community_summary_df)
|
||||
nextcloudutils.up_to_nextcloud(save_png_path + "/" + city_now + ".png", nextcloud_aim_file_path + "/" + city_now + ".png")
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
community_summary_df = calc_consumption_potential(city)
|
||||
calc_consumption_rating(community_summary_df)
|
||||
if pd.isna(city):
|
||||
for city_now in reasonable_df['city'].unique():
|
||||
#if pd.isna(reasonable_dict[city_now]):
|
||||
sync_by_city(city_now)
|
||||
else:
|
||||
sync_by_city(city)
|
||||
|
||||
sync()
|
||||
|
||||
def filter_min_max(community_summary_df):
|
||||
community_summary_df['normal_consumption_potential'] = QuantileTransformer(output_distribution="normal").fit_transform(community_summary_df[['consumption_potential']])
|
||||
# total_price5 = np.percentile(a=community_summary_df['total_price'], q=2, axis=0)
|
||||
# total_price95 = np.percentile(a=community_summary_df['total_price'], q=98, axis=0)
|
||||
# community_summary_df.loc[community_summary_df['total_price'] > total_price95, 'total_price'] = total_price95
|
||||
# community_summary_df.loc[community_summary_df['total_price'] < total_price5, 'total_price'] = total_price5
|
||||
#
|
||||
# estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=2, axis=0)
|
||||
# estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=98, axis=0)
|
||||
# community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95
|
||||
# community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5
|
||||
#
|
||||
# avg_cost5 = np.percentile(a=community_summary_df['avg_cost'], q=2, axis=0)
|
||||
# avg_cost95 = np.percentile(a=community_summary_df['avg_cost'], q=98, axis=0)
|
||||
# community_summary_df.loc[community_summary_df['avg_cost'] > avg_cost95, 'avg_cost'] = avg_cost95
|
||||
# community_summary_df.loc[community_summary_df['avg_cost'] < avg_cost5, 'avg_cost'] = avg_cost5
|
||||
|
||||
#if city in factor_dict:
|
||||
# 户均总价按极值设置可靠性不高,当户均总价补充后,图中大于最大值的量非常多
|
||||
# community_summary_df.loc[
|
||||
# community_summary_df['total_price'] < factor_dict[city]['min_total_price'], 'total_price'] = \
|
||||
# factor_dict[city]['min_total_price']
|
||||
# community_summary_df.loc[
|
||||
# community_summary_df['total_price'] > factor_dict[city]['max_total_price'], 'total_price'] = \
|
||||
# factor_dict[city]['max_total_price']
|
||||
|
||||
# community_summary_df.loc[
|
||||
# community_summary_df['estate_charge'] < factor_dict[city]['min_estate_charge'], 'estate_charge'] = \
|
||||
# factor_dict[city]['min_estate_charge']
|
||||
# community_summary_df.loc[
|
||||
# community_summary_df['estate_charge'] > factor_dict[city]['max_estate_charge'], 'estate_charge'] = \
|
||||
# factor_dict[city]['max_estate_charge']
|
||||
# community_summary_df.loc[community_summary_df['avg_cost'] < factor_dict[city]['min_avg_cost'], 'avg_cost'] = \
|
||||
# factor_dict[city]['min_avg_cost']
|
||||
# community_summary_df.loc[community_summary_df['avg_cost'] > factor_dict[city]['max_avg_cost'], 'avg_cost'] = \
|
||||
# factor_dict[city]['max_avg_cost']
|
||||
return community_summary_df
|
||||
|
||||
|
||||
def save_png(community_city):
|
||||
"""
|
||||
生成财力评级图表
|
||||
"""
|
||||
curr_city = community_city['city'].unique()[0]
|
||||
# 解决中文显示问题
|
||||
plt.rcParams['font.sans-serif'] = ['SimHei']
|
||||
plt.rcParams['axes.unicode_minus'] = False
|
||||
plt.title(curr_city)
|
||||
plt.subplot(3, 1, 1)
|
||||
community_city['consumption_rating'].hist(bins=100)
|
||||
plt.xlabel('consumption_rating')
|
||||
plt.tight_layout()
|
||||
|
||||
plt.subplot(3, 2, 3)
|
||||
community_city['total_price'].hist(bins=100)
|
||||
plt.xlabel('total_price')
|
||||
plt.tight_layout()
|
||||
|
||||
plt.subplot(3, 2, 4)
|
||||
community_city['estate_charge'].hist(bins=100)
|
||||
plt.xlabel('estate_charge')
|
||||
plt.tight_layout()
|
||||
|
||||
plt.subplot(3, 2, 5)
|
||||
community_city['avg_cost'].hist(bins=100)
|
||||
plt.xlabel('avg_cost')
|
||||
plt.tight_layout()
|
||||
|
||||
plt.subplot(3, 2, 6)
|
||||
community_city['relative_price_ranking'].hist(bins=100)
|
||||
plt.xlabel('relative_price_ranking')
|
||||
plt.tight_layout()
|
||||
|
||||
plt.savefig(save_png_path + "/" + curr_city + ".png")
|
||||
# 清图
|
||||
plt.clf()
|
||||
# 清坐标轴
|
||||
plt.cla()
|
||||
# 关窗口
|
||||
plt.close()
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
#dingutils.send_text("【分布图生成完毕】财力评级已完成, 请点击 https://nextcloud.fangjia.com/s/Jqs5BzXi8xLYeH9 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -3,13 +3,15 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database="tmp")
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -62,11 +64,10 @@ def deal(actions, item):
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}")
|
||||
seaweed_query = {'status': 0, 'cat': 'district'}
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
@@ -83,4 +84,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步教育资源至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
@@ -3,13 +3,15 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database="tmp")
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -75,10 +77,10 @@ def deal(actions, item):
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步教育资源【小学、中学】至206 {index_name}")
|
||||
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
@@ -95,4 +97,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步教育资源至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
@@ -0,0 +1,317 @@
|
||||
"""
|
||||
物业费补全
|
||||
"""
|
||||
from goojia_utils import ElasticSearchClient, areautils, dateutils
|
||||
import pandas as pd
|
||||
import logging
|
||||
import sys
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
index_name = "community_summary"
|
||||
file_path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||
mutex = Lock()
|
||||
estate_charge_min_max_df = pd.read_excel(file_path + '/全国各区域物业费中位值.xlsx')
|
||||
estate_charge_city_region_dict = dict()
|
||||
|
||||
|
||||
# 加载各区域物业费最大最小值
|
||||
def deal_estate_charge(x):
|
||||
city_now = x['city']
|
||||
region_now = x['region']
|
||||
if pd.notna(city_now) and pd.notna(region_now):
|
||||
estate_charge_city_region_dict[city_now + region_now] = {'min': x['3'], 'max': x['97']}
|
||||
elif pd.notna(city_now):
|
||||
estate_charge_city_region_dict[city_now] = {'min': x['3'], 'max': x['97']}
|
||||
estate_charge_min_max_df.apply(lambda x: deal_estate_charge(x), axis=1)
|
||||
|
||||
|
||||
def save_fix(community_summary_df):
|
||||
if 'estate_charge_fix' not in community_summary_df.columns:
|
||||
log.info("该城市无调整物业费")
|
||||
return
|
||||
df = community_summary_df[['_id', 'city', 'estate_charge_fix']]
|
||||
df = df.dropna(subset=['estate_charge_fix'])
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
# 批量更新
|
||||
m_date = dateutils.iso_format()
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
row['estate_charge_fix'] = item['estate_charge_fix']
|
||||
row['m_date'] = m_date
|
||||
|
||||
action = {'_op_type': 'update',
|
||||
'_index': index_name,
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': row,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("已更新{}".format(count))
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("更新完成,共更新: {}".format(count))
|
||||
return df
|
||||
|
||||
|
||||
|
||||
def save_fill(community_summary_df):
|
||||
if 'estate_charge' not in community_summary_df.columns:
|
||||
log.info("该城市无补充物业费")
|
||||
return
|
||||
df = community_summary_df[['_id', 'city', 'estate_charge']]
|
||||
df = df.dropna(subset=['estate_charge'])
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
# 批量更新
|
||||
m_date = dateutils.iso_format()
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
row['estate_charge'] = item['estate_charge']
|
||||
# 1 表示为物业费补充的,在实际应用中不输出
|
||||
row['estate_charge_flag'] = 1
|
||||
row['m_date'] = m_date
|
||||
|
||||
action = {'_op_type': 'update',
|
||||
'_index': index_name,
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': row,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("已更新{}".format(count))
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("更新完成,共更新: {}".format(count))
|
||||
return df
|
||||
|
||||
|
||||
def find_nearest_by_location(city, location):
|
||||
docs = list()
|
||||
try:
|
||||
scan = es206_client.scan(index_name, {
|
||||
"size": 4,
|
||||
"_source": ["finish_building_date", "estate_charge", "estate_charge_fix"],
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{
|
||||
"term": {
|
||||
"city": city
|
||||
}
|
||||
}
|
||||
],
|
||||
"must_not": [
|
||||
{
|
||||
"term": {
|
||||
"estate_charge_flag": {
|
||||
"value": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"sort": [
|
||||
{
|
||||
"_geo_distance": {
|
||||
"location": location,
|
||||
"order": "asc"
|
||||
}
|
||||
}
|
||||
]
|
||||
}, preserve_order=True)
|
||||
ignored_first = False
|
||||
for doc in scan:
|
||||
# 忽略自己
|
||||
if ignored_first is False:
|
||||
ignored_first = True
|
||||
continue
|
||||
docs.append(doc['_source'])
|
||||
if len(docs) == 3:
|
||||
return pd.DataFrame(docs)
|
||||
return None
|
||||
except Exception as e:
|
||||
logging.error(e)
|
||||
return None
|
||||
|
||||
|
||||
def get_estate_charge_for_adjust(finish_building_date, docs):
|
||||
aim_year = int(finish_building_date / 10)
|
||||
for index, item in docs.iterrows():
|
||||
if pd.notna(item['finish_building_date']) and pd.notna(item['estate_charge']) and aim_year == int(item['finish_building_date'] / 10):
|
||||
return item['estate_charge']
|
||||
return None
|
||||
|
||||
|
||||
def get_estate_charge(finish_building_date, docs):
|
||||
aim_year = int(finish_building_date / 10)
|
||||
for index, item in docs.iterrows():
|
||||
if pd.notna(item['finish_building_date']) and pd.notna(item['estate_charge_fix']) and aim_year == int(item['finish_building_date'] / 10):
|
||||
return item['estate_charge_fix']
|
||||
return None
|
||||
|
||||
|
||||
find_count = 0
|
||||
def deal_adjust(community):
|
||||
global find_count
|
||||
find_count += 1
|
||||
if find_count % 1000 == 0:
|
||||
log.info("已查找" + str(find_count))
|
||||
max_now = 7
|
||||
city_short = community['city']
|
||||
region_short = community['region']
|
||||
if (city_short + region_short) in estate_charge_city_region_dict:
|
||||
max_now = estate_charge_city_region_dict[city_short + region_short]['max']
|
||||
elif city_short in estate_charge_city_region_dict:
|
||||
max_now = estate_charge_city_region_dict[city_short]['max']
|
||||
|
||||
if community['estate_charge'] > max_now:
|
||||
estate_charge = find_estate_charge_for_adjust(community, max_now)
|
||||
if pd.notna(estate_charge):
|
||||
mutex.acquire(timeout=60)
|
||||
community['estate_charge_fix'] = estate_charge
|
||||
mutex.release()
|
||||
|
||||
def adjust_estate_charge(city_now):
|
||||
log.info(f"{city_now}开始调整物业费....")
|
||||
sql = f"select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='{city_now}' and estate_charge_flag != 1 and estate_type != '别墅' "
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
community_summary_list = list()
|
||||
community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
for community_summary in community_summary_list:
|
||||
pool.submit(deal_adjust, community_summary)
|
||||
# for community_summary in community_summary_list:
|
||||
# deal_adjust(community_summary)
|
||||
community_summary_df = pd.DataFrame(community_summary_list)
|
||||
save_fix(community_summary_df)
|
||||
|
||||
log.info(f"{city_now}完成调整物业费....")
|
||||
|
||||
|
||||
def find_estate_charge_for_adjust(community, max_now):
|
||||
"""
|
||||
查找小区周边物业费
|
||||
"""
|
||||
estate_charge = None
|
||||
if pd.isna(community['location']) or pd.isna(community['location']['lon']):
|
||||
return estate_charge
|
||||
location = [community['location']['lon'], community['location']['lat']]
|
||||
docs = find_nearest_by_location(community['city'], location)
|
||||
if len(docs) <= 0 or 'estate_charge' not in docs.columns:
|
||||
return estate_charge
|
||||
if pd.notna(community['finish_building_date']) and 'finish_building_date' in docs.columns:
|
||||
# 根据相近建筑年代赋值
|
||||
estate_charge = get_estate_charge_for_adjust(community['finish_building_date'], docs)
|
||||
if pd.isna(estate_charge) or estate_charge > max_now:
|
||||
# 取周边均值
|
||||
estate_charge = docs['estate_charge'].mean()
|
||||
# 如果查找的还大于分位值,不调整
|
||||
if estate_charge > max_now:
|
||||
estate_charge = None
|
||||
return estate_charge
|
||||
|
||||
def find_estate_charge(community, max_now):
|
||||
"""
|
||||
查找小区周边物业费
|
||||
"""
|
||||
estate_charge = None
|
||||
if pd.isna(community['location']) or pd.isna(community['location']['lon']):
|
||||
return estate_charge
|
||||
location = [community['location']['lon'], community['location']['lat']]
|
||||
docs = find_nearest_by_location(community['city'], location)
|
||||
if len(docs) <= 0 or 'estate_charge' not in docs.columns:
|
||||
return estate_charge
|
||||
if 'estate_charge_fix' not in docs.columns:
|
||||
docs['estate_charge_fix'] = None
|
||||
# 用修正后的物业费补充
|
||||
docs['estate_charge_fix'].fillna(docs['estate_charge'], inplace=True)
|
||||
if pd.notna(community['finish_building_date']) and 'finish_building_date' in docs.columns:
|
||||
# 根据相近建筑年代赋值
|
||||
estate_charge = get_estate_charge(community['finish_building_date'], docs)
|
||||
if pd.isna(estate_charge) or estate_charge > max_now:
|
||||
# 取周边均值
|
||||
estate_charge = docs['estate_charge_fix'].mean()
|
||||
if estate_charge > max_now:
|
||||
estate_charge = None
|
||||
return estate_charge
|
||||
|
||||
find_count = 0
|
||||
def deal_fill(community):
|
||||
global find_count
|
||||
find_count += 1
|
||||
if find_count % 1000 == 0:
|
||||
log.info("已查找" + str(find_count))
|
||||
|
||||
max_now = 7
|
||||
city_short = community['city']
|
||||
region_short = community['region']
|
||||
if (city_short + region_short) in estate_charge_city_region_dict:
|
||||
max_now = estate_charge_city_region_dict[city_short + region_short]['max']
|
||||
elif city_short in estate_charge_city_region_dict:
|
||||
max_now = estate_charge_city_region_dict[city_short]['max']
|
||||
if community['estate_type'] == '别墅':
|
||||
# 别墅不限制物业费最大值
|
||||
max_now = 99
|
||||
estate_charge = find_estate_charge(community, max_now)
|
||||
if pd.notna(estate_charge):
|
||||
mutex.acquire(timeout=60)
|
||||
community["estate_charge"] = estate_charge
|
||||
mutex.release()
|
||||
|
||||
def fill_estate_charge(city_now):
|
||||
log.info(f"{city_now}开始补充物业费....")
|
||||
sql = "select id, city, region, name, estate_charge, estate_charge_fix, finish_building_date, estate_type, location from community_summary where city='" + city_now + "' and (estate_charge is null or estate_charge_flag=1)"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
community_summary_list = list()
|
||||
community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
for community_summary in community_summary_list:
|
||||
pool.submit(deal_fill, community_summary)
|
||||
# for community_summary in community_summary_list:
|
||||
# deal_fill(community_summary)
|
||||
community_summary_df = pd.DataFrame(community_summary_list)
|
||||
save_fill(community_summary_df)
|
||||
log.info(f"{city_now}完成补充物业费....")
|
||||
|
||||
|
||||
def sync_by_city(city_now):
|
||||
adjust_estate_charge(city_now)
|
||||
fill_estate_charge(city_now)
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
if pd.isna(city):
|
||||
for city_now in areautils.get_all_cities():
|
||||
sync_by_city(city_now)
|
||||
else:
|
||||
sync_by_city(city)
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -0,0 +1,169 @@
|
||||
"""
|
||||
物业费等级计算
|
||||
"""
|
||||
from goojia_utils import ElasticSearchClient, nextcloudutils
|
||||
import pandas as pd
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from elasticsearch import helpers
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
from sklearn.mixture import GaussianMixture
|
||||
from sklearn.preprocessing import QuantileTransformer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
index_name = "community_summary"
|
||||
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||
|
||||
nextcloud_aim_file_path = '共享/财力评级/物业费等级图'
|
||||
|
||||
save_path_prefix = "/home/airflow/data/community"
|
||||
save_png_path = save_path_prefix + "/estate_charge_ranking"
|
||||
if not os.path.exists(save_png_path):
|
||||
os.makedirs(save_png_path)
|
||||
|
||||
factor_df = pd.read_excel(path + '/财力评级极值设置.xlsx')
|
||||
factor_dict = dict()
|
||||
for index, item in factor_df.iterrows():
|
||||
if pd.isna(item['min_total_price']):
|
||||
continue
|
||||
factor_dict[item['city']] = {'min_total_price': item['min_total_price'], 'max_total_price': item['max_total_price'], 'min_estate_charge': item['min_estate_charge'], 'max_estate_charge': item['max_estate_charge'], \
|
||||
'min_avg_cost': item['min_avg_cost'], 'max_avg_cost': item['max_avg_cost']}
|
||||
|
||||
|
||||
reasonable_df = pd.read_excel(path + '/财力评级物业费等级情况20221213.xlsx')
|
||||
reasonable_dict = reasonable_df.set_index('city')['是否合理'].to_dict()
|
||||
|
||||
|
||||
def gmm_train(total_rate, consumption_df, source, target):
|
||||
# 高斯混合模型
|
||||
city_info = consumption_df.loc[0, 'city']
|
||||
if city_info in reasonable_dict and reasonable_dict[city_info] == '不合理':
|
||||
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]])
|
||||
else:
|
||||
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
|
||||
consumption_df['cluster'] = gmm.predict(consumption_df[[source]])
|
||||
consumption_df['price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||
consumption_df[target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
||||
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||
return consumption_df
|
||||
|
||||
|
||||
|
||||
def filter_min_max(community_summary_df):
|
||||
"""
|
||||
噪音词处理
|
||||
"""
|
||||
community_summary_df['QuantileTransformer.normal'] = QuantileTransformer(output_distribution="normal").fit_transform(community_summary_df[['estate_charge']])
|
||||
# estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=1, axis=0)
|
||||
# estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=99, axis=0)
|
||||
# community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95
|
||||
# community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5
|
||||
# if city in factor_dict:
|
||||
# community_summary_df.loc[
|
||||
# community_summary_df['estate_charge'] < factor_dict[city]['min_estate_charge'], 'estate_charge'] = \
|
||||
# factor_dict[city]['min_estate_charge']
|
||||
# community_summary_df.loc[
|
||||
# community_summary_df['estate_charge'] > factor_dict[city]['max_estate_charge'], 'estate_charge'] = \
|
||||
# factor_dict[city]['max_estate_charge']
|
||||
return community_summary_df
|
||||
|
||||
|
||||
def calc_estate_charge_rating(community_summary_df):
|
||||
"""
|
||||
计算物业费等级
|
||||
"""
|
||||
df = community_summary_df[['_id', 'city', 'QuantileTransformer.normal', 'estate_charge']].copy()
|
||||
source = "QuantileTransformer.normal"
|
||||
target = "estate_charge_ranking"
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
if len(df) < 5:
|
||||
# 对于样本低于5的直接排名
|
||||
df[target] = df['estate_charge'].rank(method='dense', ascending=False).astype(int)
|
||||
else:
|
||||
df = gmm_train(5, df, source, target)
|
||||
# 批量更新
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
row['estate_charge_ranking'] = item['estate_charge_ranking']
|
||||
|
||||
action = {'_op_type': 'update', # 操作 index update create delete
|
||||
'_index': index_name, # index
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': row,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级中: 当前已评级{}".format(count))
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级完成,共更新: {}".format(count))
|
||||
return df
|
||||
|
||||
|
||||
def sync_by_city(city_now):
|
||||
log.info(f"{city_now}开始评级....")
|
||||
sql = "select id, city, region, name, estate_charge from community_summary where city='" + city_now + "'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
columns = community_summary_df.columns
|
||||
if 'estate_charge' not in columns:
|
||||
log.warning(f"{city_now}无法评级,缺少因子")
|
||||
return
|
||||
community_summary_df = community_summary_df.dropna(how='any').reset_index(drop=True).copy()
|
||||
# 噪音词处理
|
||||
community_summary_df = filter_min_max(community_summary_df)
|
||||
community_summary_df = calc_estate_charge_rating(community_summary_df)
|
||||
if len(community_summary_df) == 0:
|
||||
log.warning(f"{city_now}无法评级,数量低于5")
|
||||
return
|
||||
save_png(community_summary_df)
|
||||
nextcloudutils.up_to_nextcloud(save_png_path + "/" + city_now + ".png", nextcloud_aim_file_path + "/" + city_now + ".png")
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
if pd.isna(city):
|
||||
for city_now in reasonable_df['city'].unique():
|
||||
#if reasonable_dict[city_now] == '不合理':
|
||||
sync_by_city(city_now)
|
||||
else:
|
||||
sync_by_city(city)
|
||||
|
||||
|
||||
def save_png(community_city):
|
||||
"""
|
||||
生成财力评级图表
|
||||
"""
|
||||
plt.subplot(2, 1, 1)
|
||||
community_city['estate_charge_ranking'].hist(bins=100)
|
||||
plt.xlabel('estate_charge_ranking')
|
||||
plt.tight_layout()
|
||||
|
||||
plt.subplot(2, 1, 2)
|
||||
community_city['estate_charge'].hist(bins=100)
|
||||
plt.xlabel('estate_charge')
|
||||
plt.tight_layout()
|
||||
|
||||
plt.savefig(save_png_path + "/" + community_city['city'].unique()[0] + ".png")
|
||||
# 清图
|
||||
plt.clf()
|
||||
# 清坐标轴
|
||||
plt.cla()
|
||||
# 关窗口
|
||||
plt.close()
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
#dingutils.send_text("【分布图生成完毕】物业费评级已完成, 请点击 https://nextcloud.fangjia.com/s/HbifLoEDc5wi8HR 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
|
||||
@@ -14,6 +14,7 @@ import pandas as pd
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
@@ -140,4 +141,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步附加标签至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -3,13 +3,15 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -76,10 +78,10 @@ def deal(actions, item):
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步生活配套至206 {index_name}")
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
@@ -96,4 +98,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步生活配套至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
@@ -3,14 +3,16 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient, poiutils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -62,11 +64,10 @@ def deal(actions, item):
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步医院至206 {index_name}")
|
||||
seaweed_query = {'status': 0, 'cat': 'district'}
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
@@ -83,4 +84,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步医院至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
@@ -5,7 +5,7 @@ from goojia_utils import ElasticSearchClient, dateutils
|
||||
import logging
|
||||
import pandas as pd
|
||||
from elasticsearch import helpers
|
||||
import datetime
|
||||
import sys
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
@@ -16,46 +16,9 @@ m_date = dateutils.iso_format()
|
||||
count = 0
|
||||
|
||||
|
||||
def get_old_or_new(finish_building_date):
|
||||
"""
|
||||
或许新旧
|
||||
"""
|
||||
if pd.isna(finish_building_date):
|
||||
return "旧"
|
||||
now_year = datetime.datetime.now().year
|
||||
if now_year - int(finish_building_date) < 3:
|
||||
return "新"
|
||||
return "旧"
|
||||
|
||||
|
||||
def deal(actions, x):
|
||||
global count
|
||||
estate_charge_ranking = None
|
||||
if pd.notna(x['estate_charge']):
|
||||
estate_charge = float(x['estate_charge'])
|
||||
if estate_charge >= 1:
|
||||
estate_charge_ranking = "一级"
|
||||
elif estate_charge >= 0.75:
|
||||
estate_charge_ranking = "二级"
|
||||
elif estate_charge >= 0.50:
|
||||
estate_charge_ranking = "三级"
|
||||
elif estate_charge >= 0.35:
|
||||
estate_charge_ranking = "四级"
|
||||
elif estate_charge < 0.35:
|
||||
estate_charge_ranking = "五级"
|
||||
else:
|
||||
old_or_new = get_old_or_new(x['finish_building_date'])
|
||||
if old_or_new == '新':
|
||||
if pd.notna(x['build_type']) and '高层' in x['build_type']:
|
||||
estate_charge_ranking = '三级'
|
||||
else:
|
||||
estate_charge_ranking = '四级'
|
||||
elif old_or_new == '旧':
|
||||
estate_charge_ranking = '五级'
|
||||
|
||||
doc_info = dict()
|
||||
if pd.notna(estate_charge_ranking):
|
||||
doc_info['estate_charge_ranking'] = estate_charge_ranking
|
||||
if pd.notna(x['avg_price']) and pd.notna(x['rent_price']):
|
||||
doc_info['sell_to_rent'] = int(x['avg_price'] / x['rent_price'])
|
||||
if pd.notna(x['household_count']):
|
||||
@@ -72,14 +35,14 @@ def deal(actions, x):
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
log.info(f"物业费等级、售租比、人口数已更新{count}")
|
||||
log.info(f"售租比、人口数已更新{count}")
|
||||
actions.clear()
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步物业费等级、售租比、人口数至206 {index_name}")
|
||||
log.info(f"开始同步售租比、人口数至206 {index_name}")
|
||||
|
||||
sql = "select city, region, name, estate_charge, finish_building_date, build_type, avg_price, rent_price, household_count from community_summary"
|
||||
sql = "select city, region, name, finish_building_date, build_type, avg_price, rent_price, household_count from community_summary"
|
||||
if pd.notna(city):
|
||||
sql += " where city='" + city + "'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
@@ -91,7 +54,8 @@ def sync(city=None):
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
|
||||
log.info(f"完成同步物业费等级、售租比、人口数至206 {index_name}")
|
||||
log.info(f"完成同步售租比、人口数至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -0,0 +1,79 @@
|
||||
"""
|
||||
人口评级
|
||||
"""
|
||||
from goojia_utils import ElasticSearchClient
|
||||
import pandas as pd
|
||||
import logging
|
||||
from elasticsearch import helpers
|
||||
import sys
|
||||
|
||||
from sklearn.mixture import GaussianMixture
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
index_name = "community_summary"
|
||||
|
||||
|
||||
def gmm_train(total_rate, consumption_df, source, target):
|
||||
# 高斯混合模型
|
||||
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
||||
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||
return consumption_df
|
||||
|
||||
|
||||
def calc_person_rating(city=None):
|
||||
"""
|
||||
计算消费等级
|
||||
"""
|
||||
sql = "select id, city, region, name, household_count from community_summary"
|
||||
if pd.notna(city):
|
||||
sql += " where city='" + city + "'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
consumption_aim_df = community_summary_df[['_id', 'city', 'household_count']]
|
||||
consumption_aim_df = consumption_aim_df.dropna(subset=['household_count'])
|
||||
cities = consumption_aim_df['city'].unique()
|
||||
source = "household_count"
|
||||
target = "person_rating"
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
for city in cities:
|
||||
df = consumption_aim_df[consumption_aim_df['city'] == city].copy()
|
||||
if len(df) < 5:
|
||||
continue
|
||||
df = gmm_train(5, df, source, target)
|
||||
|
||||
# 批量更新
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
row['person_rating'] = item['person_rating']
|
||||
|
||||
action = {'_op_type': 'update',
|
||||
'_index': index_name,
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': row}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级中: 当前已评级{}".format(count))
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级完成,共更新: {}".format(count))
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
calc_person_rating(city)
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -3,11 +3,13 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -16,8 +18,10 @@ m_date = dateutils.iso_format()
|
||||
|
||||
def build_doc(item):
|
||||
doc_dict = dict()
|
||||
if 'city_avg_price' in item and pd.notna(item['city_avg_price']):
|
||||
doc_dict['city_avg_price'] = int(item['city_avg_price'])
|
||||
if 'region_avg_price' in item and pd.notna(item['region_avg_price']):
|
||||
doc_dict['region_avg_price'] = item['region_avg_price']
|
||||
doc_dict['region_avg_price'] = int(item['region_avg_price'])
|
||||
if 'avg_price' in item and pd.notna(item['avg_price']):
|
||||
doc_dict['avg_price'] = int(item['avg_price'])
|
||||
if 'avg_price_region_ranking' in item and pd.notna(item['avg_price_region_ranking']):
|
||||
@@ -35,57 +39,69 @@ def build_doc(item):
|
||||
return doc_dict
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步价格信息至206 {index_name}")
|
||||
seaweed_query = {'cat': 'district', 'status': 0}
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
cities = [city]
|
||||
else:
|
||||
cities = None
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
def sync_by_city(city):
|
||||
seaweed_df = tmp_ck.read_sql_query(f"select * from community_summary where city='{city}'")
|
||||
if len(seaweed_df) == 0:
|
||||
return
|
||||
|
||||
# 小区所在城市均价
|
||||
city_price_df = commutils.get_city_price(city=city)
|
||||
if len(city_price_df) > 0:
|
||||
city_price_df = city_price_df[['city', 'avg_price']]
|
||||
city_price_df = city_price_df.rename(columns={'avg_price': 'city_avg_price'})
|
||||
seaweed_df = pd.merge(seaweed_df, city_price_df, how='left', on=['city'])
|
||||
|
||||
# 小区所在区域均价
|
||||
region_price_df = commutils.get_region_price(city=city)
|
||||
region_price_df = region_price_df[['city', 'region', 'avg_price']]
|
||||
region_price_df = region_price_df.rename(columns={'avg_price': 'region_avg_price'})
|
||||
seaweed_df = pd.merge(seaweed_df, region_price_df, how='left', on=['city', 'region'])
|
||||
if len(region_price_df) > 0:
|
||||
region_price_df = region_price_df[['city', 'region', 'avg_price']]
|
||||
region_price_df = region_price_df.rename(columns={'avg_price': 'region_avg_price'})
|
||||
seaweed_df = pd.merge(seaweed_df, region_price_df, how='left', on=['city', 'region'])
|
||||
|
||||
# 小区均价
|
||||
avg_price_df = commutils.get_avg_price(city=city)
|
||||
avg_price_df = avg_price_df[['city', 'district_name', 'region', 'avg_price']]
|
||||
avg_price_df = avg_price_df.rename(columns={'district_name': 'name'})
|
||||
seaweed_df = pd.merge(seaweed_df, avg_price_df, how='left', on=['city', 'region', 'name'])
|
||||
if len(avg_price_df) > 0:
|
||||
avg_price_df = avg_price_df[['city', 'district_name', 'region', 'avg_price']]
|
||||
avg_price_df = avg_price_df.rename(columns={'district_name': 'name'})
|
||||
seaweed_df = pd.merge(seaweed_df, avg_price_df, how='left', on=['city', 'region', 'name'])
|
||||
|
||||
# 小区均价区域排名
|
||||
seaweed_df['avg_price_region_ranking'] = seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
|
||||
# 小区均价城市排名
|
||||
seaweed_df['avg_price_ranking'] = seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
|
||||
# 小区均价区域排名
|
||||
seaweed_df['avg_price_region_ranking'] = \
|
||||
seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)[
|
||||
'avg_price']
|
||||
# 小区均价城市排名
|
||||
seaweed_df['avg_price_ranking'] = \
|
||||
seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
|
||||
|
||||
# 12个月均价涨跌幅
|
||||
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=cities, months=-12)
|
||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
|
||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(columns={'district_name': 'name', 'up_and_downs': 'year_on_year'})
|
||||
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
|
||||
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-12)
|
||||
if len(avg_price_up_and_downs_df) > 0:
|
||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
|
||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
|
||||
columns={'district_name': 'name', 'up_and_downs': 'year_on_year'})
|
||||
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
|
||||
|
||||
# 近半年均价涨跌幅
|
||||
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=cities, months=-6)
|
||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
|
||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
|
||||
columns={'district_name': 'name', 'up_and_downs': 'half_on_year'})
|
||||
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
|
||||
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-6)
|
||||
if len(avg_price_up_and_downs_df) > 0:
|
||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
|
||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
|
||||
columns={'district_name': 'name', 'up_and_downs': 'half_on_year'})
|
||||
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
|
||||
|
||||
# 平米租金
|
||||
rent_price_df = commutils.get_rent_price(city=city)
|
||||
rent_price_df = rent_price_df[['city', 'region', 'district_name', 'avg_price']]
|
||||
rent_price_df = rent_price_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_price'})
|
||||
seaweed_df = pd.merge(seaweed_df, rent_price_df, how='left', on=['city', 'region', 'name'])
|
||||
if len(rent_price_df) > 0:
|
||||
rent_price_df = rent_price_df[['city', 'region', 'district_name', 'avg_price']]
|
||||
rent_price_df = rent_price_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_price'})
|
||||
seaweed_df = pd.merge(seaweed_df, rent_price_df, how='left', on=['city', 'region', 'name'])
|
||||
|
||||
# 政府指导价
|
||||
gov_price_df = commutils.get_gov_guide_price()
|
||||
gov_price_df = gov_price_df[['city', 'region', 'name', 'avg_price']]
|
||||
gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'})
|
||||
seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name'])
|
||||
gov_price_df = commutils.get_gov_guide_price(city=city)
|
||||
if len(gov_price_df) > 0:
|
||||
gov_price_df = gov_price_df[['city', 'region', 'name', 'avg_price']]
|
||||
gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'})
|
||||
seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name'])
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
@@ -101,20 +117,28 @@ def sync(city=None):
|
||||
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
try:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
print("更新:" + str(count))
|
||||
except Exception as e:
|
||||
print(e)
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
log.info("更新:" + str(count))
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
try:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
|
||||
|
||||
def sync(curr_city=None):
|
||||
log.info(f"开始同步价格信息至206 {index_name}")
|
||||
if pd.notna(curr_city):
|
||||
cities = [curr_city]
|
||||
else:
|
||||
cities = areautils.get_all_cities()
|
||||
for city_now in cities:
|
||||
log.info(f"开始同步{city_now}")
|
||||
sync_by_city(city_now)
|
||||
log.info(f"完成同步{city_now}")
|
||||
|
||||
log.info(f"完成同步价格信息至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
|
||||
@@ -8,6 +8,7 @@ from goojia_utils import ElasticSearchClient
|
||||
import pandas as pd
|
||||
import logging
|
||||
from elasticsearch import helpers
|
||||
import sys
|
||||
|
||||
from sklearn.mixture import GaussianMixture
|
||||
|
||||
@@ -16,12 +17,13 @@ log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
index_name = "community_summary"
|
||||
|
||||
|
||||
def gmm_train(total_rate, consumption_df, source, target):
|
||||
# 高斯混合模型
|
||||
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
|
||||
consumption_df['cluster'] = gmm.predict(consumption_df[[source]])
|
||||
consumption_df['price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||
consumption_df[target] = consumption_df['price_mean_cluster'].rank(method='dense').astype(int)
|
||||
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
||||
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||
return consumption_df
|
||||
|
||||
@@ -46,12 +48,12 @@ def calc_price_rating(city=None):
|
||||
|
||||
for city in cities:
|
||||
df = consumption_aim_df[consumption_aim_df['city'] == city].copy()
|
||||
if len(df) < 10:
|
||||
if len(df) < 5:
|
||||
continue
|
||||
df = gmm_train(10, df, source, target)
|
||||
if len(df) >= 100:
|
||||
df = gmm_train(100, df, source, score_target)
|
||||
df['price_score'] /= 10
|
||||
df = gmm_train(5, df, source, target)
|
||||
if len(df) >= 5:
|
||||
df = gmm_train(5, df, source, score_target)
|
||||
df['price_score'] /= 5
|
||||
|
||||
# 批量更新
|
||||
for index, item in df.iterrows():
|
||||
@@ -82,4 +84,5 @@ def sync(city=None):
|
||||
calc_price_rating(city)
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
|
||||
@@ -4,19 +4,23 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
from threading import Lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
m_date = dateutils.iso_format()
|
||||
mutex = Lock()
|
||||
|
||||
|
||||
def find_around_seaweed(lng, lat):
|
||||
def find_around_seaweed(city_now, lng, lat):
|
||||
"""
|
||||
查找周边均价
|
||||
"""
|
||||
@@ -27,6 +31,13 @@ def find_around_seaweed(lng, lat):
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{
|
||||
"term": {
|
||||
"city": {
|
||||
"value": city_now
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"range": {
|
||||
"avg_price": {
|
||||
@@ -54,32 +65,50 @@ def build_doc(item):
|
||||
return doc_dict
|
||||
|
||||
|
||||
find_count = 0
|
||||
|
||||
|
||||
def find_around_price_item(seaweed):
|
||||
global find_count
|
||||
find_count += 1
|
||||
if find_count % 1000 == 0:
|
||||
log.info("已查找" + str(find_count))
|
||||
if 'lng' not in seaweed:
|
||||
return
|
||||
lng = commutils.format_lat_lng(seaweed['lng'])
|
||||
lat = commutils.format_lat_lng(seaweed['lat'])
|
||||
if pd.notna(lng):
|
||||
city_now = seaweed['city']
|
||||
around_seaweed_df = find_around_seaweed(city_now, lng, lat)
|
||||
mutex.acquire(timeout=60)
|
||||
if len(around_seaweed_df) > 1:
|
||||
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
|
||||
seaweed['relative_price_ranking'] = around_seaweed_df[around_seaweed_df['id'] == seaweed['_id']].iloc[0][
|
||||
'relative_price_ranking']
|
||||
elif len(around_seaweed_df) == 1:
|
||||
seaweed['relative_price_ranking'] = 0.5
|
||||
mutex.release()
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
log.info("开始同步相对价格排名数据...")
|
||||
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
def find_around_price_item(index, item):
|
||||
if index % 1000 == 0:
|
||||
log.info("已处理" + str(index))
|
||||
if 'lng' not in item:
|
||||
return
|
||||
lng = commutils.format_lat_lng(item['lng'])
|
||||
lat = commutils.format_lat_lng(item['lat'])
|
||||
if pd.notna(lng):
|
||||
around_seaweed_df = find_around_seaweed(lng, lat)
|
||||
if len(around_seaweed_df) > 1:
|
||||
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
|
||||
seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0]
|
||||
elif len(around_seaweed_df) == 1:
|
||||
seaweed_df.loc[index, 'relative_price_ranking'] = 0.5
|
||||
seaweed_list = list()
|
||||
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
for index, item in seaweed_df.iterrows():
|
||||
pool.submit(find_around_price_item, index, item)
|
||||
for seaweed in seaweed_list:
|
||||
pool.submit(find_around_price_item, seaweed)
|
||||
|
||||
# for seaweed in seaweed_list:
|
||||
# find_around_price_item(seaweed)
|
||||
|
||||
seaweed_df = pd.DataFrame(seaweed_list)
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
@@ -90,17 +119,15 @@ def sync(city=None):
|
||||
"_op_type": "update",
|
||||
"_index": index_name,
|
||||
"_id": item['_id'],
|
||||
"doc": doc_info
|
||||
"doc": doc_info,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
try:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
log.info("更新:" + str(count))
|
||||
except Exception as e:
|
||||
log.error(e)
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
log.info("更新:" + str(count))
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
try:
|
||||
@@ -112,4 +139,5 @@ def sync(city=None):
|
||||
log.info("完成同步相对价格排名数据...")
|
||||
|
||||
|
||||
sync()
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
|
||||
@@ -3,35 +3,42 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
from threading import Lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
m_date = dateutils.iso_format()
|
||||
mutex = Lock()
|
||||
|
||||
|
||||
def build_doc(item):
|
||||
doc_dict = dict()
|
||||
if 'three_months_sell_num' in item and pd.notna(item['three_months_sell_num']):
|
||||
doc_dict['three_months_sell_num'] = int(item['three_months_sell_num'])
|
||||
|
||||
if 'half_on_year_sell_num' in item and pd.notna(item['half_on_year_sell_num']):
|
||||
doc_dict['half_on_year_sell_num'] = int(item['half_on_year_sell_num'])
|
||||
return doc_dict
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步出售信息至206 {index_name}")
|
||||
|
||||
seaweed_query = {'cat': 'district', 'status': 0}
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
with ThreadPoolExecutor(50) as pool:
|
||||
with ThreadPoolExecutor(30) as pool:
|
||||
def deal(index, item):
|
||||
if index % 1000 == 0:
|
||||
log.info("已处理" + str(index))
|
||||
@@ -47,7 +54,14 @@ def sync(city=None):
|
||||
|
||||
# 近三个月挂牌量
|
||||
sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=93)
|
||||
|
||||
# 近半年挂牌量
|
||||
sample_half_year_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=186)
|
||||
|
||||
mutex.acquire()
|
||||
seaweed_df.loc[index, 'three_months_sell_num'] = sample_num
|
||||
seaweed_df.loc[index, 'half_on_year_sell_num'] = sample_half_year_num
|
||||
mutex.release()
|
||||
|
||||
for index, item in seaweed_df.iterrows():
|
||||
pool.submit(deal, index, item)
|
||||
@@ -67,20 +81,15 @@ def sync(city=None):
|
||||
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
try:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
print("已更新:" + str(count))
|
||||
except Exception as e:
|
||||
print(e)
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
log.info("已更新:" + str(count))
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
try:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
log.info(f"完成同步出售信息至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -3,10 +3,9 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import re
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
@@ -25,59 +24,20 @@ def build_doc(item):
|
||||
return doc_dict
|
||||
|
||||
|
||||
# def avg_price_map_deal(x):
|
||||
# avg_price_map[x['city'] + x['region'] + x['name']] = x['avg_price']
|
||||
# return x
|
||||
|
||||
|
||||
def deal(seaweed_df, index, item):
|
||||
if index % 1000 == 0:
|
||||
log.info("已处理" + str(index))
|
||||
curr_city = item['city']
|
||||
curr_region = item['region']
|
||||
curr_name = item['name']
|
||||
names = [curr_name]
|
||||
|
||||
if pd.notna(item['alias']) and '[]' != item['alias']:
|
||||
alias_list = re.sub("\[|\]|'", '', item['alias']).split(',')
|
||||
if len(alias_list) > 0:
|
||||
for alias_item in alias_list:
|
||||
names.append(alias_item)
|
||||
|
||||
# 户均总价
|
||||
# if (curr_city + curr_region + curr_name) in avg_price_map:
|
||||
# avg_price = avg_price_map[curr_city + curr_region + curr_name]
|
||||
# if pd.notna(avg_price):
|
||||
# 由于覆盖率较低,暂不过滤
|
||||
# min_avg_price = 0.8 * avg_price
|
||||
# max_avg_rice = 1.2 * avg_price
|
||||
total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366)
|
||||
seaweed_df.loc[index, 'total_price'] = total_price
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步户均总价信息至206 {index_name}")
|
||||
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
|
||||
# 加载库中均价数据
|
||||
# query_sql = "SELECT city, region, name, avg_price FROM community_summary "
|
||||
# if pd.notna(city):
|
||||
# query_sql += " WHERE city='" + city + "' "
|
||||
# community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql)
|
||||
|
||||
# community_summary_df.apply(lambda x: avg_price_map_deal(x), axis=1)
|
||||
|
||||
with ThreadPoolExecutor(30) as pool:
|
||||
for index, item in seaweed_df.iterrows():
|
||||
pool.submit(deal, seaweed_df, index, item)
|
||||
query_sql = "SELECT city, region, name, avg_price, house_avg_area FROM community_summary WHERE avg_price > 0 and house_avg_area > 0 "
|
||||
if pd.notna(city):
|
||||
query_sql += " AND city='" + city + "' "
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql)
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in seaweed_df.iterrows():
|
||||
community_summary_df['total_price'] = community_summary_df['avg_price'] * community_summary_df['house_avg_area']
|
||||
community_summary_df['total_price'] = community_summary_df['total_price'].astype(int)
|
||||
for index, item in community_summary_df.iterrows():
|
||||
doc_info = build_doc(item)
|
||||
|
||||
action = {
|
||||
@@ -106,4 +66,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步户均总价信息至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
|
||||
@@ -5,6 +5,7 @@ import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, dateutils
|
||||
from elasticsearch import helpers
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
@@ -21,17 +22,9 @@ def build_doc(item):
|
||||
return doc_dict
|
||||
|
||||
|
||||
def sync():
|
||||
log.info(f"开始补全户均总价信息至206 {index_name}")
|
||||
|
||||
factor_df = pd.read_csv(path + '/小区户均总价面积因子已去极值.csv', sep='@', encoding='UTF-8')
|
||||
community_df = es206_client.scan_by_sql_to_data_frame("select city, region, name, avg_price from community_summary where avg_price is not null and total_price is null")
|
||||
|
||||
community_factor_df = pd.merge(community_df, factor_df, how='left', on=['_id', 'city', 'region', 'name'])
|
||||
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
|
||||
def save(community_factor_df):
|
||||
community_factor_df = community_factor_df[community_factor_df['total_price'].notna()]
|
||||
community_factor_df['total_price'] = community_factor_df['total_price'].astype(int)
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in community_factor_df.iterrows():
|
||||
@@ -41,25 +34,86 @@ def sync():
|
||||
"_op_type": "update",
|
||||
"_index": index_name,
|
||||
"_id": item['_id'],
|
||||
"doc": doc_info
|
||||
"doc": doc_info,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
try:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
logging.info("已补全:" + str(count))
|
||||
except Exception as e:
|
||||
log.error(e)
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
logging.info("已补全:" + str(count))
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
try:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
except Exception as e:
|
||||
log.error(e)
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
|
||||
|
||||
def complete_by_build_year_area(community_factor_df):
|
||||
build_year_area_df = pd.read_excel(path + '/全国各区各建筑年代户均面积已去极值.xlsx')
|
||||
build_year_area_dict = dict()
|
||||
for index, build_year_area in build_year_area_df.iterrows():
|
||||
build_year_area_dict[
|
||||
build_year_area['city'] + '|' + build_year_area['region'] + "|" + build_year_area['buildyear']] = build_year_area['factor']
|
||||
# 修复按城市跑批数据字段可能不存在问题
|
||||
if 'finish_building_date' not in community_factor_df.columns or 'total_price' not in community_factor_df.columns:
|
||||
return False
|
||||
for index, item in community_factor_df[
|
||||
(community_factor_df['total_price'].isna()) & (community_factor_df['finish_building_date'].notna())].iterrows():
|
||||
finish_building_date = item['finish_building_date']
|
||||
if finish_building_date < 1980:
|
||||
community_factor_df.loc[index, 'factor'] = build_year_area_dict[
|
||||
item['city'] + "|" + item['region'] + "|1980以前"] if item['city'] + "|" + item[
|
||||
'region'] + "|1980以前" in build_year_area_dict else None
|
||||
elif 1980 <= finish_building_date < 1990:
|
||||
community_factor_df.loc[index, 'factor'] = build_year_area_dict[
|
||||
item['city'] + "|" + item['region'] + "|[1980-1990)"] if item['city'] + "|" + item[
|
||||
'region'] + "|[1980-1990)" in build_year_area_dict else None
|
||||
elif 1990 <= finish_building_date < 2000:
|
||||
community_factor_df.loc[index, 'factor'] = build_year_area_dict[
|
||||
item['city'] + "|" + item['region'] + "|[1990-2000)"] if item['city'] + "|" + item[
|
||||
'region'] + "|[1990-2000)" in build_year_area_dict else None
|
||||
elif 2000 <= finish_building_date < 2010:
|
||||
community_factor_df.loc[index, 'factor'] = build_year_area_dict[
|
||||
item['city'] + "|" + item['region'] + "|[2000-2010)"] if item['city'] + "|" + item[
|
||||
'region'] + "|[2000-2010)" in build_year_area_dict else None
|
||||
elif 2010 <= finish_building_date < 2020:
|
||||
community_factor_df.loc[index, 'factor'] = build_year_area_dict[
|
||||
item['city'] + "|" + item['region'] + "|[2010-2020)"] if item['city'] + "|" + item[
|
||||
'region'] + "|[2010-2020)" in build_year_area_dict else None
|
||||
elif finish_building_date >= 2020:
|
||||
community_factor_df.loc[index, 'factor'] = build_year_area_dict[
|
||||
item['city'] + "|" + item['region'] + "|2020及以后"] if item['city'] + "|" + item[
|
||||
'region'] + "|2020及以后" in build_year_area_dict else None
|
||||
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
|
||||
|
||||
|
||||
def complete_by_region_area(community_factor_df):
|
||||
region_area_df = pd.read_excel(path + '/全国各区户均面积已去极值.xlsx')
|
||||
region_area_dict = dict()
|
||||
for index, region_area in region_area_df.iterrows():
|
||||
region_area_dict[region_area['city'] + '|' + region_area['region']] = region_area['factor']
|
||||
for index, item in community_factor_df[community_factor_df['total_price'].isna()].iterrows():
|
||||
community_factor_df.loc[index, 'factor'] = region_area_dict[item['city'] + "|" + item['region']] if item['city'] + "|" + item['region'] in region_area_dict else None
|
||||
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始补全户均总价信息至206 {index_name}")
|
||||
|
||||
sql = "select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null "
|
||||
if pd.notna(city):
|
||||
sql += " and city='" + city + "'"
|
||||
community_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
if len(community_df) > 0:
|
||||
community_df['total_price'] = None
|
||||
complete_by_build_year_area(community_df)
|
||||
complete_by_region_area(community_df)
|
||||
save(community_df)
|
||||
|
||||
log.info(f"完成补全户均总价信息至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
|
||||
@@ -3,11 +3,13 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||
import sys
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -24,14 +26,21 @@ def build_doc(item):
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步成交信息至206 {index_name}")
|
||||
|
||||
seaweed_query = {'cat': 'district', 'status': 0}
|
||||
cities = None
|
||||
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
cities = [city]
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
deal_price_df = commutils.get_deal_avg_price(cities, 3)
|
||||
if len(deal_price_df) <= 0:
|
||||
log.info(f"无成交,无需同步")
|
||||
return
|
||||
if 'match_city' not in deal_price_df.columns or 'match_region' not in deal_price_df.columns or 'match_name' not in deal_price_df.columns:
|
||||
log.info(f"无成交,无需同步")
|
||||
return
|
||||
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
|
||||
deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name',
|
||||
'avg_price': 'three_month_trade_avg_price'})
|
||||
@@ -68,4 +77,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步成交信息至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
|
||||
@@ -3,13 +3,15 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -70,10 +72,10 @@ def deal(actions, item):
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步交通至206 {index_name}")
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
tmp_sql = f"select * from community_summary "
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
@@ -86,4 +88,5 @@ def sync(city=None):
|
||||
log.info(f"完成同步交通至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(sys_city)
|
||||
@@ -26,8 +26,10 @@ def sync_district_office(type):
|
||||
for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0}, {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1,
|
||||
'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1, 'status': 1})\
|
||||
.batch_size(10000):
|
||||
updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'],
|
||||
'estate_type': doc['estate_type2'], 'status': doc['status']}
|
||||
updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'], 'status': doc['status']}
|
||||
if 'estate_type2' in doc:
|
||||
updateDict['estate_type'] = doc['estate_type2']
|
||||
|
||||
id_list.append({'_id': doc['_id']})
|
||||
if 'block' in doc:
|
||||
updateDict['block'] = doc['block']
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
"community_summary_dag",
|
||||
# 每月2日1点整分运行
|
||||
schedule_interval="0 1 2 * *",
|
||||
start_date=pendulum.datetime(2022, 10, 12, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="地址输出信息分类数据定时更新"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='comm',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t2 = BashOperator(
|
||||
task_id='price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t3 = BashOperator(
|
||||
task_id='sell',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t4 = BashOperator(
|
||||
task_id='avg_cost',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t5 = BashOperator(
|
||||
task_id='trade',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t6 = BashOperator(
|
||||
task_id='total_price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t6_1 = BashOperator(
|
||||
task_id='total_price_complete',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t7 = BashOperator(
|
||||
task_id='traffic',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t7_1 = BashOperator(
|
||||
task_id='business_district',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t8 = BashOperator(
|
||||
task_id='medical',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t9 = BashOperator(
|
||||
task_id='life',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t10 = BashOperator(
|
||||
task_id='relative_price_ranking',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t11 = BashOperator(
|
||||
task_id='consumption',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t12 = BashOperator(
|
||||
task_id='price_rating',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t13 = BashOperator(
|
||||
task_id='education',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t14 = BashOperator(
|
||||
task_id='education2',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t15 = BashOperator(
|
||||
task_id='extra',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t16 = BashOperator(
|
||||
task_id='other',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16
|
||||
@@ -3,11 +3,11 @@ import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
"amap_key_generate_dag",
|
||||
dag_id="amap_key_generate_dag",
|
||||
# 每天运行
|
||||
schedule_interval="0 2 * * *",
|
||||
start_date=pendulum.datetime(2022, 7, 18, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
schedule_interval="0/1 14 * * *",
|
||||
start_date=pendulum.datetime(2023, 3, 18, tz="Asia/Shanghai"),
|
||||
catchup=True
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='amap_key_generate',
|
||||
@@ -0,0 +1,17 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="change_new_to_old_seaweed_dag",
|
||||
# 每月1日1点运行
|
||||
schedule_interval="0 1 1 * *",
|
||||
start_date=pendulum.datetime(2023, 2, 10, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="新房变二手房定时脚本"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='change_new_to_old_seaweed_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/change_new_to_old_seaweed.py',
|
||||
)
|
||||
t1
|
||||
@@ -0,0 +1,98 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="community_summary_dag",
|
||||
# 每月2日1点整分运行
|
||||
schedule_interval="0 1 2 * *",
|
||||
start_date=pendulum.datetime(2023, 3, 16, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="地址输出信息分类数据定时更新"
|
||||
) as dag:
|
||||
comm = BashOperator(
|
||||
task_id='comm',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
price = BashOperator(
|
||||
task_id='price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
sell = BashOperator(
|
||||
task_id='sell',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
avg_cost = BashOperator(
|
||||
task_id='avg_cost',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
trade = BashOperator(
|
||||
task_id='trade',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
total_price = BashOperator(
|
||||
task_id='total_price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
total_price_complete = BashOperator(
|
||||
task_id='total_price_complete',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
traffic = BashOperator(
|
||||
task_id='traffic',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
business_district = BashOperator(
|
||||
task_id='business_district',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
medical = BashOperator(
|
||||
task_id='medical',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
life = BashOperator(
|
||||
task_id='life',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
relative_price_ranking = BashOperator(
|
||||
task_id='relative_price_ranking',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
estate_charge_complete = BashOperator(
|
||||
task_id='estate_charge_complete',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
estate_charge_ranking = BashOperator(
|
||||
task_id='estate_charge_ranking',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
consumption = BashOperator(
|
||||
task_id='consumption',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
person_rating = BashOperator(
|
||||
task_id='person_rating',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/person_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
price_rating = BashOperator(
|
||||
task_id='price_rating',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
education = BashOperator(
|
||||
task_id='education',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
education2 = BashOperator(
|
||||
task_id='education2',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
extra = BashOperator(
|
||||
task_id='extra',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
other = BashOperator(
|
||||
task_id='other',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
# 人均消费并行,值为空
|
||||
comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> [estate_charge_ranking, price_rating, person_rating] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other
|
||||
@@ -3,10 +3,10 @@ import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
"didata_sync_dag",
|
||||
dag_id="didata_sync_dag",
|
||||
# 每月28日3点整分运行
|
||||
schedule_interval="0 3 28 * *",
|
||||
start_date=pendulum.datetime(2022, 8, 22, tz="Asia/Shanghai"),
|
||||
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="地数相关数据同步type: district或office, 不填默认同步全部"
|
||||
) as dag:
|
||||
@@ -0,0 +1,18 @@
|
||||
CREATE TABLE tmp.community_summary
|
||||
(
|
||||
_id String COMMENT '小区ID',
|
||||
city String COMMENT '城市',
|
||||
region String COMMENT '区域',
|
||||
name String COMMENT '名称',
|
||||
lng Nullable(Float32) COMMENT '经度',
|
||||
lat Nullable(Float32) COMMENT '纬度',
|
||||
alias Array(String) COMMENT '别名',
|
||||
estate_type2 String COMMENT '物业类型',
|
||||
build_type String COMMENT '建筑类型',
|
||||
household_count String COMMENT '户数',
|
||||
block String COMMENT '板块',
|
||||
estate_charge String COMMENT '物业费',
|
||||
finish_building_date String COMMENT '建成年代',
|
||||
m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间'
|
||||
) ENGINE ReplacingMergeTree(m_date)
|
||||
PARTITION BY city PRIMARY KEY (city, region, name) ORDER BY (city, region, name);
|
||||
@@ -0,0 +1,44 @@
|
||||
CREATE TABLE tmp.mt_community
|
||||
(
|
||||
id String COMMENT '小区ID',
|
||||
city String COMMENT '城市',
|
||||
region String COMMENT '区域',
|
||||
name String COMMENT '名称',
|
||||
alias String COMMENT '别名',
|
||||
address String COMMENT '地址',
|
||||
addr_alias String COMMENT '地址别名',
|
||||
avg_price Nullable(INT) COMMENT '均价',
|
||||
rent_price Nullable(INT) COMMENT '平米租金',
|
||||
lng Nullable(Float32) COMMENT '经度',
|
||||
lat Nullable(Float32) COMMENT '纬度',
|
||||
estate_type2 String COMMENT '物业类型',
|
||||
block String COMMENT '板块',
|
||||
loop_line_info String COMMENT '环线',
|
||||
business_quarter String COMMENT '商圈',
|
||||
households String COMMENT '户数',
|
||||
property_fee String COMMENT '物业费',
|
||||
car_site_count String COMMENT '停车位',
|
||||
property_life String COMMENT '产权年限',
|
||||
house_property String COMMENT '产权性质',
|
||||
all_area String COMMENT '占地面积',
|
||||
build_area String COMMENT '建筑面积',
|
||||
plot_ratio String COMMENT '容积率',
|
||||
landscaping_ratio String COMMENT '绿化率',
|
||||
property_type String COMMENT '物业类型',
|
||||
build_type String COMMENT '建筑类型',
|
||||
build_structure String COMMENT '建筑结构',
|
||||
finish_building_date String COMMENT '建成年代',
|
||||
developer String COMMENT '开发商',
|
||||
house_type String COMMENT '户型',
|
||||
house_area String COMMENT '房型建筑面积',
|
||||
deal_num Nullable(INT) COMMENT '近一年成交量',
|
||||
district_facilitys String COMMENT '周边配套数量字段',
|
||||
school Nullable(INT) COMMENT '学校数量',
|
||||
market Nullable(INT) COMMENT '商场数量',
|
||||
subway Nullable(INT) COMMENT '地铁数量',
|
||||
bank Nullable(INT) COMMENT '银行数量',
|
||||
hospital Nullable(INT) COMMENT '医院数量',
|
||||
status INT COMMENT '0 正常 1 删除',
|
||||
m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间'
|
||||
) ENGINE ReplacingMergeTree(m_date)
|
||||
PARTITION BY city PRIMARY KEY (city, region, name) ORDER BY (city, region, name);
|
||||
@@ -0,0 +1,2 @@
|
||||
pip install -r /opt/airflow/dags/repo/dags/requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/
|
||||
pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/
|
||||
@@ -1,3 +0,0 @@
|
||||
"""
|
||||
建行脚本
|
||||
"""
|
||||
@@ -6,12 +6,12 @@ with DAG(
|
||||
dag_id="merge_dag",
|
||||
# 每日下午4点
|
||||
schedule_interval="0 16 * * *",
|
||||
start_date=pendulum.datetime(2022, 11, 15, tz="Asia/Shanghai"),
|
||||
start_date=pendulum.datetime(2023, 3, 18, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="每日融合文件检测"
|
||||
description="每日融合文件检测, 若想指定检测某一天如20230314,则传参day='20230314'"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='merge',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/jh/merge/merge.py',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/jh/merge/merge.py {{dag_run.conf["day"] if "day" in dag_run.conf}}',
|
||||
)
|
||||
t1
|
||||
@@ -0,0 +1,30 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="price_exception_dag",
|
||||
# 每月23日运行
|
||||
schedule_interval="0 10 23 * *",
|
||||
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="建行每月价格异常检测脚本,默认检测本月与上月差异, 若对比上月与前月价格,设置year_month=\"202210\"(前月日期即可) 若输出样本存在与否,设置year_month=sample"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='price_evaluation',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_evaluation.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}',
|
||||
)
|
||||
t2 = BashOperator(
|
||||
task_id='price_exception',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}',
|
||||
)
|
||||
t3 = BashOperator(
|
||||
task_id='no_price_community',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/jh/price/no_price_community.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}',
|
||||
)
|
||||
t4 = BashOperator(
|
||||
task_id='price_deviation',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_deviation.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}',
|
||||
)
|
||||
|
||||
t1 >> t2 >> t3 >> t4
|
||||
@@ -6,7 +6,7 @@ with DAG(
|
||||
dag_id="third_price_dag",
|
||||
# 每月23日运行
|
||||
schedule_interval="0 16 18 * *",
|
||||
start_date=pendulum.datetime(2022, 11, 21, tz="Asia/Shanghai"),
|
||||
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="建行过滤第三方均价"
|
||||
) as dag:
|
||||
+37
-230
@@ -1,242 +1,21 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[56]:
|
||||
|
||||
|
||||
import os
|
||||
import tarfile
|
||||
from goojia_utils import dateutils, ftputils, tarutils
|
||||
import datetime
|
||||
import calendar
|
||||
import requests
|
||||
import json
|
||||
import shutil
|
||||
|
||||
|
||||
# In[9]:
|
||||
|
||||
|
||||
import ftplib
|
||||
import os
|
||||
import socket
|
||||
import stat
|
||||
from os.path import isdir
|
||||
|
||||
from numpy import unicode
|
||||
|
||||
import paramiko
|
||||
|
||||
|
||||
def _check_local(local):
|
||||
if not os.path.exists(local):
|
||||
try:
|
||||
os.mkdir(local)
|
||||
except IOError as err:
|
||||
print(err)
|
||||
|
||||
|
||||
class SftpClient:
|
||||
def __init__(self):
|
||||
self.client = None
|
||||
self.sftp = None
|
||||
pass
|
||||
|
||||
def sftp_connect(self, username, password, host, port=22):
|
||||
"""
|
||||
建立连接,获取sftp句柄
|
||||
:param username
|
||||
:param password
|
||||
:param host
|
||||
:param port
|
||||
:return:
|
||||
"""
|
||||
try:
|
||||
self.client = paramiko.Transport((host, port))
|
||||
except Exception as error:
|
||||
print(error)
|
||||
else:
|
||||
try:
|
||||
self.client.connect(username=username, password=password)
|
||||
except Exception as error:
|
||||
print(error)
|
||||
else:
|
||||
self.sftp = paramiko.SFTPClient.from_transport(self.client)
|
||||
|
||||
def get(self, remote, local):
|
||||
"""
|
||||
从远程机器上下载文件(夹)到本地
|
||||
:param remote 远程路径
|
||||
:param local 本地路径
|
||||
"""
|
||||
try:
|
||||
result = self.sftp.stat(remote)
|
||||
except IOError as err:
|
||||
error = '[ERROR %s] %s: %s' % (err.errno, os.path.basename(os.path.normpath(remote)), err.strerror)
|
||||
print(error)
|
||||
else:
|
||||
# 判断远程文件是否为目录
|
||||
if stat.S_ISDIR(result.st_mode):
|
||||
dirname = os.path.basename(os.path.normpath(remote))
|
||||
local = os.path.join(local, dirname)
|
||||
_check_local(local)
|
||||
for file in self.sftp.listdir(remote):
|
||||
sub_remote = os.path.join(remote, file)
|
||||
sub_remote = sub_remote.replace('\\', '/')
|
||||
self.get(sub_remote, local)
|
||||
else:
|
||||
# 拷贝文件
|
||||
if os.path.isdir(local):
|
||||
local = os.path.join(local, os.path.basename(remote))
|
||||
try:
|
||||
self.sftp.get(remote, local)
|
||||
except IOError as err:
|
||||
print(err)
|
||||
else:
|
||||
print('[get]', local, '<==', remote)
|
||||
|
||||
# 断开连接
|
||||
def disconnect(self):
|
||||
try:
|
||||
self.client.close()
|
||||
except Exception as error:
|
||||
print(error)
|
||||
error
|
||||
|
||||
|
||||
class FtpClient:
|
||||
|
||||
def __init__(self):
|
||||
self.ftp = ftplib.FTP()
|
||||
self.ftp.set_debuglevel(2)
|
||||
pass
|
||||
|
||||
def connect_ftp(self, host, port, timeout=60, user=None, passwd=None):
|
||||
"""
|
||||
连接登录ftp
|
||||
:param host:
|
||||
:param port:
|
||||
:param user:
|
||||
:param passwd:
|
||||
:return:
|
||||
"""
|
||||
try:
|
||||
self.ftp.connect(host=host, port=port, timeout=timeout)
|
||||
print("*******已经成功连接服务器FTP服务!!!")
|
||||
except(socket.error, socket.gaierror) as e:
|
||||
print("错误:无法访问服务器FTP服务!!!")
|
||||
print(e)
|
||||
exit()
|
||||
try:
|
||||
self.ftp.login(user=user, passwd=passwd)
|
||||
except ftplib.error_perm as e:
|
||||
print("登录失败,请检查用户名和密码,错误代码")
|
||||
print(e)
|
||||
exit()
|
||||
|
||||
def cat_dir(self, filepath):
|
||||
"""
|
||||
显示ftp指定目录下的文件
|
||||
:param filepath:
|
||||
:return:
|
||||
"""
|
||||
return self.ftp.dir(filepath)
|
||||
|
||||
def make_dir(self, filepath):
|
||||
"""
|
||||
新建ftp下的目录,ftp目录下不存在此文件
|
||||
:param filepath:
|
||||
:return:
|
||||
"""
|
||||
try:
|
||||
filepath.encode("utf-8")
|
||||
mkd = self.ftp.mkd(filepath)
|
||||
return mkd
|
||||
except ftplib.error_perm as e:
|
||||
print("请查看目录下是否存在同名文件,异常为")
|
||||
print(e)
|
||||
|
||||
def remove_dir(self, dirname):
|
||||
"""
|
||||
删除目录
|
||||
:param dirname
|
||||
"""
|
||||
try:
|
||||
self.ftp.rmd(dirname)
|
||||
except ftplib.error_perm as e:
|
||||
print("异常:!!!目录已经被删除了")
|
||||
print(e)
|
||||
|
||||
def download_file(self, localpath, remotepath, filename):
|
||||
"""
|
||||
从FTP指定目录上下载文件,并保存到本地指定路径
|
||||
:param localpath: linux上路径
|
||||
:param remotepath: 本地路径
|
||||
:param filename: 文件名
|
||||
"""
|
||||
bufsize = 8192 # 设置缓冲区大小;
|
||||
ulocalpath = unicode(localpath, "utf-8")
|
||||
try:
|
||||
self.ftp.cwd(remotepath)
|
||||
filepath = ulocalpath + '/'
|
||||
file_handler = open(filepath + filename, "wb").write # 以写模式打开本地文件,---需下载到本地的路径
|
||||
# 传一个回调函数给retrbinary() 它在每接收一个二进制数据时都会被调用
|
||||
self.ftp.retrbinary("RETR " % filename, file_handler, bufsize) # 接收服务器上文件并写入本地文件
|
||||
except ftplib.error_perm as e:
|
||||
print("请检查路径是否正确,是否有权限读取,错误代码")
|
||||
print(e)
|
||||
|
||||
def upload(self, localpath):
|
||||
"""
|
||||
上传至ftp
|
||||
:param localpath: linux上路径
|
||||
"""
|
||||
bufsize = 8192
|
||||
ulocalpath = unicode(localpath, "utf-8")
|
||||
try:
|
||||
file_handler = open(ulocalpath, "rb")
|
||||
filename = os.path.split(ulocalpath)[-1]
|
||||
try:
|
||||
self.ftp.storbinary("stor " % filename, file_handler, bufsize)
|
||||
except ftplib.error_perm as e:
|
||||
print("请检查上传路径是否正确,是否连接FTP,---错误代码")
|
||||
print(e)
|
||||
except e:
|
||||
print("请检查本地路径是否存在····")
|
||||
|
||||
def lst_file(self, path):
|
||||
"""
|
||||
获取指定目录下文件
|
||||
:param path
|
||||
"""
|
||||
s = []
|
||||
try:
|
||||
for name in self.ftp.nlst(path):
|
||||
name1 = unicode(name, "utf-8")
|
||||
s.append(name1)
|
||||
return s
|
||||
except ftplib.all_errors as e:
|
||||
print("请检查目录是否正确,错误代码时-----")
|
||||
print(e)
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
退出ftp
|
||||
"""
|
||||
self.ftp.quit()
|
||||
import sys
|
||||
|
||||
result = []
|
||||
|
||||
# #### 检测传输压缩文件是否完整
|
||||
|
||||
# ##### 1. 融合每天检测
|
||||
|
||||
try:
|
||||
today = dateutils.add_months(datetime.datetime.now(), days=1)
|
||||
date = dateutils.format_date(today, '%Y%m%d')
|
||||
date = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
if date is None:
|
||||
today = dateutils.add_months(datetime.datetime.now(), days=1)
|
||||
date = dateutils.format_date(today, '%Y%m%d')
|
||||
print('当前月:' + date)
|
||||
curr_month_days = calendar.monthrange(today.year, today.month)[1]
|
||||
print('当月天数:' + str(curr_month_days))
|
||||
# curr_month_days = calendar.monthrange(today.year, today.month)[1]
|
||||
# print('当月天数:' + str(curr_month_days))
|
||||
|
||||
|
||||
# In[79]:
|
||||
@@ -245,13 +24,18 @@ try:
|
||||
path = '/home/airflow/data/rh'
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
if not os.path.exists(path + "/" + date):
|
||||
os.makedirs(path + "/" + date)
|
||||
|
||||
client = SftpClient()
|
||||
client = ftputils.SftpClient()
|
||||
client.sftp_connect(username="ccb", password="ccb!@#", host="192.168.0.45", port=17495)
|
||||
client.get(remote="/home/ccb_rh/output/" + date, local=path)
|
||||
client.get(remote="/home/ccb_rh/output/" + date + '/000000_' + date + '_FJW.tar.gz', local=path + '/' + date)
|
||||
client.get(remote="/home/ccb_rh/output/" + date + '/' + date + '.ok', local=path + '/' + date)
|
||||
client.disconnect()
|
||||
|
||||
|
||||
|
||||
|
||||
# In[80]:
|
||||
|
||||
|
||||
@@ -322,6 +106,29 @@ try:
|
||||
result.append('FJW_000000_' + cityid + '_BIGDATA_POSITION_FJW_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_' + cityid + '_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_' + cityid + '_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||
|
||||
if 'FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_000000_GUIDE_STATISTICS_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_000000_GUIDE_STATISTICS_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat文件不存在')
|
||||
|
||||
if 'FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_' + date + '_0001.dat文件不存在')
|
||||
|
||||
if len(result) == 0:
|
||||
result.append('融合检测完毕未发现异常文件')
|
||||
shutil.rmtree(path + "/" + date)
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,63 @@
|
||||
import os, sys
|
||||
import logging
|
||||
import pandas as pd
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from goojia_utils import dateutils, oracle
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
os.environ["NLS_LANG"] = "American_America.AL32UTF8"
|
||||
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
|
||||
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
path = "/home/airflow/data/jh/price"
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_date(datetime.utcnow(), "%Y%m"))
|
||||
log.info(f"测试月份:{year_month}")
|
||||
|
||||
avm_df = pd.read_sql(f"select urbn_id,urbn_nm,gujia_bds_comm_num,gujiabdsvalmnycommnum from t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month} ", con=engine)
|
||||
comm_df = pd.read_sql(f"select urbn_id,gujia_admndiv_nm,gujia_comm_id,comm_nm from base.t0831_bigdata_community_gj_h where gujia_bds_bds_st=0 ", con=engine)
|
||||
price_df = pd.read_sql(f"select gujia_bds_id as gujia_comm_id,gujia_comm_basic_prc from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month} ", con=engine)
|
||||
comm_df = comm_df.merge(avm_df, on="urbn_id", how="left")
|
||||
comm_df = comm_df.merge(price_df, on="gujia_comm_id", how="left")
|
||||
no_price_comm_df = comm_df[comm_df.gujia_comm_basic_prc.isna()][['urbn_id','urbn_nm','gujia_admndiv_nm','gujia_comm_id','comm_nm']]
|
||||
avm_df.loc[:,'无均价数量'] = avm_df['gujia_bds_comm_num'] - avm_df['gujiabdsvalmnycommnum']
|
||||
avm_df.loc[:,'num'] = 60 - avm_df['gujiabdsvalmnycommnum']
|
||||
avm_df = avm_df[avm_df['无均价数量'] > 0]
|
||||
avm_df = avm_df[['urbn_id','urbn_nm','num']]
|
||||
|
||||
file_name = f"{year_month}无价格小区统计数据.xlsx"
|
||||
aim_file_name = path + '/' + file_name
|
||||
with pd.ExcelWriter(aim_file_name) as writer:
|
||||
no_price_comm_df.to_excel(writer, sheet_name="无价格小区列表", columns=['urbn_id','urbn_nm','gujia_admndiv_nm','gujia_comm_id','comm_nm'], header=["城市id", "城市名称","区域","小区id","小区名称"], index=False)
|
||||
avm_df.to_excel(writer, sheet_name="待补充数量", columns=['urbn_id','urbn_nm','num'], header=["城市id","城市名称","待补充数量"],index=False)
|
||||
|
||||
log.info("无价格小区统计数据结束")
|
||||
def up_to_nextcloud():
|
||||
"""
|
||||
上传统计信息至nextcloud
|
||||
"""
|
||||
from nextcloud import NextCloud
|
||||
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
|
||||
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
|
||||
NEXTCLOUD_PASSWORD = "goojia123456"
|
||||
nextcloud_file_name = '共享/建行/' + file_name
|
||||
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD)
|
||||
nxc.upload_file(aim_file_name, nextcloud_file_name)
|
||||
|
||||
def send_msg():
|
||||
infos = "【准确度测评】无价格小区统计数据生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情"
|
||||
data = {
|
||||
"msgtype": "text",
|
||||
"text": {
|
||||
"content": infos
|
||||
}
|
||||
}
|
||||
# 发送消息至群
|
||||
requests.post(
|
||||
url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
|
||||
json=data)
|
||||
up_to_nextcloud()
|
||||
send_msg()
|
||||
@@ -0,0 +1,74 @@
|
||||
import os, sys
|
||||
import logging
|
||||
import pandas as pd
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from goojia_utils import dateutils, oracle
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
os.environ["NLS_LANG"] = "American_America.AL32UTF8"
|
||||
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
|
||||
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
path = "/home/airflow/data/jh/price"
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
month_now = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now()), '%Y%m')
|
||||
month_last = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-1), '%Y%m')
|
||||
month_before = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-2), '%Y%m')
|
||||
|
||||
type = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
log.info("输出类型" + type)
|
||||
|
||||
# 新增支持上一月与前月对比
|
||||
if type == month_last:
|
||||
month_now = month_last
|
||||
month_last = month_before
|
||||
log.info(f"测试月份:{month_now}")
|
||||
|
||||
price_df = pd.read_sql(f"select urbn_id,urbn_nm,gujia_admndiv_nm,gujia_bds_id as gujia_comm_id,comm_nm,gujia_bds_clmtd,gujia_comm_basic_prc from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={month_now} ", con=engine)
|
||||
deal_df = pd.read_sql(f"select gujia_bds_mdl_id,gujia_comm_id,gujia_bds_trdtckt_prc from base.t0831_bigdata_deal_gj_h where to_char(txn_dt,'yyyyMM')='{month_last}' ", con=engine)
|
||||
deal_price_df = deal_df.merge(price_df, on="gujia_comm_id", how="left")
|
||||
deal_price_df["ppe"] = round((deal_price_df["gujia_comm_basic_prc"] - deal_price_df['gujia_bds_trdtckt_prc']) / deal_price_df['gujia_bds_trdtckt_prc'], 4)
|
||||
deal_price_df = deal_price_df[abs(deal_price_df['ppe']) > 0.15]
|
||||
|
||||
# third_df = pd.read_sql(f"select gujia_trdpt_avprc_id,gujia_bds_id as gujia_comm_id,gujia_dtsr_dsc,gujia_bds_comm_avprc from base.t0831_bigdata_third_party_price_gj_h where gujiabds_avprc_dt_val={month_last} ", con=engine)
|
||||
# third_price_df = third_df.merge(price_df, on="gujia_comm_id", how="left")
|
||||
# third_price_df["ppe"] = round((third_price_df["gujia_comm_basic_prc"] - third_price_df['gujia_bds_comm_avprc']) / third_price_df['gujia_bds_comm_avprc'], 4)
|
||||
# third_price_df = third_price_df[abs(third_price_df['ppe']) > 0.15]
|
||||
|
||||
|
||||
file_name = f"{month_now}测试偏差较大案例统计数据.xlsx"
|
||||
aim_file_name = path + '/' + file_name
|
||||
with pd.ExcelWriter(aim_file_name) as writer:
|
||||
deal_price_df.to_excel(writer, sheet_name="成交", columns=['urbn_id','urbn_nm','gujia_bds_mdl_id','gujia_admndiv_nm','gujia_comm_id','comm_nm','gujia_bds_clmtd','gujia_comm_basic_prc','gujia_bds_trdtckt_prc','ppe'], header=["城市id", "城市名称","成交id","区域","小区id","小区名称",'计算类型','计算价格','成交价格','PPE'], index=False)
|
||||
# third_price_df.to_excel(writer, sheet_name="第三方", columns=['urbn_id','urbn_nm','gujia_trdpt_avprc_id','gujia_admndiv_nm','gujia_comm_id','comm_nm','gujia_bds_clmtd','gujia_comm_basic_prc','gujia_bds_comm_avprc','PPE','gujia_dtsr_dsc'], header=["城市id", "城市名称","第三方id","区域","小区id","小区名称",'计算类型','计算价格','第三方价格','PPE','来源'],index=False)
|
||||
|
||||
log.info("偏差较大案例统计数据结束")
|
||||
def up_to_nextcloud():
|
||||
"""
|
||||
上传统计信息至nextcloud
|
||||
"""
|
||||
from nextcloud import NextCloud
|
||||
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
|
||||
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
|
||||
NEXTCLOUD_PASSWORD = "goojia123456"
|
||||
nextcloud_file_name = '共享/建行/' + file_name
|
||||
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD)
|
||||
nxc.upload_file(aim_file_name, nextcloud_file_name)
|
||||
|
||||
def send_msg():
|
||||
infos = "【准确度测评】测试偏差较大案例统计数据生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情"
|
||||
data = {
|
||||
"msgtype": "text",
|
||||
"text": {
|
||||
"content": infos
|
||||
}
|
||||
}
|
||||
# 发送消息至群
|
||||
requests.post(
|
||||
url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
|
||||
json=data)
|
||||
up_to_nextcloud()
|
||||
send_msg()
|
||||
@@ -0,0 +1,129 @@
|
||||
import os, sys
|
||||
import logging
|
||||
import pandas as pd
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from goojia_utils import dateutils, oracle
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
os.environ["NLS_LANG"] = "American_America.AL32UTF8"
|
||||
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
|
||||
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
|
||||
|
||||
important_cities =["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"]
|
||||
city_order = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx")
|
||||
|
||||
|
||||
def convert_calc_type(cale_type):
|
||||
if cale_type == 1:
|
||||
return "指导价"
|
||||
elif cale_type == 2:
|
||||
return "样本"
|
||||
elif cale_type == 3:
|
||||
return "神经网络"
|
||||
|
||||
|
||||
fitment_code_map = {"0601": "毛坯", "0602": "简装", "0603": "精装"}
|
||||
|
||||
|
||||
def convert_fitment_code(fitment_code):
|
||||
if pd.isna(fitment_code):
|
||||
return None
|
||||
fitment_code = str(fitment_code)
|
||||
if fitment_code in fitment_code_map:
|
||||
return fitment_code_map[fitment_code]
|
||||
return None
|
||||
|
||||
|
||||
house_aspect_code_map = {"0301": "南北", "0302": "东南", "0303": "南", "0304": "东西", "0305": "东", "0306": "西南", "0307": "西",
|
||||
"0308": "北", "0309": "东北",
|
||||
"0310": "西北"}
|
||||
|
||||
|
||||
def convert_house_aspect_code(house_aspect_code):
|
||||
if pd.isna(house_aspect_code):
|
||||
return None
|
||||
house_aspect_code = str(house_aspect_code)
|
||||
if house_aspect_code in house_aspect_code_map:
|
||||
return house_aspect_code_map[house_aspect_code]
|
||||
return None
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
path = "/home/airflow/data/jh/price"
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_date(datetime.utcnow(), "%Y%m"))
|
||||
log.info(f"测试月份:{year_month}")
|
||||
avm_evaluation_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_bds_yrmo': 'year_month', 'gujia_bds_comm_num': 'community_num', 'gujiabdsvalmnycommnum': 'has_price_community_num', 'gujia_prc_cvr_rate': 'price_coverage', 'gujiabdsmnldpccommnum': 'artificial_calc_num', 'gujia_mnl_drc_prc_pct': 'artificial_calc_coverage', 'gujiabdssamp_comm_num': 'sample_calc_num', 'gujiaext_samp_clc_pct': 'sample_calc_coverage', 'gujiabdsnervetncomnum': 'ann_calc_num', 'gujianervethrunclcpct': 'ann_calc_coverage', 'gujiabds_tst_case_num': 'test_num', 'gujiabdsmdltstcommnum': 'deal_test_comm_num', 'gujiatst_comm_num_pct': 'test_comm_coverage', 'gujia_ppe15_cvr_rate': 'ppe15_coverage', 'gujia_rto_mdn': 'rate_median', 'gujia_rto_lv_san_ceft': 'coefficient', 'gujiabdsmdltstcasenum': 'deal_test_case_num', 'gujiamdltstcommnumpct': 'deal_test_comm_coverage', 'gujiamdltst_ppe15_pct': 'deal_ppe15_coverage', 'gujiamdlrtolvsan_ceft': 'deal_coefficient', 'gujia_mdl_md_bit_rto': 'deal_rate_median', 'gujiabdstrdpttcasenum': 'third_test_case_num', 'gujiabdstrdpttcommnum': 'third_test_comm_num', 'gujiatrdpttcommnumpct': 'third_test_comm_coverage', 'gujiatrdpttstppe15pct': 'third_ppe15_coverage', 'gujiatrdptrolvsanceft': 'third_coefficient', 'gujiatrdpt_md_bit_rto': 'third_rate_median', 'data_crt_tm': 'createtime', 'data_udt_tm': 'updatetime', 'gujia_src_co': 'sourcecode'}
|
||||
avm_evaluation_item_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_rgon_id': 'distid', 'gujia_rgon_nm': 'distname', 'gujia_comm_id': 'commid', 'gujia_comm_nm': 'commname', 'gujiabds_clc_yrmo_val': 'year_month', 'gujia_bds_clc_avprc': 'calc_avg_price', 'gujia_bds_avprc': 'avg_price', 'gujia_bds_clc_tp': 'calc_type', 'gujia_bds_rlb': 'confidence', 'gujiabds_to_mkprc_mny': 'target_avg_price', 'mdl_dt': 'dealdate', 'gujiabds_valt_unitprc': 'unit_price', 'gujia_area': 'buildarea', 'gujia_dcrt_val': 'fitment', 'gujia_twrd_val': 'houseaspect', 'gujia_bds_floor_num': 'floornum', 'gujiabdstot_floor_num': 'floorcnt', 'gujia_data_tp': 'data_type', 'gujia_rto': 'rate', 'gujia_mns_rto': 'pe', 'data_crt_tm': 'createtime'}
|
||||
avm_evaluation = pd.read_sql(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month}", con=engine)
|
||||
avm_evaluation_item = pd.read_sql(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val={year_month}", con=engine)
|
||||
avm_evaluation.rename(columns=avm_evaluation_column_map, inplace=True)
|
||||
avm_evaluation_item.rename(columns=avm_evaluation_item_column_map, inplace=True)
|
||||
community_base_price = pd.read_sql(f'''SELECT gujia_bds_id as commid,urbn_nm as cityname,gujia_admndiv_nm as distname,comm_nm as commname,
|
||||
gujia_comm_basic_prc as comm_avg_price,gujia_bds_cfdc_dgr as confidence,gujia_bds_clmtd as calc_type from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month}''',
|
||||
con=engine)
|
||||
communities = pd.read_sql(f"select gujia_comm_id as COMMID from base.t0831_bigdata_community_gj_h", con=engine)
|
||||
|
||||
community_base_price = community_base_price[community_base_price.commid.isin(communities["commid"])]
|
||||
avm_evaluation_item["fitment"] = avm_evaluation_item["fitment"].map(convert_fitment_code)
|
||||
avm_evaluation_item["houseaspect"] = avm_evaluation_item["houseaspect"].map(convert_house_aspect_code)
|
||||
avm_evaluation_item["calc_type"] = avm_evaluation_item["calc_type"].map(convert_calc_type)
|
||||
community_base_price["calc_type"] = community_base_price["calc_type"].map(convert_calc_type)
|
||||
avm_evaluation_item["dealdate"] = avm_evaluation_item["dealdate"].apply(lambda x : x.strftime("%Y-%m-%d") if pd.notna(x) else None)
|
||||
avm_evaluation_item_third = avm_evaluation_item[avm_evaluation_item.data_type=='第三方']
|
||||
avm_evaluation_item_deal = avm_evaluation_item[avm_evaluation_item.data_type=='成交']
|
||||
import_avm_evaluation_item_third = avm_evaluation_item_third[avm_evaluation_item_third.cityname.isin(important_cities)]
|
||||
import_avm_evaluation_item_deal = avm_evaluation_item_deal[avm_evaluation_item_deal.cityname.isin(important_cities)]
|
||||
import_avm_evaluation_item_deal.loc[:, "deal_total_price"] = import_avm_evaluation_item_deal["buildarea"] * import_avm_evaluation_item_deal["target_avg_price"]
|
||||
import_avm_evaluation_item_deal.loc[:, "deal_total_price"] = import_avm_evaluation_item_deal["deal_total_price"].apply(lambda x : int(x))
|
||||
import_avm_evaluation_item_deal.loc[:, "total_price"] = import_avm_evaluation_item_deal["buildarea"] * import_avm_evaluation_item_deal["calc_avg_price"]
|
||||
import_avm_evaluation_item_deal.loc[:, "total_price"] = import_avm_evaluation_item_deal["total_price"].apply(lambda x : int(x))
|
||||
import_community_base_price = community_base_price[community_base_price.cityname.isin(important_cities)]
|
||||
normal_community_base_price = community_base_price[~(community_base_price.cityname.isin(important_cities))]
|
||||
import_avm_evaluation = avm_evaluation[avm_evaluation.cityname.isin(important_cities)]
|
||||
avm_evaluation = avm_evaluation.merge(city_order, on="cityname", how="left")
|
||||
import_community_base_price = import_community_base_price.merge(city_order, on="cityname", how="left")
|
||||
normal_community_base_price = normal_community_base_price.merge(city_order, on="cityname", how="left")
|
||||
import_avm_evaluation_item_deal = import_avm_evaluation_item_deal.merge(city_order, on="cityname", how="left")
|
||||
import_avm_evaluation_item_third = import_avm_evaluation_item_third.merge(city_order, on="cityname", how="left")
|
||||
avm_evaluation.sort_values(by="cityrank", inplace=True)
|
||||
import_community_base_price.sort_values(by="cityrank", inplace=True)
|
||||
normal_community_base_price.sort_values(by="cityrank", inplace=True)
|
||||
import_avm_evaluation_item_deal.sort_values(by="cityrank", inplace=True)
|
||||
import_avm_evaluation_item_third.sort_values(by="cityrank", inplace=True)
|
||||
|
||||
file_name = f"{year_month}估值模型测试.xlsx"
|
||||
aim_file_name = path + '/' + file_name
|
||||
with pd.ExcelWriter(aim_file_name) as writer:
|
||||
avm_evaluation.to_excel(writer, sheet_name="92个城市估值概况", columns=["cityname","community_num","price_coverage","sample_calc_coverage","ann_calc_coverage","artificial_calc_coverage","deal_test_case_num","deal_ppe15_coverage","deal_test_comm_num","deal_test_comm_coverage","deal_rate_median","deal_coefficient","third_test_comm_num","third_test_comm_coverage","third_ppe15_coverage","third_rate_median","third_coefficient"], header=["城市名称","小区量","可输出均价小区占比","样本计算占比","神经网络计算占比","人工指导价占比","成交测试案例总量","成交测试PPE15占比","成交测试小区量","成交测试小区量占比","成交比率中位数","成交比率离散系数","第三方均价测试小区量","第三方均价测试小区量占比","第三方均价测试PPE15占比","第三方比率中位数","第三方比率离散系数"], index=False)
|
||||
import_community_base_price.to_excel(writer, sheet_name="测试10个城市估值数据", columns=["cityname","distname","commname","comm_avg_price","confidence","calc_type"], header=["城市","区域","小区名","估值小区均价","置信度","估值方式"],index=False)
|
||||
import_avm_evaluation_item_deal.to_excel(writer, sheet_name="10个城市成交测试案例", columns=['cityname','distname', 'commname', 'unit_price', 'confidence', 'calc_type','total_price', 'deal_total_price', 'target_avg_price','pe',"dealdate","fitment","houseaspect","floornum","floorcnt","buildarea"], header=["城市","区域","小区名","修正均价","置信度","估值方式","小区估价","对标成交价","对标均价","PPE","dealdate","fitment","houseaspect","floornum","floorcnt","buildarea"], index=False)
|
||||
import_avm_evaluation_item_third.to_excel(writer, sheet_name="10个城市第三方均价测试", columns=["cityname","distname","commname","avg_price","confidence","calc_type","target_avg_price","pe"], header=["城市","区域","小区名","小区均价","置信度","估值方式","对标第三方均价","PPE"],index=False)
|
||||
log.info("估值结果测试结束")
|
||||
def up_to_nextcloud():
|
||||
"""
|
||||
上传统计信息至nextcloud
|
||||
"""
|
||||
from nextcloud import NextCloud
|
||||
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
|
||||
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
|
||||
NEXTCLOUD_PASSWORD = "goojia123456"
|
||||
nextcloud_file_name = '共享/建行/' + file_name
|
||||
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD)
|
||||
nxc.upload_file(aim_file_name, nextcloud_file_name)
|
||||
|
||||
def send_msg():
|
||||
infos = "【准确度测评】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情"
|
||||
data = {
|
||||
"msgtype": "text",
|
||||
"text": {
|
||||
"content": infos
|
||||
}
|
||||
}
|
||||
# 发送消息至群
|
||||
requests.post(
|
||||
url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
|
||||
json=data)
|
||||
up_to_nextcloud()
|
||||
send_msg()
|
||||
@@ -7,9 +7,10 @@ import logging
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
|
||||
|
||||
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
|
||||
def simple_calc(result_df):
|
||||
"""
|
||||
新增是否有样本列
|
||||
@@ -25,8 +26,9 @@ def simple_calc(result_df):
|
||||
if len(actions) == 800:
|
||||
ins = "','".join(actions)
|
||||
ins = "'" + ins + "'"
|
||||
simple_temp_df = oracle.scan_by_sql_to_data_frame("select distinct commid from bigdata_forsale_gj where " +
|
||||
" publishtime> TO_DATE('" + month_before + "','YYYY-MM-DD') and commid in (" + ins + ")")
|
||||
simple_temp_df = pd.read_sql(
|
||||
"select distinct gujia_comm_id as commid from base.t0831_bigdata_forsale_gj_h where " +
|
||||
" anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")", con=engine)
|
||||
|
||||
simple_df = pd.concat([simple_df, simple_temp_df])
|
||||
actions.clear()
|
||||
@@ -34,8 +36,9 @@ def simple_calc(result_df):
|
||||
if len(actions) > 0:
|
||||
ins = "','".join(actions)
|
||||
ins = "'" + ins + "'"
|
||||
simple_temp_df = oracle.scan_by_sql_to_data_frame("select distinct commid from bigdata_forsale_gj where " +
|
||||
" publishtime> TO_DATE('" + month_before + "','YYYY-MM-DD') and commid in (" + ins + ")")
|
||||
simple_temp_df = pd.read_sql(
|
||||
"select distinct gujia_comm_id as commid from base.t0831_bigdata_forsale_gj_h where " +
|
||||
" anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")", con=engine)
|
||||
simple_df = pd.concat([simple_df, simple_temp_df])
|
||||
actions.clear()
|
||||
simple_df['has_sample'] = 1
|
||||
@@ -85,8 +88,8 @@ if type == month_last:
|
||||
month_now = month_last
|
||||
month_last = month_before
|
||||
|
||||
df = oracle.scan_by_sql_to_data_frame("SELECT id, cityname, distname, commid, commname, calc_type, confidence, avg_price, year_month FROM community_base_price_gj WHERE year_month IN('"+month_now+"', '"+month_last+"') " +
|
||||
" ORDER BY cityname, distname, commid, commname, year_month")
|
||||
df = pd.read_sql("SELECT gujia_bss_prc_id as id, urbn_nm as cityname, gujia_admndiv_nm as distname, gujia_bds_id as commid, comm_nm as commname, gujia_bds_clmtd as calc_type, gujia_bds_cfdc_dgr as confidence, gujia_comm_basic_prc as avg_price, gujiabds_clc_yrmo_val as year_month FROM t_app_p9pha_plda.avm_community_base_price_gj_h WHERE gujiabds_clc_yrmo_val IN('"+month_now+"', '"+month_last+"') " +
|
||||
" ORDER BY urbn_nm, gujia_admndiv_nm, gujia_bds_id, comm_nm, gujia_bds_clmtd", con=engine)
|
||||
|
||||
six_map = dict()
|
||||
five_map = dict()
|
||||
@@ -143,7 +146,7 @@ for index, item in city_count_df.iterrows():
|
||||
city_count_df = city_count_df.rename(columns={'小区ID': '小区数量'})
|
||||
city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度')
|
||||
|
||||
excel_writer.save()
|
||||
#excel_writer.save()
|
||||
excel_writer.close()
|
||||
|
||||
up_to_nextcloud()
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="price_exception_dag",
|
||||
# 每月23日运行
|
||||
schedule_interval="0 10 23 * *",
|
||||
start_date=pendulum.datetime(2022, 8, 23, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="建行每月价格异常检测脚本,默认检测本月与上月差异, 若对比上月与前月价格,设置type=\"202210\"(前月日期即可) 若输出样本存在与否,设置type=sample"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='price_exception',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t1
|
||||
@@ -9,11 +9,8 @@ with DAG(
|
||||
dag_id="mt_dag",
|
||||
# 每月5日运行
|
||||
schedule_interval="0 5 5 * *",
|
||||
start_date=pendulum.datetime(2022, 6, 28, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
default_args={
|
||||
"output_date": output_date
|
||||
}
|
||||
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
|
||||
catchup=True
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='mt_comm',
|
||||
@@ -1,13 +1,14 @@
|
||||
"""
|
||||
小区属性导出
|
||||
"""
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils, ClickHouseClient
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
path = "/home/airflow/data/mt"
|
||||
if not os.path.exists(path):
|
||||
@@ -74,5 +75,9 @@ seaweed_new_result_df = seaweed_new_result_df.rename(columns={'_id': 'id', 'mapp
|
||||
'cubage_percent' : 'plot_ratio', 'virescence_percent': 'landscaping_ratio', 'estate_type2': 'property_type',
|
||||
'main_type': 'house_type', 'main_type_area': 'house_area'})
|
||||
|
||||
seaweed_new_result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv',
|
||||
sep='@', encoding='utf-8', index=False)
|
||||
seaweed_new_result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False)
|
||||
|
||||
ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 "
|
||||
tmp_ck.execute(ck_delete_sql)
|
||||
|
||||
tmp_ck.to_sql_plus(seaweed_new_result_df, 'mt_community')
|
||||
|
||||
@@ -10,11 +10,12 @@ import requests
|
||||
path = "/home/airflow/data/mt"
|
||||
|
||||
clickhouse_client = ClickHouseClient(database='fangjia_stat')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
output_date = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(datetime.datetime.now(), '%Y%m')
|
||||
print("输出日期" + output_date)
|
||||
|
||||
result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + ".csv", sep='@', encoding='utf-8')
|
||||
result_df = tmp_ck.read_sql_query("select * from tmp.mt_community")
|
||||
|
||||
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx')
|
||||
cities = city_df['城市列表'].values.tolist()
|
||||
|
||||
+10
-3
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
每个小区近一年成交统计
|
||||
"""
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ElasticSearchClient
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ElasticSearchClient, ClickHouseClient
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import logging
|
||||
@@ -11,6 +11,7 @@ import sys
|
||||
es45_client = ElasticSearchClient(host='192.168.0.45')
|
||||
deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price')
|
||||
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx')
|
||||
cities = city_df['城市列表'].values.tolist()
|
||||
@@ -26,7 +27,7 @@ logging.info("日期信息:" + month_start + ' ' + curr_month_start)
|
||||
month_start_date = dateutils.parse_date(month_start, '%Y%m%d')
|
||||
curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
|
||||
|
||||
result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8')
|
||||
result_df = tmp_ck.read_sql_query("select * from mt_community")
|
||||
|
||||
if 'deal_num' in result_df.columns:
|
||||
result_df = result_df.drop(columns=['deal_num'])
|
||||
@@ -93,4 +94,10 @@ columns = ['id', 'city', 'region', 'name', 'alias', 'address', 'addr_alias', 'av
|
||||
'build_structure', 'finish_building_date',
|
||||
'developer', 'house_type', 'house_area', 'deal_num', 'district_facilitys', 'school', 'market', 'subway',
|
||||
'bank', 'hospital', 'status']
|
||||
result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False, columns=columns)
|
||||
|
||||
result_save_df = result_df[columns]
|
||||
|
||||
ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 "
|
||||
tmp_ck.execute(ck_delete_sql)
|
||||
|
||||
tmp_ck.to_sql_plus(result_save_df, 'mt_community')
|
||||
@@ -1,13 +1,14 @@
|
||||
"""
|
||||
周边配套数量统计
|
||||
"""
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ClickHouseClient
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import logging
|
||||
import sys
|
||||
|
||||
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx')
|
||||
cities = city_df['城市列表'].values.tolist()
|
||||
@@ -73,7 +74,8 @@ district_facilitys_new_df = district_facilitys_df[
|
||||
district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_name': 'name', 'supermarket': 'market'})
|
||||
district_facilitys_new_df = district_facilitys_new_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first')
|
||||
|
||||
result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8')
|
||||
result_df = tmp_ck.read_sql_query("select * from mt_community")
|
||||
|
||||
if 'school' in result_df.columns:
|
||||
result_df = result_df.drop(columns=['school', 'bank', 'hospital', 'market', 'subway'])
|
||||
|
||||
@@ -141,4 +143,7 @@ result_df['bank'].fillna(0, inplace=True)
|
||||
result_df['hospital'].fillna(0, inplace=True)
|
||||
result_df['subway'].fillna(0, inplace=True)
|
||||
|
||||
result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False)
|
||||
ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 "
|
||||
tmp_ck.execute(ck_delete_sql)
|
||||
|
||||
tmp_ck.to_sql_plus(result_df, 'mt_community')
|
||||
@@ -2,17 +2,17 @@
|
||||
租金、均价导出
|
||||
"""
|
||||
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils, ClickHouseClient
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import logging
|
||||
import sys
|
||||
import numpy as np
|
||||
|
||||
#district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat')
|
||||
# 月度均价表
|
||||
district_sell_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_sell_stat')
|
||||
sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
|
||||
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx')
|
||||
@@ -22,7 +22,8 @@ path = "/home/airflow/data/mt"
|
||||
|
||||
output_date = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(datetime.datetime.now(), '%Y%m')
|
||||
|
||||
seaweed_new_result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8')
|
||||
seaweed_new_result_df = tmp_ck.read_sql_query("select * from mt_community")
|
||||
|
||||
if 'avg_price' in seaweed_new_result_df:
|
||||
seaweed_new_result_df = seaweed_new_result_df.drop(columns=['avg_price', 'rent_price'])
|
||||
|
||||
@@ -111,4 +112,9 @@ result_df['ignore_comm'] = None
|
||||
result_df = result_df.apply(lambda x: format_rent(x), axis=1)
|
||||
result_df = result_df[result_df['ignore_comm'] != 1]
|
||||
|
||||
result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False)
|
||||
del result_df['ignore_comm']
|
||||
|
||||
ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 "
|
||||
tmp_ck.execute(ck_delete_sql)
|
||||
|
||||
tmp_ck.to_sql_plus(result_df, 'mt_community')
|
||||
|
||||
@@ -8,11 +8,11 @@ import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
"district_facilities_dag",
|
||||
dag_id="district_facilities_dag",
|
||||
# 每月1日1点运行
|
||||
schedule_interval="0 1 1 * *",
|
||||
start_date=pendulum.datetime(2022, 6, 18, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
|
||||
catchup=True
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='district_facilities',
|
||||
@@ -0,0 +1,3 @@
|
||||
elasticsearch==7.11.0
|
||||
transbigdata==0.4.15
|
||||
pykalman==0.9.5
|
||||
+3
-3
@@ -3,11 +3,11 @@ import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
"community_count_price_rent_dag",
|
||||
dag_id="count_price_rent_dag",
|
||||
# 每周日运行
|
||||
schedule_interval="0 10 * * 7",
|
||||
start_date=pendulum.datetime(2022, 7, 1, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
|
||||
catchup=True
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='community_count_price_rent',
|
||||
@@ -3,11 +3,11 @@ import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
"district_sell_deal_dag",
|
||||
dag_id="district_sell_deal_dag",
|
||||
# 每周五运行
|
||||
schedule_interval="0 1 * * 5",
|
||||
start_date=pendulum.datetime(2022, 6, 20, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
|
||||
catchup=True
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='district_count_thirdavg_deal',
|
||||
@@ -65,22 +65,23 @@ def third_avg_stat():
|
||||
'5i5j_esf_price', 'maitian_esf_price', '365house_esf_price', 'beike_esf_price', 'qfang_esf_price',
|
||||
'fangtan_esf_price',
|
||||
'tuboshi_esf_price', 'zhugezhaofang_esf_price', 'zhongyuan_esf_price', 'jiwu_esf_price', 'anjuke_esf_price']]
|
||||
|
||||
average_statistics_new_df.loc[:, 'has_esf_price'] = average_statistics_new_df['58tc_esf_price'] \
|
||||
.combine_first(average_statistics_new_df['cityhouse_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['soufang_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['lianjia_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['5i5j_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['maitian_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['365house_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['beike_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['qfang_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['fangtan_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['tuboshi_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['zhugezhaofang_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['zhongyuan_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['jiwu_esf_price']) \
|
||||
.combine_first(average_statistics_new_df['anjuke_esf_price'])
|
||||
average_statistics_new_df.fillna(0, inplace=True)
|
||||
average_statistics_new_df.loc[:, 'has_esf_price'] = average_statistics_new_df['58tc_esf_price'] + \
|
||||
average_statistics_new_df['cityhouse_esf_price'] + \
|
||||
average_statistics_new_df['soufang_esf_price'] + \
|
||||
average_statistics_new_df['lianjia_esf_price'] + \
|
||||
average_statistics_new_df['5i5j_esf_price'] + \
|
||||
average_statistics_new_df['maitian_esf_price'] + \
|
||||
average_statistics_new_df['365house_esf_price'] + \
|
||||
average_statistics_new_df['beike_esf_price'] + \
|
||||
average_statistics_new_df['qfang_esf_price'] + \
|
||||
average_statistics_new_df['fangtan_esf_price'] + \
|
||||
average_statistics_new_df['tuboshi_esf_price'] + \
|
||||
average_statistics_new_df['zhugezhaofang_esf_price'] + \
|
||||
average_statistics_new_df['zhongyuan_esf_price'] + \
|
||||
average_statistics_new_df['jiwu_esf_price'] + \
|
||||
average_statistics_new_df['anjuke_esf_price']
|
||||
average_statistics_new_df.loc[:, ['flag']] = average_statistics_new_df['has_esf_price'].apply(
|
||||
lambda x: True if x > 0 else False)
|
||||
average_statistics_new_df = average_statistics_new_df[average_statistics_new_df['flag'] == True]
|
||||
@@ -130,6 +131,6 @@ def deal_stat():
|
||||
|
||||
log.info("开始统计小区数量、近半年第三方均价、近半年成交信息")
|
||||
count_stat()
|
||||
third_avg_stat()
|
||||
# third_avg_stat()
|
||||
deal_stat()
|
||||
log.info("完成统计小区数量、近半年第三方均价、近半年成交信息")
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
统计近三个月、近一年样本量及小区属性
|
||||
"""
|
||||
import logging
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils, commutils
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import sys
|
||||
@@ -24,37 +24,8 @@ seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0
|
||||
.batch_size(10000)))
|
||||
seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'})
|
||||
|
||||
def sell_stat():
|
||||
"""
|
||||
最近一年有样本小区占比 最近3月有样本小区占比
|
||||
"""
|
||||
df = pd.read_excel(aim_file_name)
|
||||
sell_seaweed_df = es206_client.scan_by_sql_to_data_frame(
|
||||
"select city, region, district_name, p_date from latest_seawater where p_date > " + start_time + " and cat='sell' ")
|
||||
# 小区别名升级为单独df
|
||||
actions = list()
|
||||
for index, item in seaweed_df.iterrows():
|
||||
if type(item['alias']) == list:
|
||||
for name in item['alias']:
|
||||
actions.append({'_id': item['_id'], 'city': item['city'], 'name': name, 'region': item['region']})
|
||||
alias_df = pd.DataFrame(actions)
|
||||
|
||||
sell_seaweed_rename_df = sell_seaweed_df[['city', 'region', 'district_name']]
|
||||
sell_seaweed_rename_df = sell_seaweed_rename_df.rename(columns={'district_name': 'name'})
|
||||
sell_seaweed_merge_df = pd.merge(sell_seaweed_rename_df, alias_df, how='left', on=['city', 'region', 'name'])
|
||||
sell_seaweed_merge_df['count'] = sell_seaweed_merge_df[sell_seaweed_merge_df['_id'].notna()].groupby('_id')[
|
||||
'_id'].transform('size')
|
||||
sell_seaweed_merge_df = sell_seaweed_merge_df[['city', 'region', '_id', 'count']]
|
||||
sell_seaweed_merge_df = sell_seaweed_merge_df.drop_duplicates(subset=['city', 'region', '_id'])
|
||||
sell_seaweed_ok_df = sell_seaweed_merge_df[sell_seaweed_merge_df['count'] > sell_num]
|
||||
sell_seaweed_ok_df['city_seaweed_count'] = sell_seaweed_ok_df.groupby('city')['city'].transform('size')
|
||||
city_seaweed_count_df = sell_seaweed_ok_df[['city', 'city_seaweed_count']].drop_duplicates(
|
||||
subset=['city', 'city_seaweed_count'])
|
||||
city_seaweed_count_df = city_seaweed_count_df.rename(columns={'city': '城市', 'city_seaweed_count': '最近3月有样本小区占比'})
|
||||
df = pd.merge(df, city_seaweed_count_df, how='left', on=['城市'])
|
||||
df['最近3月有样本小区占比'] = round(df['最近3月有样本小区占比'] * 100 / df['小区量'], 2)
|
||||
df.to_excel(aim_file_name, index=False)
|
||||
|
||||
def get_sell(df, daysBefore=365):
|
||||
cities = df[['城市']].drop_duplicates(keep='first')['城市'].values.tolist()
|
||||
city_one_year_count = list()
|
||||
for city in cities:
|
||||
@@ -79,8 +50,7 @@ def sell_stat():
|
||||
names_str = "','".join(names)
|
||||
names_str = "'" + names_str + "'"
|
||||
try:
|
||||
sql = "select count(*) from latest_seawater where city='" + city + "' and region='" + region + "' and cat='sell' and district_name in(" + names_str + ") and p_date > " + one_year_time
|
||||
num = es206_client.search_by_sql(sql)['hits']['total']['value']
|
||||
num = commutils.get_sample(city, region, names_str, daysBefore=daysBefore)
|
||||
if num > sell_num:
|
||||
count += 1
|
||||
total_num += num
|
||||
@@ -92,9 +62,25 @@ def sell_stat():
|
||||
print(city + "已处理完, 近一年有样本小区数量" + str(count) + " 近一年的样本总量" + str(total_num))
|
||||
|
||||
result_df = pd.DataFrame(city_one_year_count)
|
||||
return result_df
|
||||
|
||||
|
||||
def sell_stat():
|
||||
"""
|
||||
最近一年有样本小区占比 最近3月有样本小区占比
|
||||
"""
|
||||
df = pd.read_excel(aim_file_name)
|
||||
result_df = get_sell(df, 365)
|
||||
result_df = result_df[['城市', '近一年有样本小区数量', '近一年的样本总量']]
|
||||
df = pd.merge(df, result_df, how='left', on=['城市'])
|
||||
df['最近一年有样本小区占比'] = round(df['近一年有样本小区数量'] * 100 / df['小区量'], 2)
|
||||
df['近一年有样本小区占比'] = round(df['近一年有样本小区数量'] * 100 / df['小区量'], 2)
|
||||
|
||||
result_df = get_sell(df, 93)
|
||||
result_df = result_df.rename(columns={'近一年有样本小区数量': '近三月有样本小区数量'})
|
||||
result_df = result_df[['城市', '近三月有样本小区数量']]
|
||||
df = pd.merge(df, result_df, how='left', on=['城市'])
|
||||
df['近三月有样本小区占比'] = round(df['近三月有样本小区数量'] * 100 / df['小区量'], 2)
|
||||
|
||||
df.to_excel(aim_file_name, index=False)
|
||||
|
||||
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
pip install -r requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/
|
||||
@@ -1 +0,0 @@
|
||||
pip install -r requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/
|
||||
@@ -1,5 +0,0 @@
|
||||
nextcloud-api-wrapper
|
||||
elasticsearch==7.11.0
|
||||
transbigdata==0.4.15
|
||||
pykalman==0.9.5
|
||||
goojia_utils
|
||||
Reference in New Issue
Block a user