This commit is contained in:
2023-03-20 17:44:42 +08:00
70 changed files with 23584 additions and 908 deletions
+2 -1
View File
@@ -7,4 +7,5 @@ pip install -r requirements.txt -i https://repo.fangjia.com/repository/pypi-publ
- 具体业务逻辑放入xxx/business中,如poi/business
- 添加.airflowignore xxx/business/*, 如poi/business/*
- dag放在xxx目录下即可,命名为xx_dag.py
- dag放在xxx目录下即可,命名为dag_xx.py,之前命名为xx_dag导致xx与报名相同而加入airflowignore后dag不显示问题,
如merge_dag会不显示,而dag_merge即可,怀疑是jh/merge/*的bug
+5 -2
View File
@@ -1,6 +1,9 @@
poi/business/.*
poi/business/*
stat/evaluate/*
stat/community/*
mt/supply/*
business/business/*
plugin/**
jh/merge/*
jh/para/*
jh/price/*
db/*
+22 -17
View File
@@ -1,27 +1,32 @@
from goojia_utils import REDIS_CLIENT
import json
import logging
log = logging.getLogger(__name__)
"""
高德key每日定时生成
"""
#amap_key = {"4e9592208336433ee1b7aa4d1d434418", "10bcb421d2ee4e51ed9268d6dca58ad1"}
log.info("测试生成key")
#老板新注册key,限制15万
amap_key = {"6e7b0b0539ff51a714678cd69183fe32"}
service = {'v3/geocode/geo': 150000,
'v3/geocode/regeo': 150000,
'v3/place/text': 150000,
'v5/place/text': 150000,
'v3/place/around': 150000,
'v5/place/around': 150000,
'v3/place/detail': 150000,
'v5/place/detail': 150000}
for service_key in service.keys():
val = dict()
for key in amap_key:
val[key] = service[service_key]
REDIS_CLIENT.hset("amapKey", service_key, json.dumps(val))
# amap_key = {"6e7b0b0539ff51a714678cd69183fe32"}
#
# service = {'v3/geocode/geo': 150000,
# 'v3/geocode/regeo': 150000,
# 'v3/place/text': 150000,
# 'v5/place/text': 150000,
# 'v3/place/around': 150000,
# 'v5/place/around': 150000,
# 'v3/place/detail': 150000,
# 'v5/place/detail': 150000}
#
# for service_key in service.keys():
# val = dict()
# for key in amap_key:
# val[key] = service[service_key]
#
# REDIS_CLIENT.hset("amapKey", service_key, json.dumps(val))
@@ -0,0 +1,70 @@
"""
新房变二手房定时脚本
创建年代、入住/交付时间三年后,或者有样本就变成二手房
"""
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
from pymongo import UpdateOne
import pandas as pd
import logging
import datetime
import time
log = logging.getLogger(__name__)
def do():
log.info("开始执行新房变二手房脚本")
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'is_new': 1},
{'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'open_date': 1, 'live_date': 1,
'create_year': 1}).batch_size(100000)))
if len(df) <= 0:
log.info("执行新房变二手房脚本完毕,未发现新房")
return False
actions = list()
count = 0
def deal(x):
global count
is_old = False
time_now = int(time.mktime(time.strptime(datetime.datetime.now().strftime('%Y-%m-%d'), '%Y-%m-%d')))
if 'open_date' in x and pd.notna(x['open_date']):
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(x['open_date'], years=3), '%Y-%m-%d'), '%Y-%m-%d')))
if time_old < time_now:
is_old = True
if is_old is False and 'live_date' in x and pd.notna(x['live_date']):
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['live_date']), years=3), '%Y-%m-%d'), '%Y-%m-%d')))
if time_old < time_now:
is_old = True
if is_old is False and 'create_year' in x and pd.notna(x['create_year']):
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['create_year'], '%Y-%m'), years=3), '%Y-%m-%d'), '%Y-%m-%d')))
if time_old < time_now:
is_old = True
# 查询样本库
names = [x['name']]
if type(x['alias']) == list:
for alias_item in x['alias']:
names.append(alias_item)
names_str = "','".join(names)
names_str = "'" + names_str + "'"
if is_old is False:
nums = commutils.get_sample(x['city'], x['region'], names_str)
if nums > 0:
is_old = True
if is_old:
actions.append(UpdateOne({'_id': x['_id']}, {'$set': {'is_new': 0, 'm_date': datetime.datetime.now()}}))
if len(actions) == 5000:
seaweed_conn.bulk_write(actions)
count += len(actions)
actions.clear()
df.apply(lambda x: deal(x), axis=1)
if len(actions) > 0:
seaweed_conn.bulk_write(actions)
count += len(actions)
actions.clear()
log.info(f"执行新房变二手房脚本完毕,共更新{count}")
do()
+80 -72
View File
@@ -3,45 +3,70 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, ClickHouseClient
from elasticsearch import helpers
import numpy as np
from concurrent.futures import ThreadPoolExecutor
import sys
from threading import Lock
mutex = Lock()
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
m_date = dateutils.iso_format()
path = "/home/airflow/data/community"
file_path = "/opt/airflow/dags/repo/dags/business/business/community/data"
avg_cost_min_max_df = pd.read_excel(file_path + '/全国各区域人均消费中位值.xlsx')
avg_cost_city_region_dict = dict()
def find_around_price(lng, lat):
# 加载各区域人均消费最大最小值
def deal_avg_cost(x):
city_now = x['city']
region_now = x['region']
if pd.notna(city_now) and pd.notna(region_now):
avg_cost_city_region_dict[city_now + region_now] = {'min': x['3'], 'max': x['97']}
elif pd.notna(city_now):
avg_cost_city_region_dict[city_now] = {'min': x['3'], 'max': x['97']}
avg_cost_min_max_df.apply(lambda x: deal_avg_cost(x), axis=1)
def find_around_price(seaweed):
"""
查找周边均价
返回取值为10~300,为所有有价格的1-99百分位
"""
if pd.isna(lng):
return None
lng = commutils.format_lat_lng(seaweed['lng'])
lat = commutils.format_lat_lng(seaweed['lat'])
city_short = seaweed['city']
region_short = seaweed['region']
city_display = areautils.get_city_display(city_short)
filter_query = [{"terms": {"tag": ["美食;中餐厅", "美食;外国餐厅"]}}]
if pd.notna(city_display):
filter_query.append({"term": {"city": {"value": city_display}}})
min_now = 10
max_now = 300
if (city_short+region_short) in avg_cost_city_region_dict:
min_now = avg_cost_city_region_dict[city_short + region_short]['min']
max_now = avg_cost_city_region_dict[city_short + region_short]['max']
elif city_short in avg_cost_city_region_dict:
min_now = avg_cost_city_region_dict[city_short]['min']
max_now = avg_cost_city_region_dict[city_short]['max']
filter_query.append({"range": {"price": {"gte": min_now, "lte": max_now}}})
filter_query.append({"geo_distance": {"distance": 1000, "location": [lng, lat]}})
ret_es = es206_client.elastic_client.search({
"query": {
"bool": {
"must": [
{
"term": {
"status": {
"value": 0
}
}
}
],
"filter": [
{
"geo_distance": {
"distance": 1000,
"location": [lng, lat]
}
}
]
"filter": filter_query
}
},
"size": 0,
@@ -58,69 +83,51 @@ def find_around_price(lng, lat):
def build_doc(item):
doc_dict = dict()
if 'avg_cost' in item and pd.notna(item['avg_cost']):
doc_dict['avg_cost'] = item['avg_cost']
if 'avg_cost' in item:
if pd.notna(item['avg_cost']) or item['avg_cost'] > 0:
doc_dict['avg_cost'] = int(item['avg_cost'])
else:
doc_dict['avg_cost'] = None
return doc_dict
def update_status(new_df):
actions = list()
find_count = 0
def deal(item):
action_doc = {
"_op_type": "update",
"_index": "bd_poi",
"_id": item['_id'],
"doc": {'status': 0}
}
actions.append(action_doc)
if len(actions) == 5000:
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
return item
new_df.apply(lambda x: deal(x), axis=1)
if len(actions) > 0:
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
def find_around_price_item(seaweed):
global find_count
find_count += 1
if find_count % 1000 == 0:
log.info("已查找" + str(find_count))
if 'lng' not in seaweed:
return
price = find_around_price(seaweed)
mutex.acquire(timeout=60)
if pd.notna(price):
seaweed['avg_cost'] = int(price)
else:
seaweed['avg_cost'] = None
mutex.release()
def sync(city=None):
log.info("开始同步消费数据...")
query_sql = "select id, city, tag, type, price from bd_poi where price > 0 and tag in('美食;中餐厅', '美食;外国餐厅') "
tmp_sql = f"select * from community_summary "
if pd.notna(city):
query_sql += " and city='" + areautils.get_city_display(city) + "'"
df = es206_client.scan_by_sql_to_data_frame(query_sql)
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
# 按百分位5-95筛选数据
cities = df['city'].unique()
for city in cities:
df.loc[df['city'] == city, 'minPrice5'] = np.percentile(a=df[df['city'] == city]['price'], q=5, axis=0)
df.loc[df['city'] == city, 'minPrice95'] = np.percentile(a=df[df['city'] == city]['price'], q=95, axis=0)
seaweed_list = list()
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
new_df = df[(df['price'] >= df['minPrice5']) & (df['price'] <= df['minPrice95'])]
new_df['new_mean'] = new_df.groupby(by=['city'])['price'].transform(np.mean)
with ThreadPoolExecutor(max_workers=30) as pool:
for seaweed in seaweed_list:
pool.submit(find_around_price_item, seaweed)
# 按最新的百分位更新库中价格状态
update_status(new_df)
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
def find_around_price_item(index, item):
if 'lng' not in item:
return
lng = commutils.format_lat_lng(item['lng'])
lat = commutils.format_lat_lng(item['lat'])
price = find_around_price(lng, lat)
seaweed_df.loc[index, 'avg_cost'] = price
with ThreadPoolExecutor(max_workers=50) as pool:
for index, item in seaweed_df.iterrows():
pool.submit(find_around_price_item, index, item)
# for seaweed in seaweed_list:
# find_around_price_item(seaweed)
seaweed_df = pd.DataFrame(seaweed_list)
actions = list()
count = 0
for index, item in seaweed_df.iterrows():
@@ -147,4 +154,5 @@ def sync(city=None):
log.info("完成同步消费数据...")
sync()
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
@@ -3,13 +3,15 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database="tmp")
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -88,10 +90,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步商圈至206 {index_name}")
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -104,4 +106,5 @@ def sync(city=None):
log.info(f"完成同步商圈至206 {index_name}")
sync()
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
+35 -7
View File
@@ -1,16 +1,17 @@
"""
同步小区基本信息至ES206
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient
from elasticsearch import helpers
import os
import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
m_date = dateutils.iso_format()
@@ -47,6 +48,8 @@ def build_doc(item):
estate_charge = commutils.format_property_fee(item['estate_charge'])
if pd.notna(estate_charge):
doc_dict['estate_charge'] = float(estate_charge)
# 表示该物业费已经从库中补充
doc_dict['estate_charge_flag'] = 0
finish_building_date = commutils.format_finish_building_date(item['finish_building_date'])
if pd.notna(finish_building_date):
@@ -83,8 +86,11 @@ def deal_comm(community_summary_df, seaweed_df):
def sync(city=None):
log.info(f"开始同步小区基本属性至206 {index_name}...")
seaweed_query = {'status': 0, 'cat': 'district'}
ck_sql = "ALTER TABLE tmp.community_summary DELETE WHERE 1=1 "
if pd.notna(city):
seaweed_query['city'] = city
ck_sql += " AND city='" + city + "'"
tmp_ck.execute(ck_sql)
seaweed_df = pd.DataFrame(
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1,
@@ -93,8 +99,20 @@ def sync(city=None):
'estate_charge': 1, 'finish_building_date': 1})
.batch_size(100000)))
community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}")
sql = f"select id from {index_name} "
if pd.notna(city):
sql += f" where city='{city}'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
# 清除别名中逗号
def dealAlias(x):
if isinstance(x, list) and len(x) > 0:
for index, element in enumerate(x):
if "'" in element:
x[index] = str(element).replace("'", "")
return x
seaweed_df['alias'] = seaweed_df['alias'].apply(lambda x: dealAlias(x))
if len(community_summary_df) > 0:
community_summary_df['flag'] = 1
@@ -105,6 +123,11 @@ def sync(city=None):
seaweed_existed_df = pd.DataFrame()
seaweed_not_existed_df = seaweed_df
# 写入csv文件,防止小区变动导致的后续过程更新报错,
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
# 写入中间表clickhouse,解决多worker节点问题
tmp_ck.to_sql_plus(seaweed_df, 'community_summary')
actions = list()
# 对于存在的更新
count = 0
@@ -124,6 +147,11 @@ def sync(city=None):
count += len(actions)
log.info("更新:" + str(count))
actions.clear()
if len(actions) > 0:
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
log.info("更新:" + str(count))
actions.clear()
# 对于不存在的插入
count = 0
@@ -145,13 +173,13 @@ def sync(city=None):
actions.clear()
if len(actions) > 0:
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
log.info("插入:" + str(count))
actions.clear()
# 写入csv文件,防止小区变动导致的后续过程更新报错
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
deal_comm(community_summary_df, seaweed_df)
log.info(f"完成同步小区基本属性至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+196 -67
View File
@@ -1,55 +1,71 @@
"""
月消费潜力、评级计算
"""
from goojia_utils import ElasticSearchClient
from goojia_utils import ElasticSearchClient, dateutils, nextcloudutils
import pandas as pd
import numpy as np
import logging
import os
from elasticsearch import helpers
import matplotlib.pyplot as plt
import sys
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import QuantileTransformer
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
save_path_prefix = "/home/airflow/data/community"
save_png_path = save_path_prefix + "/consumption_rating"
if not os.path.exists(save_png_path):
os.makedirs(save_png_path)
nextcloud_aim_file_path = '共享/财力评级/财力评级图'
reasonable_df = pd.read_excel(path + '/财力评级分布情况20221215.xlsx')
reasonable_dict = reasonable_df.set_index('city')['是否合理'].to_dict()
factor_df = pd.read_excel(path + '/财力评级极值设置.xlsx')
factor_dict = dict()
for index, item in factor_df.iterrows():
if pd.isna(item['min_total_price']):
continue
factor_dict[item['city']] = {'min_total_price': item['min_total_price'], 'max_total_price': item['max_total_price'], 'min_estate_charge': item['min_estate_charge'], 'max_estate_charge': item['max_estate_charge'], \
'min_avg_cost': item['min_avg_cost'], 'max_avg_cost': item['max_avg_cost']}
def calc_consumption_potential(city):
"""
计算消费潜力
"""
sql = "select id, city, region, name, total_price, estate_charge, relative_price_ranking, avg_cost from community_summary"
if pd.notna(city):
sql += " where city='" + city + "'"
sql = "select id, city, region, name, total_price, estate_charge, estate_charge_fix, relative_price_ranking, avg_cost from community_summary where city='" + city + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
# 户均总价计算时候改成万
### 极值设置
if city in factor_dict:
community_summary_df.loc[community_summary_df['total_price'] < factor_dict[city]['min_total_price'], 'total_price'] = factor_dict[city]['min_total_price']
community_summary_df.loc[community_summary_df['total_price'] > factor_dict[city]['max_total_price'], 'total_price'] = factor_dict[city]['max_total_price']
community_summary_df.loc[community_summary_df['estate_charge'] < factor_dict[city]['min_estate_charge'], 'estate_charge'] = factor_dict[city]['min_estate_charge']
community_summary_df.loc[community_summary_df['estate_charge'] > factor_dict[city]['max_estate_charge'], 'estate_charge'] = factor_dict[city]['max_estate_charge']
community_summary_df.loc[community_summary_df['avg_cost'] < factor_dict[city]['min_avg_cost'], 'avg_cost'] = factor_dict[city]['min_avg_cost']
community_summary_df.loc[community_summary_df['avg_cost'] > factor_dict[city]['max_avg_cost'], 'avg_cost'] = factor_dict[city]['max_avg_cost']
columns = community_summary_df.columns
if 'total_price' not in columns or 'estate_charge' not in columns or 'relative_price_ranking' not in columns or 'avg_cost' not in columns:
return []
community_summary_df = community_summary_df.dropna(how='any').copy()
if len(community_summary_df) == 0:
log.warning(f"{city}无法评级,缺少因子")
return []
community_summary_df['total_price'] = community_summary_df['total_price'] / 10000
# 户均总价计算时候改成万
community_summary_df['total_price'] = round(community_summary_df['total_price'] / 10000, 0)
community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform(
np.mean)
community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.std)
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(
# 噪音词处理
community_summary_df['avg_cost_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform(
community_summary_df[['avg_cost']])
# 解决某些城市无修正物业费问题
if 'estate_charge_fix' not in community_summary_df.columns:
community_summary_df['estate_charge_fix'] = None
# 修正的物业费入模
community_summary_df['estate_charge_fix'].fillna(community_summary_df['estate_charge'], inplace=True)
community_summary_df['estate_charge_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform(
community_summary_df[['estate_charge_fix']])
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge_fixed'].transform(
np.mean)
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge_fixed'].transform(
np.std)
community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])[
@@ -57,22 +73,29 @@ def calc_consumption_potential(city):
community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])[
'relative_price_ranking'].transform(np.std)
community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.mean)
community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.std)
community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost_fixed'].transform(np.mean)
community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost_fixed'].transform(np.std)
community_summary_df['total_price_score'] = 286.0134196 * (
community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df[
'total_price_std']
# 和田等小城市均值可能为0
if community_summary_df['total_price_std'].unique()[0] != 0:
community_summary_df['total_price_score'] = 286.0134196 * (community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df['total_price_std']
else:
community_summary_df['total_price_score'] = 0
community_summary_df['estate_charge_score'] = 139.1374524 * (
community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std']
if community_summary_df['estate_charge_std'].unique()[0] != 0:
community_summary_df['estate_charge_score'] = 139.1374524 * (community_summary_df['estate_charge_fixed'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std']
else:
community_summary_df['estate_charge_score'] = 0
community_summary_df['relative_price_ranking_score'] = 21.8427926 * (
community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / community_summary_df['relative_price_ranking_std']
if community_summary_df['relative_price_ranking_std'].unique()[0] != 0:
community_summary_df['relative_price_ranking_score'] = 21.8427926 * (community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / community_summary_df['relative_price_ranking_std']
else:
community_summary_df['relative_price_ranking_score'] = 0
community_summary_df['avg_cost_score'] = 67.0195871700 * (
community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df[
'avg_cost_std']
if community_summary_df['avg_cost_std'].unique()[0] != 0:
community_summary_df['avg_cost_score'] = 67.0195871700 * (community_summary_df['avg_cost_fixed'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std']
else:
community_summary_df['avg_cost_score'] = 0
community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df[
'estate_charge_score'] + community_summary_df['relative_price_ranking_score'] + community_summary_df['avg_cost_score'] + community_summary_df['avg_cost_score'] + 4099.978248
@@ -81,10 +104,10 @@ def calc_consumption_potential(city):
def gmm_train(total_rate, consumption_df, source, target):
# 高斯混合模型
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]])
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense').astype(int)
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
return consumption_df
@@ -93,47 +116,153 @@ def calc_consumption_rating(community_summary_df):
"""
计算消费等级
"""
consumption_aim_df = community_summary_df[['_id', 'city', 'consumption_potential']]
consumption_aim_df = consumption_aim_df.dropna(subset=['consumption_potential'])
cities = consumption_aim_df['city'].unique()
source = "consumption_potential"
df = community_summary_df[['_id', 'city', 'consumption_potential']]
df = df.dropna(subset=['consumption_potential'])
source = "normal_consumption_potential"
target = "consumption_rating"
actions = list()
count = 0
for city in cities:
df = consumption_aim_df[consumption_aim_df['city'] == city].copy()
if len(df) < 10:
continue
df = gmm_train(10, df, source, target)
# 批量更新
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['consumption_rating'] = item['consumption_rating']
row['consumption_potential'] = item['consumption_potential']
if len(df) < 5:
# 对于样本低于5的直接排名
df[target] = df['consumption_potential'].rank(method='dense', ascending=False).astype(int)
else:
# 极值设置
df = filter_min_max(df)
df = gmm_train(5, df, source, target)
# 批量更新
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['consumption_rating'] = item['consumption_rating']
row['consumption_potential'] = item['consumption_potential']
row['m_date'] = dateutils.iso_format()
action = {'_op_type': 'update', # 操作 index update create delete
'_index': index_name, # index
'_id': _id, # 请勿忘记ID
'doc': row,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级中: 当前已评级{}".format(count))
action = {'_op_type': 'update', # 操作 index update create delete
'_index': index_name, # index
'_id': _id, # 请勿忘记ID
'doc': row,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级中: 当前已评级{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级完成,共更新: {}".format(count))
return pd.merge(community_summary_df, df, how='left', on=['_id', 'city', 'consumption_potential'])
def sync_by_city(city_now):
log.info(f"{city_now}开始评级....")
community_summary_df = calc_consumption_potential(city_now)
if len(community_summary_df) == 0:
log.warning(f"{city_now}无法评级,缺少因子")
return
community_summary_df = calc_consumption_rating(community_summary_df)
if len(community_summary_df) == 0:
log.warning(f"{city_now}无法评级,数量低于5")
return
save_png(community_summary_df)
nextcloudutils.up_to_nextcloud(save_png_path + "/" + city_now + ".png", nextcloud_aim_file_path + "/" + city_now + ".png")
def sync(city=None):
community_summary_df = calc_consumption_potential(city)
calc_consumption_rating(community_summary_df)
if pd.isna(city):
for city_now in reasonable_df['city'].unique():
#if pd.isna(reasonable_dict[city_now]):
sync_by_city(city_now)
else:
sync_by_city(city)
sync()
def filter_min_max(community_summary_df):
community_summary_df['normal_consumption_potential'] = QuantileTransformer(output_distribution="normal").fit_transform(community_summary_df[['consumption_potential']])
# total_price5 = np.percentile(a=community_summary_df['total_price'], q=2, axis=0)
# total_price95 = np.percentile(a=community_summary_df['total_price'], q=98, axis=0)
# community_summary_df.loc[community_summary_df['total_price'] > total_price95, 'total_price'] = total_price95
# community_summary_df.loc[community_summary_df['total_price'] < total_price5, 'total_price'] = total_price5
#
# estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=2, axis=0)
# estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=98, axis=0)
# community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95
# community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5
#
# avg_cost5 = np.percentile(a=community_summary_df['avg_cost'], q=2, axis=0)
# avg_cost95 = np.percentile(a=community_summary_df['avg_cost'], q=98, axis=0)
# community_summary_df.loc[community_summary_df['avg_cost'] > avg_cost95, 'avg_cost'] = avg_cost95
# community_summary_df.loc[community_summary_df['avg_cost'] < avg_cost5, 'avg_cost'] = avg_cost5
#if city in factor_dict:
# 户均总价按极值设置可靠性不高,当户均总价补充后,图中大于最大值的量非常多
# community_summary_df.loc[
# community_summary_df['total_price'] < factor_dict[city]['min_total_price'], 'total_price'] = \
# factor_dict[city]['min_total_price']
# community_summary_df.loc[
# community_summary_df['total_price'] > factor_dict[city]['max_total_price'], 'total_price'] = \
# factor_dict[city]['max_total_price']
# community_summary_df.loc[
# community_summary_df['estate_charge'] < factor_dict[city]['min_estate_charge'], 'estate_charge'] = \
# factor_dict[city]['min_estate_charge']
# community_summary_df.loc[
# community_summary_df['estate_charge'] > factor_dict[city]['max_estate_charge'], 'estate_charge'] = \
# factor_dict[city]['max_estate_charge']
# community_summary_df.loc[community_summary_df['avg_cost'] < factor_dict[city]['min_avg_cost'], 'avg_cost'] = \
# factor_dict[city]['min_avg_cost']
# community_summary_df.loc[community_summary_df['avg_cost'] > factor_dict[city]['max_avg_cost'], 'avg_cost'] = \
# factor_dict[city]['max_avg_cost']
return community_summary_df
def save_png(community_city):
"""
生成财力评级图表
"""
curr_city = community_city['city'].unique()[0]
# 解决中文显示问题
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
plt.title(curr_city)
plt.subplot(3, 1, 1)
community_city['consumption_rating'].hist(bins=100)
plt.xlabel('consumption_rating')
plt.tight_layout()
plt.subplot(3, 2, 3)
community_city['total_price'].hist(bins=100)
plt.xlabel('total_price')
plt.tight_layout()
plt.subplot(3, 2, 4)
community_city['estate_charge'].hist(bins=100)
plt.xlabel('estate_charge')
plt.tight_layout()
plt.subplot(3, 2, 5)
community_city['avg_cost'].hist(bins=100)
plt.xlabel('avg_cost')
plt.tight_layout()
plt.subplot(3, 2, 6)
community_city['relative_price_ranking'].hist(bins=100)
plt.xlabel('relative_price_ranking')
plt.tight_layout()
plt.savefig(save_png_path + "/" + curr_city + ".png")
# 清图
plt.clf()
# 清坐标轴
plt.cla()
# 关窗口
plt.close()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
#dingutils.send_text("【分布图生成完毕】财力评级已完成, 请点击 https://nextcloud.fangjia.com/s/Jqs5BzXi8xLYeH9 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
File diff suppressed because it is too large Load Diff
@@ -3,13 +3,15 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database="tmp")
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -62,11 +64,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}")
seaweed_query = {'status': 0, 'cat': 'district'}
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -83,4 +84,5 @@ def sync(city=None):
log.info(f"完成同步教育资源至206 {index_name}")
sync()
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
@@ -3,13 +3,15 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database="tmp")
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -75,10 +77,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步教育资源【小学、中学】至206 {index_name}")
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -95,4 +97,5 @@ def sync(city=None):
log.info(f"完成同步教育资源至206 {index_name}")
sync()
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
@@ -0,0 +1,317 @@
"""
物业费补全
"""
from goojia_utils import ElasticSearchClient, areautils, dateutils
import pandas as pd
import logging
import sys
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
file_path = "/opt/airflow/dags/repo/dags/business/business/community/data"
mutex = Lock()
estate_charge_min_max_df = pd.read_excel(file_path + '/全国各区域物业费中位值.xlsx')
estate_charge_city_region_dict = dict()
# 加载各区域物业费最大最小值
def deal_estate_charge(x):
city_now = x['city']
region_now = x['region']
if pd.notna(city_now) and pd.notna(region_now):
estate_charge_city_region_dict[city_now + region_now] = {'min': x['3'], 'max': x['97']}
elif pd.notna(city_now):
estate_charge_city_region_dict[city_now] = {'min': x['3'], 'max': x['97']}
estate_charge_min_max_df.apply(lambda x: deal_estate_charge(x), axis=1)
def save_fix(community_summary_df):
if 'estate_charge_fix' not in community_summary_df.columns:
log.info("该城市无调整物业费")
return
df = community_summary_df[['_id', 'city', 'estate_charge_fix']]
df = df.dropna(subset=['estate_charge_fix'])
actions = list()
count = 0
# 批量更新
m_date = dateutils.iso_format()
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['estate_charge_fix'] = item['estate_charge_fix']
row['m_date'] = m_date
action = {'_op_type': 'update',
'_index': index_name,
'_id': _id, # 请勿忘记ID
'doc': row,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("已更新{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("更新完成,共更新: {}".format(count))
return df
def save_fill(community_summary_df):
if 'estate_charge' not in community_summary_df.columns:
log.info("该城市无补充物业费")
return
df = community_summary_df[['_id', 'city', 'estate_charge']]
df = df.dropna(subset=['estate_charge'])
actions = list()
count = 0
# 批量更新
m_date = dateutils.iso_format()
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['estate_charge'] = item['estate_charge']
# 1 表示为物业费补充的,在实际应用中不输出
row['estate_charge_flag'] = 1
row['m_date'] = m_date
action = {'_op_type': 'update',
'_index': index_name,
'_id': _id, # 请勿忘记ID
'doc': row,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("已更新{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("更新完成,共更新: {}".format(count))
return df
def find_nearest_by_location(city, location):
docs = list()
try:
scan = es206_client.scan(index_name, {
"size": 4,
"_source": ["finish_building_date", "estate_charge", "estate_charge_fix"],
"query": {
"bool": {
"filter": [
{
"term": {
"city": city
}
}
],
"must_not": [
{
"term": {
"estate_charge_flag": {
"value": 1
}
}
}
]
}
},
"sort": [
{
"_geo_distance": {
"location": location,
"order": "asc"
}
}
]
}, preserve_order=True)
ignored_first = False
for doc in scan:
# 忽略自己
if ignored_first is False:
ignored_first = True
continue
docs.append(doc['_source'])
if len(docs) == 3:
return pd.DataFrame(docs)
return None
except Exception as e:
logging.error(e)
return None
def get_estate_charge_for_adjust(finish_building_date, docs):
aim_year = int(finish_building_date / 10)
for index, item in docs.iterrows():
if pd.notna(item['finish_building_date']) and pd.notna(item['estate_charge']) and aim_year == int(item['finish_building_date'] / 10):
return item['estate_charge']
return None
def get_estate_charge(finish_building_date, docs):
aim_year = int(finish_building_date / 10)
for index, item in docs.iterrows():
if pd.notna(item['finish_building_date']) and pd.notna(item['estate_charge_fix']) and aim_year == int(item['finish_building_date'] / 10):
return item['estate_charge_fix']
return None
find_count = 0
def deal_adjust(community):
global find_count
find_count += 1
if find_count % 1000 == 0:
log.info("已查找" + str(find_count))
max_now = 7
city_short = community['city']
region_short = community['region']
if (city_short + region_short) in estate_charge_city_region_dict:
max_now = estate_charge_city_region_dict[city_short + region_short]['max']
elif city_short in estate_charge_city_region_dict:
max_now = estate_charge_city_region_dict[city_short]['max']
if community['estate_charge'] > max_now:
estate_charge = find_estate_charge_for_adjust(community, max_now)
if pd.notna(estate_charge):
mutex.acquire(timeout=60)
community['estate_charge_fix'] = estate_charge
mutex.release()
def adjust_estate_charge(city_now):
log.info(f"{city_now}开始调整物业费....")
sql = f"select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='{city_now}' and estate_charge_flag != 1 and estate_type != '别墅' "
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
community_summary_list = list()
community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1)
with ThreadPoolExecutor(max_workers=30) as pool:
for community_summary in community_summary_list:
pool.submit(deal_adjust, community_summary)
# for community_summary in community_summary_list:
# deal_adjust(community_summary)
community_summary_df = pd.DataFrame(community_summary_list)
save_fix(community_summary_df)
log.info(f"{city_now}完成调整物业费....")
def find_estate_charge_for_adjust(community, max_now):
"""
查找小区周边物业费
"""
estate_charge = None
if pd.isna(community['location']) or pd.isna(community['location']['lon']):
return estate_charge
location = [community['location']['lon'], community['location']['lat']]
docs = find_nearest_by_location(community['city'], location)
if len(docs) <= 0 or 'estate_charge' not in docs.columns:
return estate_charge
if pd.notna(community['finish_building_date']) and 'finish_building_date' in docs.columns:
# 根据相近建筑年代赋值
estate_charge = get_estate_charge_for_adjust(community['finish_building_date'], docs)
if pd.isna(estate_charge) or estate_charge > max_now:
# 取周边均值
estate_charge = docs['estate_charge'].mean()
# 如果查找的还大于分位值,不调整
if estate_charge > max_now:
estate_charge = None
return estate_charge
def find_estate_charge(community, max_now):
"""
查找小区周边物业费
"""
estate_charge = None
if pd.isna(community['location']) or pd.isna(community['location']['lon']):
return estate_charge
location = [community['location']['lon'], community['location']['lat']]
docs = find_nearest_by_location(community['city'], location)
if len(docs) <= 0 or 'estate_charge' not in docs.columns:
return estate_charge
if 'estate_charge_fix' not in docs.columns:
docs['estate_charge_fix'] = None
# 用修正后的物业费补充
docs['estate_charge_fix'].fillna(docs['estate_charge'], inplace=True)
if pd.notna(community['finish_building_date']) and 'finish_building_date' in docs.columns:
# 根据相近建筑年代赋值
estate_charge = get_estate_charge(community['finish_building_date'], docs)
if pd.isna(estate_charge) or estate_charge > max_now:
# 取周边均值
estate_charge = docs['estate_charge_fix'].mean()
if estate_charge > max_now:
estate_charge = None
return estate_charge
find_count = 0
def deal_fill(community):
global find_count
find_count += 1
if find_count % 1000 == 0:
log.info("已查找" + str(find_count))
max_now = 7
city_short = community['city']
region_short = community['region']
if (city_short + region_short) in estate_charge_city_region_dict:
max_now = estate_charge_city_region_dict[city_short + region_short]['max']
elif city_short in estate_charge_city_region_dict:
max_now = estate_charge_city_region_dict[city_short]['max']
if community['estate_type'] == '别墅':
# 别墅不限制物业费最大值
max_now = 99
estate_charge = find_estate_charge(community, max_now)
if pd.notna(estate_charge):
mutex.acquire(timeout=60)
community["estate_charge"] = estate_charge
mutex.release()
def fill_estate_charge(city_now):
log.info(f"{city_now}开始补充物业费....")
sql = "select id, city, region, name, estate_charge, estate_charge_fix, finish_building_date, estate_type, location from community_summary where city='" + city_now + "' and (estate_charge is null or estate_charge_flag=1)"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
community_summary_list = list()
community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1)
with ThreadPoolExecutor(max_workers=30) as pool:
for community_summary in community_summary_list:
pool.submit(deal_fill, community_summary)
# for community_summary in community_summary_list:
# deal_fill(community_summary)
community_summary_df = pd.DataFrame(community_summary_list)
save_fill(community_summary_df)
log.info(f"{city_now}完成补充物业费....")
def sync_by_city(city_now):
adjust_estate_charge(city_now)
fill_estate_charge(city_now)
def sync(city=None):
if pd.isna(city):
for city_now in areautils.get_all_cities():
sync_by_city(city_now)
else:
sync_by_city(city)
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -0,0 +1,169 @@
"""
物业费等级计算
"""
from goojia_utils import ElasticSearchClient, nextcloudutils
import pandas as pd
import logging
import os
import sys
from elasticsearch import helpers
import matplotlib.pyplot as plt
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import QuantileTransformer
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
nextcloud_aim_file_path = '共享/财力评级/物业费等级图'
save_path_prefix = "/home/airflow/data/community"
save_png_path = save_path_prefix + "/estate_charge_ranking"
if not os.path.exists(save_png_path):
os.makedirs(save_png_path)
factor_df = pd.read_excel(path + '/财力评级极值设置.xlsx')
factor_dict = dict()
for index, item in factor_df.iterrows():
if pd.isna(item['min_total_price']):
continue
factor_dict[item['city']] = {'min_total_price': item['min_total_price'], 'max_total_price': item['max_total_price'], 'min_estate_charge': item['min_estate_charge'], 'max_estate_charge': item['max_estate_charge'], \
'min_avg_cost': item['min_avg_cost'], 'max_avg_cost': item['max_avg_cost']}
reasonable_df = pd.read_excel(path + '/财力评级物业费等级情况20221213.xlsx')
reasonable_dict = reasonable_df.set_index('city')['是否合理'].to_dict()
def gmm_train(total_rate, consumption_df, source, target):
# 高斯混合模型
city_info = consumption_df.loc[0, 'city']
if city_info in reasonable_dict and reasonable_dict[city_info] == '不合理':
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]])
else:
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
consumption_df['cluster'] = gmm.predict(consumption_df[[source]])
consumption_df['price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df[target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
return consumption_df
def filter_min_max(community_summary_df):
"""
噪音词处理
"""
community_summary_df['QuantileTransformer.normal'] = QuantileTransformer(output_distribution="normal").fit_transform(community_summary_df[['estate_charge']])
# estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=1, axis=0)
# estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=99, axis=0)
# community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95
# community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5
# if city in factor_dict:
# community_summary_df.loc[
# community_summary_df['estate_charge'] < factor_dict[city]['min_estate_charge'], 'estate_charge'] = \
# factor_dict[city]['min_estate_charge']
# community_summary_df.loc[
# community_summary_df['estate_charge'] > factor_dict[city]['max_estate_charge'], 'estate_charge'] = \
# factor_dict[city]['max_estate_charge']
return community_summary_df
def calc_estate_charge_rating(community_summary_df):
"""
计算物业费等级
"""
df = community_summary_df[['_id', 'city', 'QuantileTransformer.normal', 'estate_charge']].copy()
source = "QuantileTransformer.normal"
target = "estate_charge_ranking"
actions = list()
count = 0
if len(df) < 5:
# 对于样本低于5的直接排名
df[target] = df['estate_charge'].rank(method='dense', ascending=False).astype(int)
else:
df = gmm_train(5, df, source, target)
# 批量更新
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['estate_charge_ranking'] = item['estate_charge_ranking']
action = {'_op_type': 'update', # 操作 index update create delete
'_index': index_name, # index
'_id': _id, # 请勿忘记ID
'doc': row,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级中: 当前已评级{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级完成,共更新: {}".format(count))
return df
def sync_by_city(city_now):
log.info(f"{city_now}开始评级....")
sql = "select id, city, region, name, estate_charge from community_summary where city='" + city_now + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
columns = community_summary_df.columns
if 'estate_charge' not in columns:
log.warning(f"{city_now}无法评级,缺少因子")
return
community_summary_df = community_summary_df.dropna(how='any').reset_index(drop=True).copy()
# 噪音词处理
community_summary_df = filter_min_max(community_summary_df)
community_summary_df = calc_estate_charge_rating(community_summary_df)
if len(community_summary_df) == 0:
log.warning(f"{city_now}无法评级,数量低于5")
return
save_png(community_summary_df)
nextcloudutils.up_to_nextcloud(save_png_path + "/" + city_now + ".png", nextcloud_aim_file_path + "/" + city_now + ".png")
def sync(city=None):
if pd.isna(city):
for city_now in reasonable_df['city'].unique():
#if reasonable_dict[city_now] == '不合理':
sync_by_city(city_now)
else:
sync_by_city(city)
def save_png(community_city):
"""
生成财力评级图表
"""
plt.subplot(2, 1, 1)
community_city['estate_charge_ranking'].hist(bins=100)
plt.xlabel('estate_charge_ranking')
plt.tight_layout()
plt.subplot(2, 1, 2)
community_city['estate_charge'].hist(bins=100)
plt.xlabel('estate_charge')
plt.tight_layout()
plt.savefig(save_png_path + "/" + community_city['city'].unique()[0] + ".png")
# 清图
plt.clf()
# 清坐标轴
plt.cla()
# 关窗口
plt.close()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
#dingutils.send_text("【分布图生成完毕】物业费评级已完成, 请点击 https://nextcloud.fangjia.com/s/HbifLoEDc5wi8HR 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
+3 -1
View File
@@ -14,6 +14,7 @@ import pandas as pd
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -140,4 +141,5 @@ def sync(city=None):
log.info(f"完成同步附加标签至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+8 -5
View File
@@ -3,13 +3,15 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -76,10 +78,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步生活配套至206 {index_name}")
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -96,4 +98,5 @@ def sync(city=None):
log.info(f"完成同步生活配套至206 {index_name}")
sync()
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
+8 -6
View File
@@ -3,14 +3,16 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient, poiutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -62,11 +64,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步医院至206 {index_name}")
seaweed_query = {'status': 0, 'cat': 'district'}
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -83,4 +84,5 @@ def sync(city=None):
log.info(f"完成同步医院至206 {index_name}")
sync()
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
+7 -43
View File
@@ -5,7 +5,7 @@ from goojia_utils import ElasticSearchClient, dateutils
import logging
import pandas as pd
from elasticsearch import helpers
import datetime
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -16,46 +16,9 @@ m_date = dateutils.iso_format()
count = 0
def get_old_or_new(finish_building_date):
"""
或许新旧
"""
if pd.isna(finish_building_date):
return "旧"
now_year = datetime.datetime.now().year
if now_year - int(finish_building_date) < 3:
return "新"
return "旧"
def deal(actions, x):
global count
estate_charge_ranking = None
if pd.notna(x['estate_charge']):
estate_charge = float(x['estate_charge'])
if estate_charge >= 1:
estate_charge_ranking = "一级"
elif estate_charge >= 0.75:
estate_charge_ranking = "二级"
elif estate_charge >= 0.50:
estate_charge_ranking = "三级"
elif estate_charge >= 0.35:
estate_charge_ranking = "四级"
elif estate_charge < 0.35:
estate_charge_ranking = "五级"
else:
old_or_new = get_old_or_new(x['finish_building_date'])
if old_or_new == '新':
if pd.notna(x['build_type']) and '高层' in x['build_type']:
estate_charge_ranking = '三级'
else:
estate_charge_ranking = '四级'
elif old_or_new == '旧':
estate_charge_ranking = '五级'
doc_info = dict()
if pd.notna(estate_charge_ranking):
doc_info['estate_charge_ranking'] = estate_charge_ranking
if pd.notna(x['avg_price']) and pd.notna(x['rent_price']):
doc_info['sell_to_rent'] = int(x['avg_price'] / x['rent_price'])
if pd.notna(x['household_count']):
@@ -72,14 +35,14 @@ def deal(actions, x):
if len(actions) == 5000:
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
log.info(f"物业费等级、售租比、人口数已更新{count}")
log.info(f"售租比、人口数已更新{count}")
actions.clear()
def sync(city=None):
log.info(f"开始同步物业费等级、售租比、人口数至206 {index_name}")
log.info(f"开始同步售租比、人口数至206 {index_name}")
sql = "select city, region, name, estate_charge, finish_building_date, build_type, avg_price, rent_price, household_count from community_summary"
sql = "select city, region, name, finish_building_date, build_type, avg_price, rent_price, household_count from community_summary"
if pd.notna(city):
sql += " where city='" + city + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
@@ -91,7 +54,8 @@ def sync(city=None):
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
log.info(f"完成同步物业费等级、售租比、人口数至206 {index_name}")
log.info(f"完成同步售租比、人口数至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -0,0 +1,79 @@
"""
人口评级
"""
from goojia_utils import ElasticSearchClient
import pandas as pd
import logging
from elasticsearch import helpers
import sys
from sklearn.mixture import GaussianMixture
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
def gmm_train(total_rate, consumption_df, source, target):
# 高斯混合模型
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]])
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
return consumption_df
def calc_person_rating(city=None):
"""
计算消费等级
"""
sql = "select id, city, region, name, household_count from community_summary"
if pd.notna(city):
sql += " where city='" + city + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
consumption_aim_df = community_summary_df[['_id', 'city', 'household_count']]
consumption_aim_df = consumption_aim_df.dropna(subset=['household_count'])
cities = consumption_aim_df['city'].unique()
source = "household_count"
target = "person_rating"
actions = list()
count = 0
for city in cities:
df = consumption_aim_df[consumption_aim_df['city'] == city].copy()
if len(df) < 5:
continue
df = gmm_train(5, df, source, target)
# 批量更新
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['person_rating'] = item['person_rating']
action = {'_op_type': 'update',
'_index': index_name,
'_id': _id, # 请勿忘记ID
'doc': row}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级中: 当前已评级{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级完成,共更新: {}".format(count))
def sync(city=None):
calc_person_rating(city)
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+72 -48
View File
@@ -3,11 +3,13 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, ClickHouseClient
from elasticsearch import helpers
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -16,8 +18,10 @@ m_date = dateutils.iso_format()
def build_doc(item):
doc_dict = dict()
if 'city_avg_price' in item and pd.notna(item['city_avg_price']):
doc_dict['city_avg_price'] = int(item['city_avg_price'])
if 'region_avg_price' in item and pd.notna(item['region_avg_price']):
doc_dict['region_avg_price'] = item['region_avg_price']
doc_dict['region_avg_price'] = int(item['region_avg_price'])
if 'avg_price' in item and pd.notna(item['avg_price']):
doc_dict['avg_price'] = int(item['avg_price'])
if 'avg_price_region_ranking' in item and pd.notna(item['avg_price_region_ranking']):
@@ -35,57 +39,69 @@ def build_doc(item):
return doc_dict
def sync(city=None):
log.info(f"开始同步价格信息至206 {index_name}")
seaweed_query = {'cat': 'district', 'status': 0}
if pd.notna(city):
seaweed_query['city'] = city
cities = [city]
else:
cities = None
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
def sync_by_city(city):
seaweed_df = tmp_ck.read_sql_query(f"select * from community_summary where city='{city}'")
if len(seaweed_df) == 0:
return
# 小区所在城市均价
city_price_df = commutils.get_city_price(city=city)
if len(city_price_df) > 0:
city_price_df = city_price_df[['city', 'avg_price']]
city_price_df = city_price_df.rename(columns={'avg_price': 'city_avg_price'})
seaweed_df = pd.merge(seaweed_df, city_price_df, how='left', on=['city'])
# 小区所在区域均价
region_price_df = commutils.get_region_price(city=city)
region_price_df = region_price_df[['city', 'region', 'avg_price']]
region_price_df = region_price_df.rename(columns={'avg_price': 'region_avg_price'})
seaweed_df = pd.merge(seaweed_df, region_price_df, how='left', on=['city', 'region'])
if len(region_price_df) > 0:
region_price_df = region_price_df[['city', 'region', 'avg_price']]
region_price_df = region_price_df.rename(columns={'avg_price': 'region_avg_price'})
seaweed_df = pd.merge(seaweed_df, region_price_df, how='left', on=['city', 'region'])
# 小区均价
avg_price_df = commutils.get_avg_price(city=city)
avg_price_df = avg_price_df[['city', 'district_name', 'region', 'avg_price']]
avg_price_df = avg_price_df.rename(columns={'district_name': 'name'})
seaweed_df = pd.merge(seaweed_df, avg_price_df, how='left', on=['city', 'region', 'name'])
if len(avg_price_df) > 0:
avg_price_df = avg_price_df[['city', 'district_name', 'region', 'avg_price']]
avg_price_df = avg_price_df.rename(columns={'district_name': 'name'})
seaweed_df = pd.merge(seaweed_df, avg_price_df, how='left', on=['city', 'region', 'name'])
# 小区均价区域排名
seaweed_df['avg_price_region_ranking'] = seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
# 小区均价城市排名
seaweed_df['avg_price_ranking'] = seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
# 小区均价区域排名
seaweed_df['avg_price_region_ranking'] = \
seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)[
'avg_price']
# 小区均价城市排名
seaweed_df['avg_price_ranking'] = \
seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
# 12个月均价涨跌幅
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=cities, months=-12)
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(columns={'district_name': 'name', 'up_and_downs': 'year_on_year'})
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-12)
if len(avg_price_up_and_downs_df) > 0:
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
columns={'district_name': 'name', 'up_and_downs': 'year_on_year'})
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
# 近半年均价涨跌幅
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=cities, months=-6)
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
columns={'district_name': 'name', 'up_and_downs': 'half_on_year'})
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-6)
if len(avg_price_up_and_downs_df) > 0:
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
columns={'district_name': 'name', 'up_and_downs': 'half_on_year'})
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
# 平米租金
rent_price_df = commutils.get_rent_price(city=city)
rent_price_df = rent_price_df[['city', 'region', 'district_name', 'avg_price']]
rent_price_df = rent_price_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_price'})
seaweed_df = pd.merge(seaweed_df, rent_price_df, how='left', on=['city', 'region', 'name'])
if len(rent_price_df) > 0:
rent_price_df = rent_price_df[['city', 'region', 'district_name', 'avg_price']]
rent_price_df = rent_price_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_price'})
seaweed_df = pd.merge(seaweed_df, rent_price_df, how='left', on=['city', 'region', 'name'])
# 政府指导价
gov_price_df = commutils.get_gov_guide_price()
gov_price_df = gov_price_df[['city', 'region', 'name', 'avg_price']]
gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'})
seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name'])
gov_price_df = commutils.get_gov_guide_price(city=city)
if len(gov_price_df) > 0:
gov_price_df = gov_price_df[['city', 'region', 'name', 'avg_price']]
gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'})
seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name'])
actions = list()
count = 0
@@ -101,20 +117,28 @@ def sync(city=None):
actions.append(action)
if len(actions) == 5000:
try:
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
print("更新:" + str(count))
except Exception as e:
print(e)
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
log.info("更新:" + str(count))
actions.clear()
if len(actions) > 0:
try:
helpers.bulk(es206_client.elastic_client, actions)
except Exception as e:
print(e)
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
def sync(curr_city=None):
log.info(f"开始同步价格信息至206 {index_name}")
if pd.notna(curr_city):
cities = [curr_city]
else:
cities = areautils.get_all_cities()
for city_now in cities:
log.info(f"开始同步{city_now}")
sync_by_city(city_now)
log.info(f"完成同步{city_now}")
log.info(f"完成同步价格信息至206 {index_name}")
sync()
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
@@ -8,6 +8,7 @@ from goojia_utils import ElasticSearchClient
import pandas as pd
import logging
from elasticsearch import helpers
import sys
from sklearn.mixture import GaussianMixture
@@ -16,12 +17,13 @@ log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
def gmm_train(total_rate, consumption_df, source, target):
# 高斯混合模型
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
consumption_df['cluster'] = gmm.predict(consumption_df[[source]])
consumption_df['price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df[target] = consumption_df['price_mean_cluster'].rank(method='dense').astype(int)
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]])
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
return consumption_df
@@ -46,12 +48,12 @@ def calc_price_rating(city=None):
for city in cities:
df = consumption_aim_df[consumption_aim_df['city'] == city].copy()
if len(df) < 10:
if len(df) < 5:
continue
df = gmm_train(10, df, source, target)
if len(df) >= 100:
df = gmm_train(100, df, source, score_target)
df['price_score'] /= 10
df = gmm_train(5, df, source, target)
if len(df) >= 5:
df = gmm_train(5, df, source, score_target)
df['price_score'] /= 5
# 批量更新
for index, item in df.iterrows():
@@ -82,4 +84,5 @@ def sync(city=None):
calc_price_rating(city)
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -4,19 +4,23 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import sys
from threading import Lock
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
mutex = Lock()
def find_around_seaweed(lng, lat):
def find_around_seaweed(city_now, lng, lat):
"""
查找周边均价
"""
@@ -27,6 +31,13 @@ def find_around_seaweed(lng, lat):
"query": {
"bool": {
"filter": [
{
"term": {
"city": {
"value": city_now
}
}
},
{
"range": {
"avg_price": {
@@ -54,32 +65,50 @@ def build_doc(item):
return doc_dict
find_count = 0
def find_around_price_item(seaweed):
global find_count
find_count += 1
if find_count % 1000 == 0:
log.info("已查找" + str(find_count))
if 'lng' not in seaweed:
return
lng = commutils.format_lat_lng(seaweed['lng'])
lat = commutils.format_lat_lng(seaweed['lat'])
if pd.notna(lng):
city_now = seaweed['city']
around_seaweed_df = find_around_seaweed(city_now, lng, lat)
mutex.acquire(timeout=60)
if len(around_seaweed_df) > 1:
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
seaweed['relative_price_ranking'] = around_seaweed_df[around_seaweed_df['id'] == seaweed['_id']].iloc[0][
'relative_price_ranking']
elif len(around_seaweed_df) == 1:
seaweed['relative_price_ranking'] = 0.5
mutex.release()
def sync(city=None):
log.info("开始同步相对价格排名数据...")
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
def find_around_price_item(index, item):
if index % 1000 == 0:
log.info("已处理" + str(index))
if 'lng' not in item:
return
lng = commutils.format_lat_lng(item['lng'])
lat = commutils.format_lat_lng(item['lat'])
if pd.notna(lng):
around_seaweed_df = find_around_seaweed(lng, lat)
if len(around_seaweed_df) > 1:
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0]
elif len(around_seaweed_df) == 1:
seaweed_df.loc[index, 'relative_price_ranking'] = 0.5
seaweed_list = list()
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
with ThreadPoolExecutor(max_workers=30) as pool:
for index, item in seaweed_df.iterrows():
pool.submit(find_around_price_item, index, item)
for seaweed in seaweed_list:
pool.submit(find_around_price_item, seaweed)
# for seaweed in seaweed_list:
# find_around_price_item(seaweed)
seaweed_df = pd.DataFrame(seaweed_list)
actions = list()
count = 0
@@ -90,17 +119,15 @@ def sync(city=None):
"_op_type": "update",
"_index": index_name,
"_id": item['_id'],
"doc": doc_info
"doc": doc_info,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
try:
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
log.info("更新:" + str(count))
except Exception as e:
log.error(e)
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
log.info("更新:" + str(count))
actions.clear()
if len(actions) > 0:
try:
@@ -112,4 +139,5 @@ def sync(city=None):
log.info("完成同步相对价格排名数据...")
sync()
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
+25 -16
View File
@@ -3,35 +3,42 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import sys
from threading import Lock
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
mutex = Lock()
def build_doc(item):
doc_dict = dict()
if 'three_months_sell_num' in item and pd.notna(item['three_months_sell_num']):
doc_dict['three_months_sell_num'] = int(item['three_months_sell_num'])
if 'half_on_year_sell_num' in item and pd.notna(item['half_on_year_sell_num']):
doc_dict['half_on_year_sell_num'] = int(item['half_on_year_sell_num'])
return doc_dict
def sync(city=None):
log.info(f"开始同步出售信息至206 {index_name}")
seaweed_query = {'cat': 'district', 'status': 0}
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
with ThreadPoolExecutor(50) as pool:
with ThreadPoolExecutor(30) as pool:
def deal(index, item):
if index % 1000 == 0:
log.info("已处理" + str(index))
@@ -47,7 +54,14 @@ def sync(city=None):
# 近三个月挂牌量
sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=93)
# 近半年挂牌量
sample_half_year_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=186)
mutex.acquire()
seaweed_df.loc[index, 'three_months_sell_num'] = sample_num
seaweed_df.loc[index, 'half_on_year_sell_num'] = sample_half_year_num
mutex.release()
for index, item in seaweed_df.iterrows():
pool.submit(deal, index, item)
@@ -67,20 +81,15 @@ def sync(city=None):
actions.append(action)
if len(actions) == 5000:
try:
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
print("已更新:" + str(count))
except Exception as e:
print(e)
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
log.info("已更新:" + str(count))
actions.clear()
if len(actions) > 0:
try:
helpers.bulk(es206_client.elastic_client, actions)
except Exception as e:
print(e)
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
log.info(f"完成同步出售信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+11 -50
View File
@@ -3,10 +3,9 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import re
import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -25,59 +24,20 @@ def build_doc(item):
return doc_dict
# def avg_price_map_deal(x):
# avg_price_map[x['city'] + x['region'] + x['name']] = x['avg_price']
# return x
def deal(seaweed_df, index, item):
if index % 1000 == 0:
log.info("已处理" + str(index))
curr_city = item['city']
curr_region = item['region']
curr_name = item['name']
names = [curr_name]
if pd.notna(item['alias']) and '[]' != item['alias']:
alias_list = re.sub("\[|\]|'", '', item['alias']).split(',')
if len(alias_list) > 0:
for alias_item in alias_list:
names.append(alias_item)
# 户均总价
# if (curr_city + curr_region + curr_name) in avg_price_map:
# avg_price = avg_price_map[curr_city + curr_region + curr_name]
# if pd.notna(avg_price):
# 由于覆盖率较低,暂不过滤
# min_avg_price = 0.8 * avg_price
# max_avg_rice = 1.2 * avg_price
total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366)
seaweed_df.loc[index, 'total_price'] = total_price
def sync(city=None):
log.info(f"开始同步户均总价信息至206 {index_name}")
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
# 加载库中均价数据
# query_sql = "SELECT city, region, name, avg_price FROM community_summary "
# if pd.notna(city):
# query_sql += " WHERE city='" + city + "' "
# community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql)
# community_summary_df.apply(lambda x: avg_price_map_deal(x), axis=1)
with ThreadPoolExecutor(30) as pool:
for index, item in seaweed_df.iterrows():
pool.submit(deal, seaweed_df, index, item)
query_sql = "SELECT city, region, name, avg_price, house_avg_area FROM community_summary WHERE avg_price > 0 and house_avg_area > 0 "
if pd.notna(city):
query_sql += " AND city='" + city + "' "
community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql)
actions = list()
count = 0
for index, item in seaweed_df.iterrows():
community_summary_df['total_price'] = community_summary_df['avg_price'] * community_summary_df['house_avg_area']
community_summary_df['total_price'] = community_summary_df['total_price'].astype(int)
for index, item in community_summary_df.iterrows():
doc_info = build_doc(item)
action = {
@@ -106,4 +66,5 @@ def sync(city=None):
log.info(f"完成同步户均总价信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -5,6 +5,7 @@ import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, dateutils
from elasticsearch import helpers
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -21,17 +22,9 @@ def build_doc(item):
return doc_dict
def sync():
log.info(f"开始补全户均总价信息至206 {index_name}")
factor_df = pd.read_csv(path + '/小区户均总价面积因子已去极值.csv', sep='@', encoding='UTF-8')
community_df = es206_client.scan_by_sql_to_data_frame("select city, region, name, avg_price from community_summary where avg_price is not null and total_price is null")
community_factor_df = pd.merge(community_df, factor_df, how='left', on=['_id', 'city', 'region', 'name'])
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
def save(community_factor_df):
community_factor_df = community_factor_df[community_factor_df['total_price'].notna()]
community_factor_df['total_price'] = community_factor_df['total_price'].astype(int)
actions = list()
count = 0
for index, item in community_factor_df.iterrows():
@@ -41,25 +34,86 @@ def sync():
"_op_type": "update",
"_index": index_name,
"_id": item['_id'],
"doc": doc_info
"doc": doc_info,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
try:
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
logging.info("已补全:" + str(count))
except Exception as e:
log.error(e)
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
logging.info("已补全:" + str(count))
actions.clear()
if len(actions) > 0:
try:
helpers.bulk(es206_client.elastic_client, actions)
except Exception as e:
log.error(e)
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
def complete_by_build_year_area(community_factor_df):
build_year_area_df = pd.read_excel(path + '/全国各区各建筑年代户均面积已去极值.xlsx')
build_year_area_dict = dict()
for index, build_year_area in build_year_area_df.iterrows():
build_year_area_dict[
build_year_area['city'] + '|' + build_year_area['region'] + "|" + build_year_area['buildyear']] = build_year_area['factor']
# 修复按城市跑批数据字段可能不存在问题
if 'finish_building_date' not in community_factor_df.columns or 'total_price' not in community_factor_df.columns:
return False
for index, item in community_factor_df[
(community_factor_df['total_price'].isna()) & (community_factor_df['finish_building_date'].notna())].iterrows():
finish_building_date = item['finish_building_date']
if finish_building_date < 1980:
community_factor_df.loc[index, 'factor'] = build_year_area_dict[
item['city'] + "|" + item['region'] + "|1980以前"] if item['city'] + "|" + item[
'region'] + "|1980以前" in build_year_area_dict else None
elif 1980 <= finish_building_date < 1990:
community_factor_df.loc[index, 'factor'] = build_year_area_dict[
item['city'] + "|" + item['region'] + "|[1980-1990)"] if item['city'] + "|" + item[
'region'] + "|[1980-1990)" in build_year_area_dict else None
elif 1990 <= finish_building_date < 2000:
community_factor_df.loc[index, 'factor'] = build_year_area_dict[
item['city'] + "|" + item['region'] + "|[1990-2000)"] if item['city'] + "|" + item[
'region'] + "|[1990-2000)" in build_year_area_dict else None
elif 2000 <= finish_building_date < 2010:
community_factor_df.loc[index, 'factor'] = build_year_area_dict[
item['city'] + "|" + item['region'] + "|[2000-2010)"] if item['city'] + "|" + item[
'region'] + "|[2000-2010)" in build_year_area_dict else None
elif 2010 <= finish_building_date < 2020:
community_factor_df.loc[index, 'factor'] = build_year_area_dict[
item['city'] + "|" + item['region'] + "|[2010-2020)"] if item['city'] + "|" + item[
'region'] + "|[2010-2020)" in build_year_area_dict else None
elif finish_building_date >= 2020:
community_factor_df.loc[index, 'factor'] = build_year_area_dict[
item['city'] + "|" + item['region'] + "|2020及以后"] if item['city'] + "|" + item[
'region'] + "|2020及以后" in build_year_area_dict else None
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
def complete_by_region_area(community_factor_df):
region_area_df = pd.read_excel(path + '/全国各区户均面积已去极值.xlsx')
region_area_dict = dict()
for index, region_area in region_area_df.iterrows():
region_area_dict[region_area['city'] + '|' + region_area['region']] = region_area['factor']
for index, item in community_factor_df[community_factor_df['total_price'].isna()].iterrows():
community_factor_df.loc[index, 'factor'] = region_area_dict[item['city'] + "|" + item['region']] if item['city'] + "|" + item['region'] in region_area_dict else None
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
def sync(city=None):
log.info(f"开始补全户均总价信息至206 {index_name}")
sql = "select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null "
if pd.notna(city):
sql += " and city='" + city + "'"
community_df = es206_client.scan_by_sql_to_data_frame(sql)
if len(community_df) > 0:
community_df['total_price'] = None
complete_by_build_year_area(community_df)
complete_by_region_area(community_df)
save(community_df)
log.info(f"完成补全户均总价信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+15 -5
View File
@@ -3,11 +3,13 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils
import sys
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
from elasticsearch import helpers
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -24,14 +26,21 @@ def build_doc(item):
def sync(city=None):
log.info(f"开始同步成交信息至206 {index_name}")
seaweed_query = {'cat': 'district', 'status': 0}
cities = None
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_query['city'] = city
cities = [city]
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
deal_price_df = commutils.get_deal_avg_price(cities, 3)
if len(deal_price_df) <= 0:
log.info(f"无成交,无需同步")
return
if 'match_city' not in deal_price_df.columns or 'match_region' not in deal_price_df.columns or 'match_name' not in deal_price_df.columns:
log.info(f"无成交,无需同步")
return
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name',
'avg_price': 'three_month_trade_avg_price'})
@@ -68,4 +77,5 @@ def sync(city=None):
log.info(f"完成同步成交信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+8 -5
View File
@@ -3,13 +3,15 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -70,10 +72,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步交通至206 {index_name}")
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
tmp_sql = f"select * from community_summary "
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -86,4 +88,5 @@ def sync(city=None):
log.info(f"完成同步交通至206 {index_name}")
sync()
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
sync(sys_city)
+4 -2
View File
@@ -26,8 +26,10 @@ def sync_district_office(type):
for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0}, {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1,
'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1, 'status': 1})\
.batch_size(10000):
updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'],
'estate_type': doc['estate_type2'], 'status': doc['status']}
updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'], 'status': doc['status']}
if 'estate_type2' in doc:
updateDict['estate_type'] = doc['estate_type2']
id_list.append({'_id': doc['_id']})
if 'block' in doc:
updateDict['block'] = doc['block']
-85
View File
@@ -1,85 +0,0 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
"community_summary_dag",
# 每月2日1点整分运行
schedule_interval="0 1 2 * *",
start_date=pendulum.datetime(2022, 10, 12, tz="Asia/Shanghai"),
catchup=True,
description="地址输出信息分类数据定时更新"
) as dag:
t1 = BashOperator(
task_id='comm',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t2 = BashOperator(
task_id='price',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t3 = BashOperator(
task_id='sell',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t4 = BashOperator(
task_id='avg_cost',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t5 = BashOperator(
task_id='trade',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t6 = BashOperator(
task_id='total_price',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t6_1 = BashOperator(
task_id='total_price_complete',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t7 = BashOperator(
task_id='traffic',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t7_1 = BashOperator(
task_id='business_district',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t8 = BashOperator(
task_id='medical',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t9 = BashOperator(
task_id='life',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t10 = BashOperator(
task_id='relative_price_ranking',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t11 = BashOperator(
task_id='consumption',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t12 = BashOperator(
task_id='price_rating',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t13 = BashOperator(
task_id='education',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t14 = BashOperator(
task_id='education2',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t15 = BashOperator(
task_id='extra',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t16 = BashOperator(
task_id='other',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16
@@ -3,11 +3,11 @@ import pendulum
from airflow import DAG
with DAG(
"amap_key_generate_dag",
dag_id="amap_key_generate_dag",
# 每天运行
schedule_interval="0 2 * * *",
start_date=pendulum.datetime(2022, 7, 18, tz="Asia/Shanghai"),
catchup=True,
schedule_interval="0/1 14 * * *",
start_date=pendulum.datetime(2023, 3, 18, tz="Asia/Shanghai"),
catchup=True
) as dag:
t1 = BashOperator(
task_id='amap_key_generate',
@@ -0,0 +1,17 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
dag_id="change_new_to_old_seaweed_dag",
# 每月1日1点运行
schedule_interval="0 1 1 * *",
start_date=pendulum.datetime(2023, 2, 10, tz="Asia/Shanghai"),
catchup=True,
description="新房变二手房定时脚本"
) as dag:
t1 = BashOperator(
task_id='change_new_to_old_seaweed_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/change_new_to_old_seaweed.py',
)
t1
+98
View File
@@ -0,0 +1,98 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
dag_id="community_summary_dag",
# 每月2日1点整分运行
schedule_interval="0 1 2 * *",
start_date=pendulum.datetime(2023, 3, 16, tz="Asia/Shanghai"),
catchup=True,
description="地址输出信息分类数据定时更新"
) as dag:
comm = BashOperator(
task_id='comm',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
price = BashOperator(
task_id='price',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
sell = BashOperator(
task_id='sell',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
avg_cost = BashOperator(
task_id='avg_cost',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
trade = BashOperator(
task_id='trade',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
total_price = BashOperator(
task_id='total_price',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
total_price_complete = BashOperator(
task_id='total_price_complete',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
traffic = BashOperator(
task_id='traffic',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
business_district = BashOperator(
task_id='business_district',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
medical = BashOperator(
task_id='medical',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
life = BashOperator(
task_id='life',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
relative_price_ranking = BashOperator(
task_id='relative_price_ranking',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
estate_charge_complete = BashOperator(
task_id='estate_charge_complete',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
estate_charge_ranking = BashOperator(
task_id='estate_charge_ranking',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
consumption = BashOperator(
task_id='consumption',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
person_rating = BashOperator(
task_id='person_rating',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/person_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
price_rating = BashOperator(
task_id='price_rating',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
education = BashOperator(
task_id='education',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
education2 = BashOperator(
task_id='education2',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
extra = BashOperator(
task_id='extra',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
other = BashOperator(
task_id='other',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
# 人均消费并行,值为空
comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> [estate_charge_ranking, price_rating, person_rating] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other
@@ -3,10 +3,10 @@ import pendulum
from airflow import DAG
with DAG(
"didata_sync_dag",
dag_id="didata_sync_dag",
# 每月28日3点整分运行
schedule_interval="0 3 28 * *",
start_date=pendulum.datetime(2022, 8, 22, tz="Asia/Shanghai"),
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
catchup=True,
description="地数相关数据同步type: district或office, 不填默认同步全部"
) as dag:
+18
View File
@@ -0,0 +1,18 @@
CREATE TABLE tmp.community_summary
(
_id String COMMENT '小区ID',
city String COMMENT '城市',
region String COMMENT '区域',
name String COMMENT '名称',
lng Nullable(Float32) COMMENT '经度',
lat Nullable(Float32) COMMENT '纬度',
alias Array(String) COMMENT '别名',
estate_type2 String COMMENT '物业类型',
build_type String COMMENT '建筑类型',
household_count String COMMENT '户数',
block String COMMENT '板块',
estate_charge String COMMENT '物业费',
finish_building_date String COMMENT '建成年代',
m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间'
) ENGINE ReplacingMergeTree(m_date)
PARTITION BY city PRIMARY KEY (city, region, name) ORDER BY (city, region, name);
+44
View File
@@ -0,0 +1,44 @@
CREATE TABLE tmp.mt_community
(
id String COMMENT '小区ID',
city String COMMENT '城市',
region String COMMENT '区域',
name String COMMENT '名称',
alias String COMMENT '别名',
address String COMMENT '地址',
addr_alias String COMMENT '地址别名',
avg_price Nullable(INT) COMMENT '均价',
rent_price Nullable(INT) COMMENT '平米租金',
lng Nullable(Float32) COMMENT '经度',
lat Nullable(Float32) COMMENT '纬度',
estate_type2 String COMMENT '物业类型',
block String COMMENT '板块',
loop_line_info String COMMENT '环线',
business_quarter String COMMENT '商圈',
households String COMMENT '户数',
property_fee String COMMENT '物业费',
car_site_count String COMMENT '停车位',
property_life String COMMENT '产权年限',
house_property String COMMENT '产权性质',
all_area String COMMENT '占地面积',
build_area String COMMENT '建筑面积',
plot_ratio String COMMENT '容积率',
landscaping_ratio String COMMENT '绿化率',
property_type String COMMENT '物业类型',
build_type String COMMENT '建筑类型',
build_structure String COMMENT '建筑结构',
finish_building_date String COMMENT '建成年代',
developer String COMMENT '开发商',
house_type String COMMENT '户型',
house_area String COMMENT '房型建筑面积',
deal_num Nullable(INT) COMMENT '近一年成交量',
district_facilitys String COMMENT '周边配套数量字段',
school Nullable(INT) COMMENT '学校数量',
market Nullable(INT) COMMENT '商场数量',
subway Nullable(INT) COMMENT '地铁数量',
bank Nullable(INT) COMMENT '银行数量',
hospital Nullable(INT) COMMENT '医院数量',
status INT COMMENT '0 正常 1 删除',
m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间'
) ENGINE ReplacingMergeTree(m_date)
PARTITION BY city PRIMARY KEY (city, region, name) ORDER BY (city, region, name);
+2
View File
@@ -0,0 +1,2 @@
pip install -r /opt/airflow/dags/repo/dags/requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/
pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/
-3
View File
@@ -1,3 +0,0 @@
"""
建行脚本
"""
@@ -6,12 +6,12 @@ with DAG(
dag_id="merge_dag",
# 每日下午4点
schedule_interval="0 16 * * *",
start_date=pendulum.datetime(2022, 11, 15, tz="Asia/Shanghai"),
start_date=pendulum.datetime(2023, 3, 18, tz="Asia/Shanghai"),
catchup=True,
description="每日融合文件检测"
description="每日融合文件检测, 若想指定检测某一天如20230314,则传参day='20230314'"
) as dag:
t1 = BashOperator(
task_id='merge',
bash_command='python /opt/airflow/dags/repo/dags/jh/merge/merge.py',
bash_command='python /opt/airflow/dags/repo/dags/jh/merge/merge.py {{dag_run.conf["day"] if "day" in dag_run.conf}}',
)
t1
+30
View File
@@ -0,0 +1,30 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
dag_id="price_exception_dag",
# 每月23日运行
schedule_interval="0 10 23 * *",
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
catchup=True,
description="建行每月价格异常检测脚本,默认检测本月与上月差异, 若对比上月与前月价格,设置year_month=\"202210\"(前月日期即可) 若输出样本存在与否,设置year_month=sample"
) as dag:
t1 = BashOperator(
task_id='price_evaluation',
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_evaluation.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}',
)
t2 = BashOperator(
task_id='price_exception',
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}',
)
t3 = BashOperator(
task_id='no_price_community',
bash_command='python /opt/airflow/dags/repo/dags/jh/price/no_price_community.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}',
)
t4 = BashOperator(
task_id='price_deviation',
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_deviation.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}',
)
t1 >> t2 >> t3 >> t4
@@ -6,7 +6,7 @@ with DAG(
dag_id="third_price_dag",
# 每月23日运行
schedule_interval="0 16 18 * *",
start_date=pendulum.datetime(2022, 11, 21, tz="Asia/Shanghai"),
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
catchup=True,
description="建行过滤第三方均价"
) as dag:
View File
+37 -230
View File
@@ -1,242 +1,21 @@
#!/usr/bin/env python
# coding: utf-8
# In[56]:
import os
import tarfile
from goojia_utils import dateutils, ftputils, tarutils
import datetime
import calendar
import requests
import json
import shutil
# In[9]:
import ftplib
import os
import socket
import stat
from os.path import isdir
from numpy import unicode
import paramiko
def _check_local(local):
if not os.path.exists(local):
try:
os.mkdir(local)
except IOError as err:
print(err)
class SftpClient:
def __init__(self):
self.client = None
self.sftp = None
pass
def sftp_connect(self, username, password, host, port=22):
"""
建立连接,获取sftp句柄
:param username
:param password
:param host
:param port
:return:
"""
try:
self.client = paramiko.Transport((host, port))
except Exception as error:
print(error)
else:
try:
self.client.connect(username=username, password=password)
except Exception as error:
print(error)
else:
self.sftp = paramiko.SFTPClient.from_transport(self.client)
def get(self, remote, local):
"""
从远程机器上下载文件(夹)到本地
:param remote 远程路径
:param local 本地路径
"""
try:
result = self.sftp.stat(remote)
except IOError as err:
error = '[ERROR %s] %s: %s' % (err.errno, os.path.basename(os.path.normpath(remote)), err.strerror)
print(error)
else:
# 判断远程文件是否为目录
if stat.S_ISDIR(result.st_mode):
dirname = os.path.basename(os.path.normpath(remote))
local = os.path.join(local, dirname)
_check_local(local)
for file in self.sftp.listdir(remote):
sub_remote = os.path.join(remote, file)
sub_remote = sub_remote.replace('\\', '/')
self.get(sub_remote, local)
else:
# 拷贝文件
if os.path.isdir(local):
local = os.path.join(local, os.path.basename(remote))
try:
self.sftp.get(remote, local)
except IOError as err:
print(err)
else:
print('[get]', local, '<==', remote)
# 断开连接
def disconnect(self):
try:
self.client.close()
except Exception as error:
print(error)
error
class FtpClient:
def __init__(self):
self.ftp = ftplib.FTP()
self.ftp.set_debuglevel(2)
pass
def connect_ftp(self, host, port, timeout=60, user=None, passwd=None):
"""
连接登录ftp
:param host:
:param port:
:param user:
:param passwd:
:return:
"""
try:
self.ftp.connect(host=host, port=port, timeout=timeout)
print("*******已经成功连接服务器FTP服务!!!")
except(socket.error, socket.gaierror) as e:
print("错误:无法访问服务器FTP服务!!!")
print(e)
exit()
try:
self.ftp.login(user=user, passwd=passwd)
except ftplib.error_perm as e:
print("登录失败,请检查用户名和密码,错误代码")
print(e)
exit()
def cat_dir(self, filepath):
"""
显示ftp指定目录下的文件
:param filepath:
:return:
"""
return self.ftp.dir(filepath)
def make_dir(self, filepath):
"""
新建ftp下的目录,ftp目录下不存在此文件
:param filepath:
:return:
"""
try:
filepath.encode("utf-8")
mkd = self.ftp.mkd(filepath)
return mkd
except ftplib.error_perm as e:
print("请查看目录下是否存在同名文件,异常为")
print(e)
def remove_dir(self, dirname):
"""
删除目录
:param dirname
"""
try:
self.ftp.rmd(dirname)
except ftplib.error_perm as e:
print("异常:!!!目录已经被删除了")
print(e)
def download_file(self, localpath, remotepath, filename):
"""
从FTP指定目录上下载文件,并保存到本地指定路径
:param localpath: linux上路径
:param remotepath: 本地路径
:param filename: 文件名
"""
bufsize = 8192 # 设置缓冲区大小;
ulocalpath = unicode(localpath, "utf-8")
try:
self.ftp.cwd(remotepath)
filepath = ulocalpath + '/'
file_handler = open(filepath + filename, "wb").write # 以写模式打开本地文件,---需下载到本地的路径
# 传一个回调函数给retrbinary() 它在每接收一个二进制数据时都会被调用
self.ftp.retrbinary("RETR " % filename, file_handler, bufsize) # 接收服务器上文件并写入本地文件
except ftplib.error_perm as e:
print("请检查路径是否正确,是否有权限读取,错误代码")
print(e)
def upload(self, localpath):
"""
上传至ftp
:param localpath: linux上路径
"""
bufsize = 8192
ulocalpath = unicode(localpath, "utf-8")
try:
file_handler = open(ulocalpath, "rb")
filename = os.path.split(ulocalpath)[-1]
try:
self.ftp.storbinary("stor " % filename, file_handler, bufsize)
except ftplib.error_perm as e:
print("请检查上传路径是否正确,是否连接FTP,---错误代码")
print(e)
except e:
print("请检查本地路径是否存在····")
def lst_file(self, path):
"""
获取指定目录下文件
:param path
"""
s = []
try:
for name in self.ftp.nlst(path):
name1 = unicode(name, "utf-8")
s.append(name1)
return s
except ftplib.all_errors as e:
print("请检查目录是否正确,错误代码时-----")
print(e)
def close(self):
"""
退出ftp
"""
self.ftp.quit()
import sys
result = []
# #### 检测传输压缩文件是否完整
# ##### 1. 融合每天检测
try:
today = dateutils.add_months(datetime.datetime.now(), days=1)
date = dateutils.format_date(today, '%Y%m%d')
date = sys.argv[1] if len(sys.argv) > 1 else None
if date is None:
today = dateutils.add_months(datetime.datetime.now(), days=1)
date = dateutils.format_date(today, '%Y%m%d')
print('当前月:' + date)
curr_month_days = calendar.monthrange(today.year, today.month)[1]
print('当月天数:' + str(curr_month_days))
# curr_month_days = calendar.monthrange(today.year, today.month)[1]
# print('当月天数:' + str(curr_month_days))
# In[79]:
@@ -245,13 +24,18 @@ try:
path = '/home/airflow/data/rh'
if not os.path.exists(path):
os.makedirs(path)
if not os.path.exists(path + "/" + date):
os.makedirs(path + "/" + date)
client = SftpClient()
client = ftputils.SftpClient()
client.sftp_connect(username="ccb", password="ccb!@#", host="192.168.0.45", port=17495)
client.get(remote="/home/ccb_rh/output/" + date, local=path)
client.get(remote="/home/ccb_rh/output/" + date + '/000000_' + date + '_FJW.tar.gz', local=path + '/' + date)
client.get(remote="/home/ccb_rh/output/" + date + '/' + date + '.ok', local=path + '/' + date)
client.disconnect()
# In[80]:
@@ -322,6 +106,29 @@ try:
result.append('FJW_000000_' + cityid + '_BIGDATA_POSITION_FJW_ALL_' + date + '_0001.dat文件不存在')
if 'FJW_000000_' + cityid + '_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_' + cityid + '_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_GUIDE_STATISTICS_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_GUIDE_STATISTICS_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_' + date + '_0001.dat文件不存在')
if len(result) == 0:
result.append('融合检测完毕未发现异常文件')
shutil.rmtree(path + "/" + date)
Binary file not shown.
View File
+63
View File
@@ -0,0 +1,63 @@
import os, sys
import logging
import pandas as pd
import requests
from datetime import datetime
from goojia_utils import dateutils, oracle
from sqlalchemy import create_engine
os.environ["NLS_LANG"] = "American_America.AL32UTF8"
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
log = logging.getLogger(__name__)
path = "/home/airflow/data/jh/price"
if not os.path.exists(path):
os.makedirs(path)
year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_date(datetime.utcnow(), "%Y%m"))
log.info(f"测试月份:{year_month}")
avm_df = pd.read_sql(f"select urbn_id,urbn_nm,gujia_bds_comm_num,gujiabdsvalmnycommnum from t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month} ", con=engine)
comm_df = pd.read_sql(f"select urbn_id,gujia_admndiv_nm,gujia_comm_id,comm_nm from base.t0831_bigdata_community_gj_h where gujia_bds_bds_st=0 ", con=engine)
price_df = pd.read_sql(f"select gujia_bds_id as gujia_comm_id,gujia_comm_basic_prc from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month} ", con=engine)
comm_df = comm_df.merge(avm_df, on="urbn_id", how="left")
comm_df = comm_df.merge(price_df, on="gujia_comm_id", how="left")
no_price_comm_df = comm_df[comm_df.gujia_comm_basic_prc.isna()][['urbn_id','urbn_nm','gujia_admndiv_nm','gujia_comm_id','comm_nm']]
avm_df.loc[:,'无均价数量'] = avm_df['gujia_bds_comm_num'] - avm_df['gujiabdsvalmnycommnum']
avm_df.loc[:,'num'] = 60 - avm_df['gujiabdsvalmnycommnum']
avm_df = avm_df[avm_df['无均价数量'] > 0]
avm_df = avm_df[['urbn_id','urbn_nm','num']]
file_name = f"{year_month}无价格小区统计数据.xlsx"
aim_file_name = path + '/' + file_name
with pd.ExcelWriter(aim_file_name) as writer:
no_price_comm_df.to_excel(writer, sheet_name="无价格小区列表", columns=['urbn_id','urbn_nm','gujia_admndiv_nm','gujia_comm_id','comm_nm'], header=["城市id", "城市名称","区域","小区id","小区名称"], index=False)
avm_df.to_excel(writer, sheet_name="待补充数量", columns=['urbn_id','urbn_nm','num'], header=["城市id","城市名称","待补充数量"],index=False)
log.info("无价格小区统计数据结束")
def up_to_nextcloud():
"""
上传统计信息至nextcloud
"""
from nextcloud import NextCloud
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
NEXTCLOUD_PASSWORD = "goojia123456"
nextcloud_file_name = '共享/建行/' + file_name
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD)
nxc.upload_file(aim_file_name, nextcloud_file_name)
def send_msg():
infos = "【准确度测评】无价格小区统计数据生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情"
data = {
"msgtype": "text",
"text": {
"content": infos
}
}
# 发送消息至群
requests.post(
url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
json=data)
up_to_nextcloud()
send_msg()
+74
View File
@@ -0,0 +1,74 @@
import os, sys
import logging
import pandas as pd
import requests
from datetime import datetime
from goojia_utils import dateutils, oracle
from sqlalchemy import create_engine
os.environ["NLS_LANG"] = "American_America.AL32UTF8"
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
log = logging.getLogger(__name__)
path = "/home/airflow/data/jh/price"
if not os.path.exists(path):
os.makedirs(path)
month_now = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now()), '%Y%m')
month_last = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-1), '%Y%m')
month_before = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-2), '%Y%m')
type = sys.argv[1] if len(sys.argv) > 1 else ''
log.info("输出类型" + type)
# 新增支持上一月与前月对比
if type == month_last:
month_now = month_last
month_last = month_before
log.info(f"测试月份:{month_now}")
price_df = pd.read_sql(f"select urbn_id,urbn_nm,gujia_admndiv_nm,gujia_bds_id as gujia_comm_id,comm_nm,gujia_bds_clmtd,gujia_comm_basic_prc from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={month_now} ", con=engine)
deal_df = pd.read_sql(f"select gujia_bds_mdl_id,gujia_comm_id,gujia_bds_trdtckt_prc from base.t0831_bigdata_deal_gj_h where to_char(txn_dt,'yyyyMM')='{month_last}' ", con=engine)
deal_price_df = deal_df.merge(price_df, on="gujia_comm_id", how="left")
deal_price_df["ppe"] = round((deal_price_df["gujia_comm_basic_prc"] - deal_price_df['gujia_bds_trdtckt_prc']) / deal_price_df['gujia_bds_trdtckt_prc'], 4)
deal_price_df = deal_price_df[abs(deal_price_df['ppe']) > 0.15]
# third_df = pd.read_sql(f"select gujia_trdpt_avprc_id,gujia_bds_id as gujia_comm_id,gujia_dtsr_dsc,gujia_bds_comm_avprc from base.t0831_bigdata_third_party_price_gj_h where gujiabds_avprc_dt_val={month_last} ", con=engine)
# third_price_df = third_df.merge(price_df, on="gujia_comm_id", how="left")
# third_price_df["ppe"] = round((third_price_df["gujia_comm_basic_prc"] - third_price_df['gujia_bds_comm_avprc']) / third_price_df['gujia_bds_comm_avprc'], 4)
# third_price_df = third_price_df[abs(third_price_df['ppe']) > 0.15]
file_name = f"{month_now}测试偏差较大案例统计数据.xlsx"
aim_file_name = path + '/' + file_name
with pd.ExcelWriter(aim_file_name) as writer:
deal_price_df.to_excel(writer, sheet_name="成交", columns=['urbn_id','urbn_nm','gujia_bds_mdl_id','gujia_admndiv_nm','gujia_comm_id','comm_nm','gujia_bds_clmtd','gujia_comm_basic_prc','gujia_bds_trdtckt_prc','ppe'], header=["城市id", "城市名称","成交id","区域","小区id","小区名称",'计算类型','计算价格','成交价格','PPE'], index=False)
# third_price_df.to_excel(writer, sheet_name="第三方", columns=['urbn_id','urbn_nm','gujia_trdpt_avprc_id','gujia_admndiv_nm','gujia_comm_id','comm_nm','gujia_bds_clmtd','gujia_comm_basic_prc','gujia_bds_comm_avprc','PPE','gujia_dtsr_dsc'], header=["城市id", "城市名称","第三方id","区域","小区id","小区名称",'计算类型','计算价格','第三方价格','PPE','来源'],index=False)
log.info("偏差较大案例统计数据结束")
def up_to_nextcloud():
"""
上传统计信息至nextcloud
"""
from nextcloud import NextCloud
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
NEXTCLOUD_PASSWORD = "goojia123456"
nextcloud_file_name = '共享/建行/' + file_name
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD)
nxc.upload_file(aim_file_name, nextcloud_file_name)
def send_msg():
infos = "【准确度测评】测试偏差较大案例统计数据生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情"
data = {
"msgtype": "text",
"text": {
"content": infos
}
}
# 发送消息至群
requests.post(
url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
json=data)
up_to_nextcloud()
send_msg()
+129
View File
@@ -0,0 +1,129 @@
import os, sys
import logging
import pandas as pd
import requests
from datetime import datetime
from goojia_utils import dateutils, oracle
from sqlalchemy import create_engine
os.environ["NLS_LANG"] = "American_America.AL32UTF8"
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
important_cities =["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"]
city_order = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx")
def convert_calc_type(cale_type):
if cale_type == 1:
return "指导价"
elif cale_type == 2:
return "样本"
elif cale_type == 3:
return "神经网络"
fitment_code_map = {"0601": "毛坯", "0602": "简装", "0603": "精装"}
def convert_fitment_code(fitment_code):
if pd.isna(fitment_code):
return None
fitment_code = str(fitment_code)
if fitment_code in fitment_code_map:
return fitment_code_map[fitment_code]
return None
house_aspect_code_map = {"0301": "南北", "0302": "东南", "0303": "南", "0304": "东西", "0305": "东", "0306": "西南", "0307": "西",
"0308": "北", "0309": "东北",
"0310": "西北"}
def convert_house_aspect_code(house_aspect_code):
if pd.isna(house_aspect_code):
return None
house_aspect_code = str(house_aspect_code)
if house_aspect_code in house_aspect_code_map:
return house_aspect_code_map[house_aspect_code]
return None
log = logging.getLogger(__name__)
path = "/home/airflow/data/jh/price"
if not os.path.exists(path):
os.makedirs(path)
year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_date(datetime.utcnow(), "%Y%m"))
log.info(f"测试月份:{year_month}")
avm_evaluation_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_bds_yrmo': 'year_month', 'gujia_bds_comm_num': 'community_num', 'gujiabdsvalmnycommnum': 'has_price_community_num', 'gujia_prc_cvr_rate': 'price_coverage', 'gujiabdsmnldpccommnum': 'artificial_calc_num', 'gujia_mnl_drc_prc_pct': 'artificial_calc_coverage', 'gujiabdssamp_comm_num': 'sample_calc_num', 'gujiaext_samp_clc_pct': 'sample_calc_coverage', 'gujiabdsnervetncomnum': 'ann_calc_num', 'gujianervethrunclcpct': 'ann_calc_coverage', 'gujiabds_tst_case_num': 'test_num', 'gujiabdsmdltstcommnum': 'deal_test_comm_num', 'gujiatst_comm_num_pct': 'test_comm_coverage', 'gujia_ppe15_cvr_rate': 'ppe15_coverage', 'gujia_rto_mdn': 'rate_median', 'gujia_rto_lv_san_ceft': 'coefficient', 'gujiabdsmdltstcasenum': 'deal_test_case_num', 'gujiamdltstcommnumpct': 'deal_test_comm_coverage', 'gujiamdltst_ppe15_pct': 'deal_ppe15_coverage', 'gujiamdlrtolvsan_ceft': 'deal_coefficient', 'gujia_mdl_md_bit_rto': 'deal_rate_median', 'gujiabdstrdpttcasenum': 'third_test_case_num', 'gujiabdstrdpttcommnum': 'third_test_comm_num', 'gujiatrdpttcommnumpct': 'third_test_comm_coverage', 'gujiatrdpttstppe15pct': 'third_ppe15_coverage', 'gujiatrdptrolvsanceft': 'third_coefficient', 'gujiatrdpt_md_bit_rto': 'third_rate_median', 'data_crt_tm': 'createtime', 'data_udt_tm': 'updatetime', 'gujia_src_co': 'sourcecode'}
avm_evaluation_item_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_rgon_id': 'distid', 'gujia_rgon_nm': 'distname', 'gujia_comm_id': 'commid', 'gujia_comm_nm': 'commname', 'gujiabds_clc_yrmo_val': 'year_month', 'gujia_bds_clc_avprc': 'calc_avg_price', 'gujia_bds_avprc': 'avg_price', 'gujia_bds_clc_tp': 'calc_type', 'gujia_bds_rlb': 'confidence', 'gujiabds_to_mkprc_mny': 'target_avg_price', 'mdl_dt': 'dealdate', 'gujiabds_valt_unitprc': 'unit_price', 'gujia_area': 'buildarea', 'gujia_dcrt_val': 'fitment', 'gujia_twrd_val': 'houseaspect', 'gujia_bds_floor_num': 'floornum', 'gujiabdstot_floor_num': 'floorcnt', 'gujia_data_tp': 'data_type', 'gujia_rto': 'rate', 'gujia_mns_rto': 'pe', 'data_crt_tm': 'createtime'}
avm_evaluation = pd.read_sql(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month}", con=engine)
avm_evaluation_item = pd.read_sql(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val={year_month}", con=engine)
avm_evaluation.rename(columns=avm_evaluation_column_map, inplace=True)
avm_evaluation_item.rename(columns=avm_evaluation_item_column_map, inplace=True)
community_base_price = pd.read_sql(f'''SELECT gujia_bds_id as commid,urbn_nm as cityname,gujia_admndiv_nm as distname,comm_nm as commname,
gujia_comm_basic_prc as comm_avg_price,gujia_bds_cfdc_dgr as confidence,gujia_bds_clmtd as calc_type from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month}''',
con=engine)
communities = pd.read_sql(f"select gujia_comm_id as COMMID from base.t0831_bigdata_community_gj_h", con=engine)
community_base_price = community_base_price[community_base_price.commid.isin(communities["commid"])]
avm_evaluation_item["fitment"] = avm_evaluation_item["fitment"].map(convert_fitment_code)
avm_evaluation_item["houseaspect"] = avm_evaluation_item["houseaspect"].map(convert_house_aspect_code)
avm_evaluation_item["calc_type"] = avm_evaluation_item["calc_type"].map(convert_calc_type)
community_base_price["calc_type"] = community_base_price["calc_type"].map(convert_calc_type)
avm_evaluation_item["dealdate"] = avm_evaluation_item["dealdate"].apply(lambda x : x.strftime("%Y-%m-%d") if pd.notna(x) else None)
avm_evaluation_item_third = avm_evaluation_item[avm_evaluation_item.data_type=='第三方']
avm_evaluation_item_deal = avm_evaluation_item[avm_evaluation_item.data_type=='成交']
import_avm_evaluation_item_third = avm_evaluation_item_third[avm_evaluation_item_third.cityname.isin(important_cities)]
import_avm_evaluation_item_deal = avm_evaluation_item_deal[avm_evaluation_item_deal.cityname.isin(important_cities)]
import_avm_evaluation_item_deal.loc[:, "deal_total_price"] = import_avm_evaluation_item_deal["buildarea"] * import_avm_evaluation_item_deal["target_avg_price"]
import_avm_evaluation_item_deal.loc[:, "deal_total_price"] = import_avm_evaluation_item_deal["deal_total_price"].apply(lambda x : int(x))
import_avm_evaluation_item_deal.loc[:, "total_price"] = import_avm_evaluation_item_deal["buildarea"] * import_avm_evaluation_item_deal["calc_avg_price"]
import_avm_evaluation_item_deal.loc[:, "total_price"] = import_avm_evaluation_item_deal["total_price"].apply(lambda x : int(x))
import_community_base_price = community_base_price[community_base_price.cityname.isin(important_cities)]
normal_community_base_price = community_base_price[~(community_base_price.cityname.isin(important_cities))]
import_avm_evaluation = avm_evaluation[avm_evaluation.cityname.isin(important_cities)]
avm_evaluation = avm_evaluation.merge(city_order, on="cityname", how="left")
import_community_base_price = import_community_base_price.merge(city_order, on="cityname", how="left")
normal_community_base_price = normal_community_base_price.merge(city_order, on="cityname", how="left")
import_avm_evaluation_item_deal = import_avm_evaluation_item_deal.merge(city_order, on="cityname", how="left")
import_avm_evaluation_item_third = import_avm_evaluation_item_third.merge(city_order, on="cityname", how="left")
avm_evaluation.sort_values(by="cityrank", inplace=True)
import_community_base_price.sort_values(by="cityrank", inplace=True)
normal_community_base_price.sort_values(by="cityrank", inplace=True)
import_avm_evaluation_item_deal.sort_values(by="cityrank", inplace=True)
import_avm_evaluation_item_third.sort_values(by="cityrank", inplace=True)
file_name = f"{year_month}估值模型测试.xlsx"
aim_file_name = path + '/' + file_name
with pd.ExcelWriter(aim_file_name) as writer:
avm_evaluation.to_excel(writer, sheet_name="92个城市估值概况", columns=["cityname","community_num","price_coverage","sample_calc_coverage","ann_calc_coverage","artificial_calc_coverage","deal_test_case_num","deal_ppe15_coverage","deal_test_comm_num","deal_test_comm_coverage","deal_rate_median","deal_coefficient","third_test_comm_num","third_test_comm_coverage","third_ppe15_coverage","third_rate_median","third_coefficient"], header=["城市名称","小区量","可输出均价小区占比","样本计算占比","神经网络计算占比","人工指导价占比","成交测试案例总量","成交测试PPE15占比","成交测试小区量","成交测试小区量占比","成交比率中位数","成交比率离散系数","第三方均价测试小区量","第三方均价测试小区量占比","第三方均价测试PPE15占比","第三方比率中位数","第三方比率离散系数"], index=False)
import_community_base_price.to_excel(writer, sheet_name="测试10个城市估值数据", columns=["cityname","distname","commname","comm_avg_price","confidence","calc_type"], header=["城市","区域","小区名","估值小区均价","置信度","估值方式"],index=False)
import_avm_evaluation_item_deal.to_excel(writer, sheet_name="10个城市成交测试案例", columns=['cityname','distname', 'commname', 'unit_price', 'confidence', 'calc_type','total_price', 'deal_total_price', 'target_avg_price','pe',"dealdate","fitment","houseaspect","floornum","floorcnt","buildarea"], header=["城市","区域","小区名","修正均价","置信度","估值方式","小区估价","对标成交价","对标均价","PPE","dealdate","fitment","houseaspect","floornum","floorcnt","buildarea"], index=False)
import_avm_evaluation_item_third.to_excel(writer, sheet_name="10个城市第三方均价测试", columns=["cityname","distname","commname","avg_price","confidence","calc_type","target_avg_price","pe"], header=["城市","区域","小区名","小区均价","置信度","估值方式","对标第三方均价","PPE"],index=False)
log.info("估值结果测试结束")
def up_to_nextcloud():
"""
上传统计信息至nextcloud
"""
from nextcloud import NextCloud
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
NEXTCLOUD_PASSWORD = "goojia123456"
nextcloud_file_name = '共享/建行/' + file_name
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD)
nxc.upload_file(aim_file_name, nextcloud_file_name)
def send_msg():
infos = "【准确度测评】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情"
data = {
"msgtype": "text",
"text": {
"content": infos
}
}
# 发送消息至群
requests.post(
url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
json=data)
up_to_nextcloud()
send_msg()
+11 -8
View File
@@ -7,9 +7,10 @@ import logging
import os
import sys
import requests
from sqlalchemy import create_engine
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
def simple_calc(result_df):
"""
新增是否有样本列
@@ -25,8 +26,9 @@ def simple_calc(result_df):
if len(actions) == 800:
ins = "','".join(actions)
ins = "'" + ins + "'"
simple_temp_df = oracle.scan_by_sql_to_data_frame("select distinct commid from bigdata_forsale_gj where " +
" publishtime> TO_DATE('" + month_before + "','YYYY-MM-DD') and commid in (" + ins + ")")
simple_temp_df = pd.read_sql(
"select distinct gujia_comm_id as commid from base.t0831_bigdata_forsale_gj_h where " +
" anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")", con=engine)
simple_df = pd.concat([simple_df, simple_temp_df])
actions.clear()
@@ -34,8 +36,9 @@ def simple_calc(result_df):
if len(actions) > 0:
ins = "','".join(actions)
ins = "'" + ins + "'"
simple_temp_df = oracle.scan_by_sql_to_data_frame("select distinct commid from bigdata_forsale_gj where " +
" publishtime> TO_DATE('" + month_before + "','YYYY-MM-DD') and commid in (" + ins + ")")
simple_temp_df = pd.read_sql(
"select distinct gujia_comm_id as commid from base.t0831_bigdata_forsale_gj_h where " +
" anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")", con=engine)
simple_df = pd.concat([simple_df, simple_temp_df])
actions.clear()
simple_df['has_sample'] = 1
@@ -85,8 +88,8 @@ if type == month_last:
month_now = month_last
month_last = month_before
df = oracle.scan_by_sql_to_data_frame("SELECT id, cityname, distname, commid, commname, calc_type, confidence, avg_price, year_month FROM community_base_price_gj WHERE year_month IN('"+month_now+"', '"+month_last+"') " +
" ORDER BY cityname, distname, commid, commname, year_month")
df = pd.read_sql("SELECT gujia_bss_prc_id as id, urbn_nm as cityname, gujia_admndiv_nm as distname, gujia_bds_id as commid, comm_nm as commname, gujia_bds_clmtd as calc_type, gujia_bds_cfdc_dgr as confidence, gujia_comm_basic_prc as avg_price, gujiabds_clc_yrmo_val as year_month FROM t_app_p9pha_plda.avm_community_base_price_gj_h WHERE gujiabds_clc_yrmo_val IN('"+month_now+"', '"+month_last+"') " +
" ORDER BY urbn_nm, gujia_admndiv_nm, gujia_bds_id, comm_nm, gujia_bds_clmtd", con=engine)
six_map = dict()
five_map = dict()
@@ -143,7 +146,7 @@ for index, item in city_count_df.iterrows():
city_count_df = city_count_df.rename(columns={'小区ID': '小区数量'})
city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度')
excel_writer.save()
#excel_writer.save()
excel_writer.close()
up_to_nextcloud()
-17
View File
@@ -1,17 +0,0 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
dag_id="price_exception_dag",
# 每月23日运行
schedule_interval="0 10 23 * *",
start_date=pendulum.datetime(2022, 8, 23, tz="Asia/Shanghai"),
catchup=True,
description="建行每月价格异常检测脚本,默认检测本月与上月差异, 若对比上月与前月价格,设置type=\"202210\"(前月日期即可) 若输出样本存在与否,设置type=sample"
) as dag:
t1 = BashOperator(
task_id='price_exception',
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t1
+2 -5
View File
@@ -9,11 +9,8 @@ with DAG(
dag_id="mt_dag",
# 每月5日运行
schedule_interval="0 5 5 * *",
start_date=pendulum.datetime(2022, 6, 28, tz="Asia/Shanghai"),
catchup=True,
default_args={
"output_date": output_date
}
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
catchup=True
) as dag:
t1 = BashOperator(
task_id='mt_comm',
+8 -3
View File
@@ -1,13 +1,14 @@
"""
小区属性导出
"""
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils, ClickHouseClient
import pandas as pd
import datetime
import os
import sys
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
tmp_ck = ClickHouseClient(database='tmp')
path = "/home/airflow/data/mt"
if not os.path.exists(path):
@@ -74,5 +75,9 @@ seaweed_new_result_df = seaweed_new_result_df.rename(columns={'_id': 'id', 'mapp
'cubage_percent' : 'plot_ratio', 'virescence_percent': 'landscaping_ratio', 'estate_type2': 'property_type',
'main_type': 'house_type', 'main_type_area': 'house_area'})
seaweed_new_result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv',
sep='@', encoding='utf-8', index=False)
seaweed_new_result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False)
ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 "
tmp_ck.execute(ck_delete_sql)
tmp_ck.to_sql_plus(seaweed_new_result_df, 'mt_community')
+2 -1
View File
@@ -10,11 +10,12 @@ import requests
path = "/home/airflow/data/mt"
clickhouse_client = ClickHouseClient(database='fangjia_stat')
tmp_ck = ClickHouseClient(database='tmp')
output_date = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(datetime.datetime.now(), '%Y%m')
print("输出日期" + output_date)
result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + ".csv", sep='@', encoding='utf-8')
result_df = tmp_ck.read_sql_query("select * from tmp.mt_community")
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx')
cities = city_df['城市列表'].values.tolist()
+10 -3
View File
@@ -1,7 +1,7 @@
"""
每个小区近一年成交统计
"""
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ElasticSearchClient
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ElasticSearchClient, ClickHouseClient
import pandas as pd
import datetime
import logging
@@ -11,6 +11,7 @@ import sys
es45_client = ElasticSearchClient(host='192.168.0.45')
deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price')
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
tmp_ck = ClickHouseClient(database='tmp')
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx')
cities = city_df['城市列表'].values.tolist()
@@ -26,7 +27,7 @@ logging.info("日期信息:" + month_start + ' ' + curr_month_start)
month_start_date = dateutils.parse_date(month_start, '%Y%m%d')
curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8')
result_df = tmp_ck.read_sql_query("select * from mt_community")
if 'deal_num' in result_df.columns:
result_df = result_df.drop(columns=['deal_num'])
@@ -93,4 +94,10 @@ columns = ['id', 'city', 'region', 'name', 'alias', 'address', 'addr_alias', 'av
'build_structure', 'finish_building_date',
'developer', 'house_type', 'house_area', 'deal_num', 'district_facilitys', 'school', 'market', 'subway',
'bank', 'hospital', 'status']
result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False, columns=columns)
result_save_df = result_df[columns]
ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 "
tmp_ck.execute(ck_delete_sql)
tmp_ck.to_sql_plus(result_save_df, 'mt_community')
+8 -3
View File
@@ -1,13 +1,14 @@
"""
周边配套数量统计
"""
from goojia_utils import MAIN_MONGO_CLIENT, dateutils
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ClickHouseClient
import pandas as pd
import datetime
import logging
import sys
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
tmp_ck = ClickHouseClient(database='tmp')
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx')
cities = city_df['城市列表'].values.tolist()
@@ -73,7 +74,8 @@ district_facilitys_new_df = district_facilitys_df[
district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_name': 'name', 'supermarket': 'market'})
district_facilitys_new_df = district_facilitys_new_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first')
result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8')
result_df = tmp_ck.read_sql_query("select * from mt_community")
if 'school' in result_df.columns:
result_df = result_df.drop(columns=['school', 'bank', 'hospital', 'market', 'subway'])
@@ -141,4 +143,7 @@ result_df['bank'].fillna(0, inplace=True)
result_df['hospital'].fillna(0, inplace=True)
result_df['subway'].fillna(0, inplace=True)
result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False)
ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 "
tmp_ck.execute(ck_delete_sql)
tmp_ck.to_sql_plus(result_df, 'mt_community')
+10 -4
View File
@@ -2,17 +2,17 @@
租金、均价导出
"""
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils, ClickHouseClient
import pandas as pd
import datetime
import logging
import sys
import numpy as np
#district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat')
# 月度均价表
district_sell_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_sell_stat')
sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent')
tmp_ck = ClickHouseClient(database='tmp')
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx')
@@ -22,7 +22,8 @@ path = "/home/airflow/data/mt"
output_date = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(datetime.datetime.now(), '%Y%m')
seaweed_new_result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8')
seaweed_new_result_df = tmp_ck.read_sql_query("select * from mt_community")
if 'avg_price' in seaweed_new_result_df:
seaweed_new_result_df = seaweed_new_result_df.drop(columns=['avg_price', 'rent_price'])
@@ -111,4 +112,9 @@ result_df['ignore_comm'] = None
result_df = result_df.apply(lambda x: format_rent(x), axis=1)
result_df = result_df[result_df['ignore_comm'] != 1]
result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False)
del result_df['ignore_comm']
ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 "
tmp_ck.execute(ck_delete_sql)
tmp_ck.to_sql_plus(result_df, 'mt_community')
@@ -8,11 +8,11 @@ import pendulum
from airflow import DAG
with DAG(
"district_facilities_dag",
dag_id="district_facilities_dag",
# 每月1日1点运行
schedule_interval="0 1 1 * *",
start_date=pendulum.datetime(2022, 6, 18, tz="Asia/Shanghai"),
catchup=True,
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
catchup=True
) as dag:
t1 = BashOperator(
task_id='district_facilities',
+3
View File
@@ -0,0 +1,3 @@
elasticsearch==7.11.0
transbigdata==0.4.15
pykalman==0.9.5
@@ -3,11 +3,11 @@ import pendulum
from airflow import DAG
with DAG(
"community_count_price_rent_dag",
dag_id="count_price_rent_dag",
# 每周日运行
schedule_interval="0 10 * * 7",
start_date=pendulum.datetime(2022, 7, 1, tz="Asia/Shanghai"),
catchup=True,
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
catchup=True
) as dag:
t1 = BashOperator(
task_id='community_count_price_rent',
@@ -3,11 +3,11 @@ import pendulum
from airflow import DAG
with DAG(
"district_sell_deal_dag",
dag_id="district_sell_deal_dag",
# 每周五运行
schedule_interval="0 1 * * 5",
start_date=pendulum.datetime(2022, 6, 20, tz="Asia/Shanghai"),
catchup=True,
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
catchup=True
) as dag:
t1 = BashOperator(
task_id='district_count_thirdavg_deal',
@@ -65,22 +65,23 @@ def third_avg_stat():
'5i5j_esf_price', 'maitian_esf_price', '365house_esf_price', 'beike_esf_price', 'qfang_esf_price',
'fangtan_esf_price',
'tuboshi_esf_price', 'zhugezhaofang_esf_price', 'zhongyuan_esf_price', 'jiwu_esf_price', 'anjuke_esf_price']]
average_statistics_new_df.loc[:, 'has_esf_price'] = average_statistics_new_df['58tc_esf_price'] \
.combine_first(average_statistics_new_df['cityhouse_esf_price']) \
.combine_first(average_statistics_new_df['soufang_esf_price']) \
.combine_first(average_statistics_new_df['lianjia_esf_price']) \
.combine_first(average_statistics_new_df['5i5j_esf_price']) \
.combine_first(average_statistics_new_df['maitian_esf_price']) \
.combine_first(average_statistics_new_df['365house_esf_price']) \
.combine_first(average_statistics_new_df['beike_esf_price']) \
.combine_first(average_statistics_new_df['qfang_esf_price']) \
.combine_first(average_statistics_new_df['fangtan_esf_price']) \
.combine_first(average_statistics_new_df['tuboshi_esf_price']) \
.combine_first(average_statistics_new_df['zhugezhaofang_esf_price']) \
.combine_first(average_statistics_new_df['zhongyuan_esf_price']) \
.combine_first(average_statistics_new_df['jiwu_esf_price']) \
.combine_first(average_statistics_new_df['anjuke_esf_price'])
average_statistics_new_df.fillna(0, inplace=True)
average_statistics_new_df.loc[:, 'has_esf_price'] = average_statistics_new_df['58tc_esf_price'] + \
average_statistics_new_df['cityhouse_esf_price'] + \
average_statistics_new_df['soufang_esf_price'] + \
average_statistics_new_df['lianjia_esf_price'] + \
average_statistics_new_df['5i5j_esf_price'] + \
average_statistics_new_df['maitian_esf_price'] + \
average_statistics_new_df['365house_esf_price'] + \
average_statistics_new_df['beike_esf_price'] + \
average_statistics_new_df['qfang_esf_price'] + \
average_statistics_new_df['fangtan_esf_price'] + \
average_statistics_new_df['tuboshi_esf_price'] + \
average_statistics_new_df['zhugezhaofang_esf_price'] + \
average_statistics_new_df['zhongyuan_esf_price'] + \
average_statistics_new_df['jiwu_esf_price'] + \
average_statistics_new_df['anjuke_esf_price']
average_statistics_new_df.loc[:, ['flag']] = average_statistics_new_df['has_esf_price'].apply(
lambda x: True if x > 0 else False)
average_statistics_new_df = average_statistics_new_df[average_statistics_new_df['flag'] == True]
@@ -130,6 +131,6 @@ def deal_stat():
log.info("开始统计小区数量、近半年第三方均价、近半年成交信息")
count_stat()
third_avg_stat()
# third_avg_stat()
deal_stat()
log.info("完成统计小区数量、近半年第三方均价、近半年成交信息")
+20 -34
View File
@@ -2,7 +2,7 @@
统计近三个月、近一年样本量及小区属性
"""
import logging
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils, commutils
import pandas as pd
import datetime
import sys
@@ -24,37 +24,8 @@ seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0
.batch_size(10000)))
seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'})
def sell_stat():
"""
最近一年有样本小区占比 最近3月有样本小区占比
"""
df = pd.read_excel(aim_file_name)
sell_seaweed_df = es206_client.scan_by_sql_to_data_frame(
"select city, region, district_name, p_date from latest_seawater where p_date > " + start_time + " and cat='sell' ")
# 小区别名升级为单独df
actions = list()
for index, item in seaweed_df.iterrows():
if type(item['alias']) == list:
for name in item['alias']:
actions.append({'_id': item['_id'], 'city': item['city'], 'name': name, 'region': item['region']})
alias_df = pd.DataFrame(actions)
sell_seaweed_rename_df = sell_seaweed_df[['city', 'region', 'district_name']]
sell_seaweed_rename_df = sell_seaweed_rename_df.rename(columns={'district_name': 'name'})
sell_seaweed_merge_df = pd.merge(sell_seaweed_rename_df, alias_df, how='left', on=['city', 'region', 'name'])
sell_seaweed_merge_df['count'] = sell_seaweed_merge_df[sell_seaweed_merge_df['_id'].notna()].groupby('_id')[
'_id'].transform('size')
sell_seaweed_merge_df = sell_seaweed_merge_df[['city', 'region', '_id', 'count']]
sell_seaweed_merge_df = sell_seaweed_merge_df.drop_duplicates(subset=['city', 'region', '_id'])
sell_seaweed_ok_df = sell_seaweed_merge_df[sell_seaweed_merge_df['count'] > sell_num]
sell_seaweed_ok_df['city_seaweed_count'] = sell_seaweed_ok_df.groupby('city')['city'].transform('size')
city_seaweed_count_df = sell_seaweed_ok_df[['city', 'city_seaweed_count']].drop_duplicates(
subset=['city', 'city_seaweed_count'])
city_seaweed_count_df = city_seaweed_count_df.rename(columns={'city': '城市', 'city_seaweed_count': '最近3月有样本小区占比'})
df = pd.merge(df, city_seaweed_count_df, how='left', on=['城市'])
df['最近3月有样本小区占比'] = round(df['最近3月有样本小区占比'] * 100 / df['小区量'], 2)
df.to_excel(aim_file_name, index=False)
def get_sell(df, daysBefore=365):
cities = df[['城市']].drop_duplicates(keep='first')['城市'].values.tolist()
city_one_year_count = list()
for city in cities:
@@ -79,8 +50,7 @@ def sell_stat():
names_str = "','".join(names)
names_str = "'" + names_str + "'"
try:
sql = "select count(*) from latest_seawater where city='" + city + "' and region='" + region + "' and cat='sell' and district_name in(" + names_str + ") and p_date > " + one_year_time
num = es206_client.search_by_sql(sql)['hits']['total']['value']
num = commutils.get_sample(city, region, names_str, daysBefore=daysBefore)
if num > sell_num:
count += 1
total_num += num
@@ -92,9 +62,25 @@ def sell_stat():
print(city + "已处理完, 近一年有样本小区数量" + str(count) + " 近一年的样本总量" + str(total_num))
result_df = pd.DataFrame(city_one_year_count)
return result_df
def sell_stat():
"""
最近一年有样本小区占比 最近3月有样本小区占比
"""
df = pd.read_excel(aim_file_name)
result_df = get_sell(df, 365)
result_df = result_df[['城市', '近一年有样本小区数量', '近一年的样本总量']]
df = pd.merge(df, result_df, how='left', on=['城市'])
df['最近一年有样本小区占比'] = round(df['近一年有样本小区数量'] * 100 / df['小区量'], 2)
df['近一年有样本小区占比'] = round(df['近一年有样本小区数量'] * 100 / df['小区量'], 2)
result_df = get_sell(df, 93)
result_df = result_df.rename(columns={'近一年有样本小区数量': '近三月有样本小区数量'})
result_df = result_df[['城市', '近三月有样本小区数量']]
df = pd.merge(df, result_df, how='left', on=['城市'])
df['近三月有样本小区占比'] = round(df['近三月有样本小区数量'] * 100 / df['小区量'], 2)
df.to_excel(aim_file_name, index=False)
-1
View File
@@ -1 +0,0 @@
pip install -r requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/
-1
View File
@@ -1 +0,0 @@
pip install -r requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/
-5
View File
@@ -1,5 +0,0 @@
nextcloud-api-wrapper
elasticsearch==7.11.0
transbigdata==0.4.15
pykalman==0.9.5
goojia_utils