Merge branch 'master' of http://gitlab.fangjia.com/dev0/airflow
This commit is contained in:
@@ -47,11 +47,13 @@ def build_doc(item):
|
||||
|
||||
estate_charge = commutils.format_property_fee(item['estate_charge'])
|
||||
if pd.notna(estate_charge):
|
||||
doc_dict['estate_charge'] = float(estate_charge)
|
||||
# 表示该物业费已经从库中补充
|
||||
doc_dict['estate_charge_flag'] = 0
|
||||
# 先打10分,对于其他的流程在修改分数
|
||||
doc_dict['credit_detail'] = {'estate_charge_score': 10}
|
||||
# 剔除过大物业费导致ES报错
|
||||
if float(estate_charge) <= 1000:
|
||||
doc_dict['estate_charge'] = float(estate_charge)
|
||||
# 表示该物业费已经从库中补充
|
||||
doc_dict['estate_charge_flag'] = 0
|
||||
# 先打10分,对于其他的流程在修改分数
|
||||
doc_dict['credit_detail'] = {'estate_charge_score': 10}
|
||||
|
||||
finish_building_date = commutils.format_finish_building_date(item['finish_building_date'])
|
||||
if pd.notna(finish_building_date):
|
||||
|
||||
@@ -124,13 +124,15 @@ def calc_consumption_rating(community_summary_df):
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
if len(df) < 5:
|
||||
# 总评级
|
||||
total_rate = 6
|
||||
if len(df) < total_rate:
|
||||
# 对于样本低于5的直接排名
|
||||
df[target] = df['consumption_potential'].rank(method='dense', ascending=False).astype(int)
|
||||
else:
|
||||
# 极值设置
|
||||
df = filter_min_max(df)
|
||||
df = gmm_train(5, df, source, target)
|
||||
df = gmm_train(total_rate, df, source, target)
|
||||
# 批量更新
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
"""
|
||||
省市区简拼转全拼
|
||||
"""
|
||||
from goojia_utils import ElasticSearchClient, dateutils
|
||||
import logging
|
||||
import pandas as pd
|
||||
from elasticsearch import helpers
|
||||
import sys
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es89_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
m_date = dateutils.iso_format()
|
||||
|
||||
count = 0
|
||||
|
||||
short_df = es89_client.scan_by_sql_to_data_frame("select province, city, city_short, district, district_short from administrative_mapping where type='区域' ")
|
||||
short_map = {}
|
||||
def deal_short_map(x):
|
||||
short_map[x['city_short'] + x['district_short']] = x['province'] + '|' + x['city'] + '|' + x['district']
|
||||
short_df.apply(lambda x: deal_short_map(x), axis=1)
|
||||
|
||||
def deal(actions, x):
|
||||
global count
|
||||
doc_info = dict()
|
||||
if pd.notna(x['city']) and pd.notna(x['region']):
|
||||
if x['city'] + x['region'] in short_map:
|
||||
short_info = short_map[x['city'] + x['region']]
|
||||
doc_info['province_full'] = short_info.split('|')[0]
|
||||
doc_info['city_full'] = short_info.split('|')[1]
|
||||
doc_info['region_full'] = short_info.split('|')[2]
|
||||
|
||||
action_doc = {
|
||||
"_op_type": "update",
|
||||
"_index": index_name,
|
||||
"_id": x['_id'],
|
||||
"doc": doc_info
|
||||
}
|
||||
if len(doc_info) > 0:
|
||||
actions.append(action_doc)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
log.info(f"已更新{count}")
|
||||
actions.clear()
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始转换省市区全拼至206 {index_name}")
|
||||
|
||||
sql = "select city, region from community_summary"
|
||||
if pd.notna(city):
|
||||
sql += " where city='" + city + "'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
actions = list()
|
||||
community_summary_df.apply(lambda x: deal(actions, x), axis=1)
|
||||
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
|
||||
log.info(f"完成转换省市区全拼至206 {index_name}")
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -23,10 +23,13 @@ def sync_district_office(type):
|
||||
for city in cities:
|
||||
actions = list()
|
||||
id_list = list()
|
||||
for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0}, {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1,
|
||||
'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1, 'status': 1})\
|
||||
for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0},
|
||||
{'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1,
|
||||
'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1,
|
||||
'status': 1}) \
|
||||
.batch_size(10000):
|
||||
updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'], 'status': doc['status']}
|
||||
updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'],
|
||||
'status': doc['status']}
|
||||
if 'estate_type2' in doc:
|
||||
updateDict['estate_type'] = doc['estate_type2']
|
||||
|
||||
@@ -42,7 +45,7 @@ def sync_district_office(type):
|
||||
if 'addr_alias' in doc:
|
||||
updateDict['addr_alias'] = doc['addr_alias']
|
||||
if 'lng' in doc and doc['lng'] is not None and str(doc['lng']) != 'nan':
|
||||
updateDict['location'] = {'lon': doc['lng'] / 10**10, 'lat': doc['lat'] / 10**10}
|
||||
updateDict['location'] = {'lon': doc['lng'] / 10 ** 10, 'lat': doc['lat'] / 10 ** 10}
|
||||
updateDict['update_date'] = update_date
|
||||
action = UpdateOne({'_id': doc['_id']}, {'$set': updateDict}, upsert=True)
|
||||
|
||||
@@ -51,14 +54,16 @@ def sync_district_office(type):
|
||||
seaweed_didata_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
|
||||
didata_id_df = pd.DataFrame(list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000)))
|
||||
didata_id_df = pd.DataFrame(
|
||||
list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000)))
|
||||
if len(didata_id_df) > 0:
|
||||
didata_id_df = didata_id_df[['_id']]
|
||||
id_df = pd.DataFrame(id_list)
|
||||
deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False)
|
||||
|
||||
for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows():
|
||||
action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, {'$set': {'status': 1, 'update_date': update_date}}, upsert=False)
|
||||
action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])},
|
||||
{'$set': {'status': 1, 'update_date': update_date}}, upsert=False)
|
||||
actions.append(action)
|
||||
if len(actions) == 10000:
|
||||
seaweed_didata_conn.bulk_write(actions)
|
||||
@@ -69,8 +74,9 @@ def sync_district_office(type):
|
||||
actions.clear()
|
||||
log.info(city + "-" + type + "-同步完成")
|
||||
|
||||
|
||||
if type == 'all':
|
||||
sync_district_office("district")
|
||||
sync_district_office("office")
|
||||
else:
|
||||
sync_district_office(type)
|
||||
sync_district_office(type)
|
||||
|
||||
@@ -110,5 +110,11 @@ with DAG(
|
||||
task_id='other',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
# 统一输出为省市区全拼
|
||||
full_name = BashOperator(
|
||||
task_id='full_name',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/full_name.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
# 人均消费并行,值为空
|
||||
comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> city_level_and_price_rating >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other
|
||||
comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> city_level_and_price_rating >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other \
|
||||
>> full_name
|
||||
|
||||
+2
-2
@@ -1,2 +1,2 @@
|
||||
pip install -r /opt/airflow/dags/repo/dags/requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/
|
||||
pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/
|
||||
pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/
|
||||
pip install -r /opt/airflow/dags/repo/dags/requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/
|
||||
@@ -114,10 +114,10 @@ try:
|
||||
result.append('FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_000000_EVALUATE_RATE_CITY_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_000000_EVALUATE_RATE_CITY_FJW_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_' + date + '_0001.dat文件不存在')
|
||||
# if 'FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
# result.append('FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||
|
||||
|
||||
@@ -1,13 +1,18 @@
|
||||
"""
|
||||
价格异常检测脚本
|
||||
"""
|
||||
from goojia_utils import dateutils, pgsql
|
||||
from goojia_utils import dateutils, pgsql, MAIN_MONGO_CLIENT, data_utils
|
||||
import pandas as pd
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
# 获取独家小区
|
||||
exclusive_seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0, 'mark_map.exclusive': '是'}, {'city': 1, 'region': 1, 'name': 1, 'mark_map.exclusive': 1}).batch_size(100000)))
|
||||
exclusive_seaweed_df = data_utils.flatten_json(exclusive_seaweed_df, 'mark_map')
|
||||
exclusive_seaweed_df = exclusive_seaweed_df.rename(columns={'name': 'commname', 'exclusive': '独家小区'})
|
||||
|
||||
def simple_calc(result_df):
|
||||
"""
|
||||
@@ -111,7 +116,7 @@ for key in six_map:
|
||||
if abs(six_price - five_price) * 100 / five_price > 10:
|
||||
actions.append({'city': info[0], 'region': info[1], 'commid': info[2], 'commname': info[3],
|
||||
'five_price': five_price,'five_calc_type': five_map[key].split("|")[1],
|
||||
'six_price': six_price,'six_calc_type': six_map[key].split("|")[1],
|
||||
'six_price': six_price,'six_calc_type': six_map[key].split("|")[1], '差异值': abs(six_price - five_price) * 100 / five_price,
|
||||
'涨跌幅': (six_price - five_price) * 100 / five_price, 'confidence': six_map[key].split("|")[2]})
|
||||
if info[0] not in city_seaweed_dict:
|
||||
city_seaweed_dict[info[0]] = 1
|
||||
@@ -132,6 +137,7 @@ excel_writer = pd.ExcelWriter(aim_file_name)
|
||||
if type == 'sample':
|
||||
result_df = simple_calc(result_df)
|
||||
|
||||
result_df = pd.merge(result_df, exclusive_seaweed_df, on=['city', 'region', 'commname'], how='left')
|
||||
result_df = result_df.drop_duplicates(subset=['commid'], keep='first')
|
||||
result_df = result_df.rename(columns={'city': '城市', 'region': '区域', 'commid': '小区ID', 'commname': '小区名', 'five_price': '上一月价格',
|
||||
'five_calc_type': '上一月计算方式', 'six_price': '当月价格', 'six_calc_type': '当月计算方式', 'confidence': '置信度'})
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from goojia_utils import ElasticSearchClient,dingutils,dateutils
|
||||
from goojia_utils import ElasticSearchClient, dingutils, dateutils
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import logging
|
||||
@@ -7,22 +7,23 @@ log = logging.getLogger(__name__)
|
||||
|
||||
# 获取当前日期和2个月前的日期,统计2个月样本数据
|
||||
now = datetime.datetime.today()
|
||||
first_day = datetime.datetime(now.year,now.month, 1)
|
||||
first_day = datetime.datetime(now.year, now.month, 1)
|
||||
now_day = first_day.strftime('%Y%m%d')
|
||||
last = dateutils.add_months(first_day,0,-2,0,0)
|
||||
last = dateutils.add_months(first_day, 0, -2, 0, 0)
|
||||
last_day = last.strftime('%Y%m%d')
|
||||
|
||||
log.info(f"系统当前时间 : {now}")
|
||||
|
||||
# 区域维度,统计2个月内样本数据量
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+str(last_day)+' and p_date<'+str(now_day)
|
||||
sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + str(
|
||||
last_day) + ' and p_date<' + str(now_day)
|
||||
log.info(f"城市区域维度 sql : {sql}")
|
||||
df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
df_region = pd.DataFrame(df.groupby(by=['city','region'])['_id'].count()).reset_index()
|
||||
df_region = df_region.rename(columns={'_id':'count'})
|
||||
df_region_res = df_region[df_region['count']<20]
|
||||
df_region = pd.DataFrame(df.groupby(by=['city', 'region'])['_id'].count()).reset_index()
|
||||
df_region = df_region.rename(columns={'_id': 'count'})
|
||||
df_region_res = df_region[df_region['count'] < 20]
|
||||
|
||||
if df_region_res.empty == False:
|
||||
text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
|
||||
@@ -31,7 +32,7 @@ if df_region_res.empty == False:
|
||||
# 当前时间
|
||||
now = datetime.datetime.today()
|
||||
# 获取当前月份的第一天
|
||||
first_day_of_month = datetime.datetime(now.year,now.month, 1)
|
||||
first_day_of_month = datetime.datetime(now.year, now.month, 1)
|
||||
# 获取1个月前的第一天
|
||||
last_month = first_day_of_month - datetime.timedelta(days=1)
|
||||
last_day_of_month = last_month.replace(day=1)
|
||||
@@ -46,29 +47,29 @@ strat_time2 = last2_day_of_month.strftime('%Y%m%d')
|
||||
end_time2 = last_day_of_month.strftime('%Y%m%d')
|
||||
|
||||
# 房源网站维度,按当月数量,和当月数量相比上月减少百分比
|
||||
sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+strat_time+' and p_date<='+end_time
|
||||
sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+strat_time2+' and p_date<='+end_time2
|
||||
sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time + ' and p_date<=' + end_time
|
||||
sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time2 + ' and p_date<=' + end_time2
|
||||
log.info(f"房源网站维度-百分比 当月sql : {sql1}")
|
||||
log.info(f"房源网站维度-百分比 上月sql : {sql2}")
|
||||
df_root1 = es206_client.scan_by_sql_to_data_frame(sql1)
|
||||
df_root2 = es206_client.scan_by_sql_to_data_frame(sql2)
|
||||
|
||||
|
||||
df_root_group1 = pd.DataFrame(df_root1.groupby(by=['root'])['_id'].count()).reset_index()
|
||||
df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).reset_index()
|
||||
|
||||
df_root_group1 = df_root_group1.rename(columns={'_id':'count'})
|
||||
df_root_group2 = df_root_group2.rename(columns={'_id':'count'})
|
||||
df_root_group1 = df_root_group1.rename(columns={'_id': 'count'})
|
||||
df_root_group2 = df_root_group2.rename(columns={'_id': 'count'})
|
||||
|
||||
# 房源网站,百分比维度,减少30%
|
||||
df_root_merge = pd.merge(df_root_group1,df_root_group2,how='left',on=['root'])
|
||||
df_root_merge = pd.merge(df_root_group1, df_root_group2, how='left', on=['root'])
|
||||
|
||||
count_new = 'count_'+strat_time[0:6]
|
||||
count_old = 'count_'+strat_time2[0:6]
|
||||
count_new = 'count_' + strat_time[0:6]
|
||||
count_old = 'count_' + strat_time2[0:6]
|
||||
|
||||
df_root_merge = df_root_merge.rename(columns={'count_x':count_new,'count_y':count_old})
|
||||
df_root_merge = df_root_merge.rename(columns={'count_x': count_new, 'count_y': count_old})
|
||||
|
||||
df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge[count_new]<df_root_merge[count_old]) & ((df_root_merge[count_old]-df_root_merge[count_new])/df_root_merge[count_old]>0.3))].index)
|
||||
df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge[count_new] < df_root_merge[count_old]) & (
|
||||
(df_root_merge[count_old] - df_root_merge[count_new]) / df_root_merge[count_old] > 0.3))].index)
|
||||
|
||||
if df_root_res.empty == False:
|
||||
if text == '':
|
||||
@@ -81,9 +82,10 @@ topCities = cities.head(70)
|
||||
city_list = list(topCities['cityname'])
|
||||
log.info(f"房源网站维度-数量 city_list : {city_list}")
|
||||
|
||||
df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root','city'])['_id'].count()).reset_index()
|
||||
df_root_city_group1 = df_root_city_group1.rename(columns={'_id':'count'})
|
||||
df_root_city_res = df_root_city_group1[(df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count']<100)].reset_index()
|
||||
df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root', 'city'])['_id'].count()).reset_index()
|
||||
df_root_city_group1 = df_root_city_group1.rename(columns={'_id': 'count'})
|
||||
df_root_city_res = df_root_city_group1[
|
||||
(df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count'] < 100)].reset_index()
|
||||
|
||||
if df_root_city_res.empty == False:
|
||||
if text == '':
|
||||
@@ -93,6 +95,7 @@ path = "/home/airflow/data/jh/price"
|
||||
file_name = f"{now_day}房源样本数量检测.xlsx"
|
||||
air_file_name = path + '/' + file_name
|
||||
|
||||
|
||||
def up_to_nextcloud():
|
||||
"""
|
||||
上传统计信息至nextcloud
|
||||
@@ -105,21 +108,17 @@ def up_to_nextcloud():
|
||||
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD)
|
||||
nxc.upload_file(air_file_name, nextcloud_file_name)
|
||||
|
||||
|
||||
if text != '':
|
||||
with pd.ExcelWriter(air_file_name) as writer:
|
||||
if df_region_res.empty == False:
|
||||
df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city","region",'count'], index=False)
|
||||
df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city", "region", 'count'], index=False)
|
||||
if df_root_city_res.empty == False:
|
||||
df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root','city','count'], index=False)
|
||||
df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root', 'city', 'count'],
|
||||
index=False)
|
||||
if df_root_res.empty == False:
|
||||
df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root',count_old,count_new], index=False)
|
||||
df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root', count_old, count_new],
|
||||
index=False)
|
||||
up_to_nextcloud()
|
||||
token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736'
|
||||
dingutils.send_text(text,token)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
dingutils.send_text(text, token)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
elasticsearch==7.11.0
|
||||
transbigdata==0.4.15
|
||||
pykalman==0.9.5
|
||||
pykalman==0.9.5
|
||||
matplotlib
|
||||
@@ -0,0 +1,249 @@
|
||||
"""
|
||||
统计城市 区域 价格涨跌幅
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from sqlalchemy import create_engine
|
||||
from goojia_utils import dateutils
|
||||
import datetime
|
||||
import os
|
||||
pathAll = "/home/airflow/data/cityPrice"
|
||||
if not os.path.exists(pathAll):
|
||||
os.makedirs(pathAll)
|
||||
# 创建数据库引擎
|
||||
engine235 = create_engine("postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5432/fangjia?client_encoding=utf8",
|
||||
echo=False)
|
||||
pd.set_option('mode.chained_assignment', None)
|
||||
year_month_day = dateutils.format_date(datetime.datetime.now())
|
||||
# 执行SQL查询获取城市价格数据
|
||||
city_sql = "SELECT DISTINCT urbn_nm FROM price_stat.avm_city_price"
|
||||
city_names_set = set(pd.read_sql(city_sql, engine235)['urbn_nm'])
|
||||
city_names = list(city_names_set)
|
||||
# 假设这是你的pgsql数据
|
||||
all_city_data = pd.DataFrame()
|
||||
# 执行SQL查询获取区域价格数据
|
||||
region_sql = "SELECT DISTINCT urbn_nm,gxda_admndiv_nm FROM price_stat.avm_district_price"
|
||||
region_data = pd.read_sql(region_sql, engine235)
|
||||
city_data_useinregion = pd.DataFrame()
|
||||
# 假设这是你的pgsql数据
|
||||
all_region_data = pd.DataFrame()
|
||||
|
||||
def up_to_nextcloud(aim_file_name,aim_file_path):
|
||||
"""
|
||||
上传统计信息至nextcloud
|
||||
"""
|
||||
from nextcloud import NextCloud
|
||||
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
|
||||
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
|
||||
NEXTCLOUD_PASSWORD = "goojia123456"
|
||||
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
|
||||
nxc.upload_file(aim_file_name, aim_file_path)
|
||||
def city_increase(city_names, all_city_data):
|
||||
global city_data_useinregion # 声明为全局变量
|
||||
# 遍历城市列表,对每个城市进行查询
|
||||
for city in city_names:
|
||||
query = f"SELECT urbn_nm as 城市, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_city_price WHERE urbn_nm = '{city}'"
|
||||
city_data = pd.read_sql(query, engine235)
|
||||
|
||||
if city_data.empty:
|
||||
continue
|
||||
|
||||
# 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变
|
||||
city_data['original_date'] = city_data['月份']
|
||||
|
||||
# 将日期列转换为日期类型
|
||||
city_data['月份'] = pd.to_datetime(city_data['月份'], format='%Y%m')
|
||||
|
||||
# 对价格数据按照月份排序
|
||||
city_data.sort_values(by='月份', inplace=True)
|
||||
|
||||
# 计算涨跌幅
|
||||
city_data['涨跌幅'] = city_data['估家计算均价'].diff() / city_data['估家计算均价'].shift(1) * 100
|
||||
|
||||
# 计算涨跌幅,并保留两位小数,展示为百分比形式
|
||||
price_change = city_data['估家计算均价'].pct_change() * 100
|
||||
city_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', '')
|
||||
|
||||
# 将原始日期列替换为计算涨跌幅后的日期列
|
||||
city_data['月份'] = city_data['original_date']
|
||||
|
||||
# 删除临时列
|
||||
city_data.drop(columns=['original_date'], inplace=True)
|
||||
|
||||
# 将城市数据添加到总的数据中
|
||||
all_city_data = pd.concat([all_city_data, city_data], ignore_index=True)
|
||||
city_data_useinregion = all_city_data
|
||||
return all_city_data
|
||||
|
||||
|
||||
def city_missing():
|
||||
df = city_increase(city_names, all_city_data)
|
||||
df['月份1'] = df['月份']
|
||||
# 将日期字符串转换为日期时间对象
|
||||
df["月份"] = pd.to_datetime(df["月份"], format="%Y%m")
|
||||
|
||||
# 获取当前时间
|
||||
current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
# 计算前一年到上月的时间范围
|
||||
one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1)
|
||||
last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1)
|
||||
|
||||
# 创建一个包含所有可能月份的日期范围
|
||||
all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS")
|
||||
|
||||
# 获取每个城市缺失的月份
|
||||
missing_months = []
|
||||
for city, group in df.groupby("城市"):
|
||||
available_months = group["月份"]
|
||||
missing = all_months.difference(available_months)
|
||||
if len(missing) > 0:
|
||||
missing_str = ",".join([m.strftime("%Y%m") for m in missing])
|
||||
missing_months.append({"城市": city, "缺失月份": missing_str})
|
||||
if len(missing_months) > 0:
|
||||
# 创建DataFrame来存储结果
|
||||
missing_df = pd.DataFrame(missing_months)
|
||||
path = pathAll+'/城市价格数据缺失'+year_month_day+'.xlsx'
|
||||
missing_df.to_excel(path, index=False)
|
||||
logging.info(f"城市价格缺失数据已写入Excel文件:{path}")
|
||||
|
||||
cloudPath = '共享/统计/城市区域涨跌幅/' + '城市价格数据缺失'+year_month_day+'.xlsx'
|
||||
up_to_nextcloud(path, cloudPath)
|
||||
logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘")
|
||||
|
||||
# print(missing_df)
|
||||
df['月份'] = df['月份1']
|
||||
# 删除月份1列
|
||||
df.drop(columns=['月份1'], inplace=True)
|
||||
output_path = pathAll+'/城市价格涨跌幅'+year_month_day+'.xlsx'
|
||||
df.to_excel(output_path, index=False)
|
||||
logging.info(f"城市涨跌幅数据已写入Excel文件:{output_path}")
|
||||
|
||||
cloudPath = '共享/统计/城市区域涨跌幅/' + '城市价格涨跌幅'+year_month_day+'.xlsx'
|
||||
up_to_nextcloud(output_path, cloudPath)
|
||||
logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘")
|
||||
|
||||
|
||||
def region_increase(region_data, all_region_data):
|
||||
# 遍历区域列表,对每个区域进行查询
|
||||
for index, row in region_data.iterrows():
|
||||
city = row['urbn_nm']
|
||||
region = row['gxda_admndiv_nm']
|
||||
query = f"SELECT urbn_nm as 城市, gxda_admndiv_nm as 区域, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_district_price WHERE urbn_nm = '{city}' AND gxda_admndiv_nm = '{region}'"
|
||||
region_data = pd.read_sql(query, engine235)
|
||||
|
||||
if region_data.empty:
|
||||
continue
|
||||
|
||||
# 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变
|
||||
region_data['original_date'] = region_data['月份']
|
||||
|
||||
# 将日期列转换为日期类型
|
||||
region_data['月份'] = pd.to_datetime(region_data['月份'], format='%Y%m')
|
||||
|
||||
# 对价格数据按照月份排序
|
||||
region_data.sort_values(by='月份', inplace=True)
|
||||
|
||||
# 计算涨跌幅
|
||||
region_data['涨跌幅'] = region_data['估家计算均价'].diff() / region_data['估家计算均价'].shift(1) * 100
|
||||
|
||||
# 计算涨跌幅,并保留两位小数,展示为百分比形式
|
||||
price_change = region_data['估家计算均价'].pct_change() * 100
|
||||
region_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%',
|
||||
'')
|
||||
|
||||
# 将原始日期列替换为计算涨跌幅后的日期列
|
||||
region_data['月份'] = region_data['original_date']
|
||||
|
||||
# 删除临时列
|
||||
region_data.drop(columns=['original_date'], inplace=True)
|
||||
|
||||
# 将区域数据添加到总的数据中
|
||||
all_region_data = pd.concat([all_region_data, region_data], ignore_index=True)
|
||||
|
||||
return all_region_data
|
||||
|
||||
|
||||
# 城市价个和区域价格比较
|
||||
def add_price_comparison_by_month(region_data):
|
||||
# 遍历区域数据,比较城市和区域的价格,并添加比值超过3倍的情况到区域数据中
|
||||
for index, row in region_data.iterrows():
|
||||
city = row['城市']
|
||||
region = row['区域']
|
||||
region_month = row['月份']
|
||||
region_price = row['估家计算均价']
|
||||
city_data = city_data_useinregion
|
||||
# 获取城市对应月份的价格
|
||||
city_price_row = city_data[(city_data['城市'] == city) & (city_data['月份'] == region_month)]
|
||||
if city_price_row.empty:
|
||||
continue
|
||||
city_price = city_price_row['估家城市均价'].iloc[0]
|
||||
|
||||
# 比较价格并计算比值
|
||||
price_ratio = max(city_price, region_price) / min(city_price, region_price)
|
||||
|
||||
# 如果比值超过3倍,将结果添加到区域数据中
|
||||
if price_ratio > 3:
|
||||
region_data.loc[index, '城市区域价格比值超3倍'] = price_ratio
|
||||
|
||||
return region_data
|
||||
|
||||
|
||||
def region_missing():
|
||||
df = region_increase(region_data, all_region_data)
|
||||
df['月份1'] = df['月份']
|
||||
# 将日期字符串转换为日期时间对象
|
||||
df["月份"] = pd.to_datetime(df["月份"], format="%Y%m")
|
||||
|
||||
# 获取当前时间
|
||||
current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
# 计算前一年到上月的时间范围
|
||||
one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1)
|
||||
last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1)
|
||||
|
||||
# 创建一个包含所有可能月份的日期范围
|
||||
all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS")
|
||||
|
||||
# 获取每个城市-区域组合缺失的月份
|
||||
missing_months = []
|
||||
for (city, region), group in df.groupby(["城市", "区域"]):
|
||||
available_months = group["月份"]
|
||||
missing = all_months.difference(available_months)
|
||||
if len(missing) > 0:
|
||||
missing_str = ",".join([m.strftime("%Y%m") for m in missing])
|
||||
missing_months.append({"城市": city, "区域": region, "缺失月份": missing_str})
|
||||
|
||||
if missing_months:
|
||||
# 创建DataFrame来存储结果
|
||||
missing_df = pd.DataFrame(missing_months)
|
||||
path = pathAll+'/区域价格数据缺失'+year_month_day+'.xlsx'
|
||||
missing_df.to_excel(path, index=False)
|
||||
logging.info(f"区域价格缺失数据已写入Excel文件:{path}")
|
||||
cloudPath = '共享/统计/城市区域涨跌幅/' + '区域价格数据缺失' + year_month_day + '.xlsx'
|
||||
up_to_nextcloud(path, cloudPath)
|
||||
logging.info(f"区域价格缺失数据Excel文件已经上传到云盘")
|
||||
|
||||
# 如果区域在 missing_df 中,则将 df 中的该区域数据删除
|
||||
for city, region in zip(missing_df['城市'], missing_df['区域']):
|
||||
df = df[(df['城市'] != city) | (df['区域'] != region)]
|
||||
|
||||
df['月份'] = df['月份1']
|
||||
# 删除月份1列
|
||||
df.drop(columns=['月份1'], inplace=True)
|
||||
|
||||
# 添加价格比值超过3倍的列
|
||||
# print(city_data_useinregion)
|
||||
df = add_price_comparison_by_month(df)
|
||||
|
||||
output_path = pathAll+'/区域价格涨跌幅'+year_month_day+'.xlsx'
|
||||
df.to_excel(output_path, index=False)
|
||||
logging.info(f"区域涨跌幅数据已写入Excel文件:{output_path}")
|
||||
|
||||
cloudPath = '共享/统计/城市区域涨跌幅/' + '区域价格涨跌幅'+year_month_day+'.xlsx'
|
||||
up_to_nextcloud(output_path, cloudPath)
|
||||
logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘")
|
||||
|
||||
|
||||
city_missing()
|
||||
region_missing()
|
||||
@@ -0,0 +1,97 @@
|
||||
import logging
|
||||
import pandas as pd
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
# 创建数据库引擎
|
||||
engine235 = create_engine("postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5432/fangjia?client_encoding=utf8",
|
||||
echo=False)
|
||||
|
||||
# 执行SQL查询获取区域价格数据
|
||||
region_sql = "SELECT DISTINCT urbn_nm,gxda_admndiv_nm FROM price_stat.avm_district_price_copy1"
|
||||
region_data = pd.read_sql(region_sql, engine235)
|
||||
|
||||
# 假设这是你的pgsql数据
|
||||
all_region_data = pd.DataFrame()
|
||||
pd.set_option('mode.chained_assignment', None)
|
||||
|
||||
def region_increase(region_data, all_region_data):
|
||||
# 遍历区域列表,对每个区域进行查询
|
||||
for index, row in region_data.iterrows():
|
||||
city = row['urbn_nm']
|
||||
region = row['gxda_admndiv_nm']
|
||||
query = f"SELECT urbn_nm as 城市, gxda_admndiv_nm as 区域, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_district_price_copy1 WHERE urbn_nm = '{city}' AND gxda_admndiv_nm = '{region}'"
|
||||
region_data = pd.read_sql(query, engine235)
|
||||
|
||||
if region_data.empty:
|
||||
continue
|
||||
|
||||
# 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变
|
||||
region_data['original_date'] = region_data['月份']
|
||||
|
||||
# 将日期列转换为日期类型
|
||||
region_data['月份'] = pd.to_datetime(region_data['月份'], format='%Y%m')
|
||||
|
||||
# 对价格数据按照月份排序
|
||||
region_data.sort_values(by='月份', inplace=True)
|
||||
|
||||
# 计算涨跌幅
|
||||
region_data['涨跌幅'] = region_data['估家计算均价'].diff() / region_data['估家计算均价'].shift(1) * 100
|
||||
|
||||
# 计算涨跌幅,并保留两位小数,展示为百分比形式
|
||||
price_change = region_data['估家计算均价'].pct_change() * 100
|
||||
region_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', '')
|
||||
|
||||
# 将原始日期列替换为计算涨跌幅后的日期列
|
||||
region_data['月份'] = region_data['original_date']
|
||||
|
||||
# 删除临时列
|
||||
region_data.drop(columns=['original_date'], inplace=True)
|
||||
|
||||
# 将区域数据添加到总的数据中
|
||||
all_region_data = pd.concat([all_region_data, region_data], ignore_index=True)
|
||||
|
||||
return all_region_data
|
||||
|
||||
def region_missing():
|
||||
df = region_increase(region_data, all_region_data)
|
||||
df['月份1'] = df['月份']
|
||||
# 将日期字符串转换为日期时间对象
|
||||
df["月份"] = pd.to_datetime(df["月份"], format="%Y%m")
|
||||
|
||||
# 获取当前时间
|
||||
current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
# 计算前一年到上月的时间范围
|
||||
one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1)
|
||||
last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1)
|
||||
|
||||
# 创建一个包含所有可能月份的日期范围
|
||||
all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS")
|
||||
|
||||
# 获取每个城市-区域组合缺失的月份
|
||||
missing_months = []
|
||||
for (city, region), group in df.groupby(["城市", "区域"]):
|
||||
available_months = group["月份"]
|
||||
missing = all_months.difference(available_months)
|
||||
if len(missing) > 0:
|
||||
missing_str = ",".join([m.strftime("%Y%m") for m in missing])
|
||||
missing_months.append({"城市": city, "区域": region, "缺失月份": missing_str})
|
||||
|
||||
if missing_months:
|
||||
# 创建DataFrame来存储结果
|
||||
missing_df = pd.DataFrame(missing_months)
|
||||
path = 'E:\\goojia\\file\\excel文件\\airflow\\missing_region.xlsx'
|
||||
missing_df.to_excel(path, index=False)
|
||||
|
||||
# 如果区域在 missing_df 中,则将 df 中的该区域数据删除
|
||||
for city, region in zip(missing_df['城市'], missing_df['区域']):
|
||||
df = df[(df['城市'] != city) | (df['区域'] != region)]
|
||||
|
||||
df['月份'] = df['月份1']
|
||||
# 删除月份1列
|
||||
df.drop(columns=['月份1'], inplace=True)
|
||||
output_path = 'E:\\goojia\\file\\excel文件\\airflow\\all_region_price_with_change.xlsx'
|
||||
df.to_excel(output_path, index=False)
|
||||
logging.info(f"区域涨跌幅数据已写入Excel文件:{output_path}")
|
||||
|
||||
region_missing()
|
||||
@@ -0,0 +1,52 @@
|
||||
"""
|
||||
小区周边1公里交通地铁入库业务逻辑
|
||||
"""
|
||||
import datetime
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT
|
||||
from goojia_utils import dateutils
|
||||
from sklearn.mixture import GaussianMixture
|
||||
from pymongo import InsertOne, DeleteMany, ReplaceOne, UpdateOne
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat')
|
||||
district_analy_data_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_analy_data')
|
||||
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
|
||||
weekend = list(district_stat_conn.find().sort([("weekend", -1)]).limit(1))[0]['weekend']
|
||||
level_map = {"1": "S", "2": "A", "3": "B", "4": "C", "5": "D"}
|
||||
logger.info(f"开始计算楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}")
|
||||
for server_area in server_area_conn.find({'display': {'$ne': '全国'}}, {"display": 1}):
|
||||
city = server_area["display"]
|
||||
data = pd.DataFrame(
|
||||
district_stat_conn.find({"city": city, "weekend": weekend},
|
||||
{"_id": 0, "region": 1, "district_name": 1, "avg_price": 1}))
|
||||
if "avg_price" not in data or len(data['avg_price'].unique()) < 5:
|
||||
continue
|
||||
data['city'] = city
|
||||
logger.info(f"开始计算{city}楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}")
|
||||
# 高斯混合模型
|
||||
gmm = GaussianMixture(n_components=5, covariance_type='tied', max_iter=200, n_init=10).fit(
|
||||
data[["avg_price"]])
|
||||
data.loc[:, 'cluster'] = gmm.predict(data[["avg_price"]])
|
||||
data.loc[:, 'price_mean_cluster'] = data.groupby('cluster')["avg_price"].transform('mean')
|
||||
data.loc[:, "target"] = data['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
||||
data = data.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||
data["community_rating"] = data['target'].apply(lambda x: level_map[str(x)])
|
||||
opts = []
|
||||
for item in data.to_dict('records'):
|
||||
opts.append(UpdateOne({"city": city, "region": item["region"], "district_name": item["district_name"]},
|
||||
{"$set": {"community_rating": item["community_rating"]}}, upsert=True
|
||||
))
|
||||
# 按照每 1000 条分割 updates 列表
|
||||
batch_size = 1000
|
||||
batches = [opts[i:i + batch_size] for i in range(0, len(opts), batch_size)]
|
||||
# 逐个批次执行批量更新操作
|
||||
cnt = 0
|
||||
for batch in batches:
|
||||
cnt = cnt + 1
|
||||
logger.info(f"{city}楼盘评级更新数据库, 进度:{cnt}/{len(batches)}")
|
||||
result = district_analy_data_conn.bulk_write(batch)
|
||||
print(f"当前批次更新的文档数量: {result.modified_count}")
|
||||
logger.info(f"楼盘评级计算结束{dateutils.format_date(datetime.datetime.utcnow(), '%Y-%m-%d')}")
|
||||
@@ -0,0 +1,17 @@
|
||||
from airflow import DAG
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
|
||||
with DAG(
|
||||
dag_id="community_rating_dag",
|
||||
# 每月的最后一天运行
|
||||
schedule_interval="0 20 * * 6",
|
||||
start_date=pendulum.datetime(2024, 7, 12, tz="Asia/Shanghai"),
|
||||
description="小区楼盘评级计算",
|
||||
catchup=True
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='community_rating',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/stat/community/rate.py'
|
||||
)
|
||||
t1
|
||||
@@ -0,0 +1,17 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="increase_decrease_city_region_dag",
|
||||
# 每月的最后一天运行
|
||||
schedule_interval="0 10 L * *",
|
||||
start_date=pendulum.datetime(2024, 5, 30, tz="Asia/Shanghai"),
|
||||
description="城市区域涨跌幅数据统计",
|
||||
catchup=True
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='increase_decrease_city_region',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/stat/cityPrice/increase_decrease_city_region.py',
|
||||
)
|
||||
t1
|
||||
Reference in New Issue
Block a user