Merge remote-tracking branch 'origin/master'

This commit is contained in:
Jian Shen
2023-03-11 10:24:50 +08:00
33 changed files with 880 additions and 568 deletions
@@ -0,0 +1,70 @@
"""
新房变二手房定时脚本
创建年代、入住/交付时间三年后,或者有样本就变成二手房
"""
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
from pymongo import UpdateOne
import pandas as pd
import logging
import datetime
import time
log = logging.getLogger(__name__)
def do():
log.info("开始执行新房变二手房脚本")
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'is_new': 1},
{'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'open_date': 1, 'live_date': 1,
'create_year': 1}).batch_size(100000)))
if len(df) <= 0:
log.info("执行新房变二手房脚本完毕,未发现新房")
return False
actions = list()
count = 0
def deal(x):
global count
is_old = False
time_now = int(time.mktime(time.strptime(datetime.datetime.now().strftime('%Y-%m-%d'), '%Y-%m-%d')))
if 'open_date' in x and pd.notna(x['open_date']):
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(x['open_date'], years=3), '%Y-%m-%d'), '%Y-%m-%d')))
if time_old < time_now:
is_old = True
if is_old is False and 'live_date' in x and pd.notna(x['live_date']):
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['live_date']), years=3), '%Y-%m-%d'), '%Y-%m-%d')))
if time_old < time_now:
is_old = True
if is_old is False and 'create_year' in x and pd.notna(x['create_year']):
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['create_year'], '%Y-%m'), years=3), '%Y-%m-%d'), '%Y-%m-%d')))
if time_old < time_now:
is_old = True
# 查询样本库
names = [x['name']]
if type(x['alias']) == list:
for alias_item in x['alias']:
names.append(alias_item)
names_str = "','".join(names)
names_str = "'" + names_str + "'"
if is_old is False:
nums = commutils.get_sample(x['city'], x['region'], names_str)
if nums > 0:
is_old = True
if is_old:
actions.append(UpdateOne({'_id': x['_id']}, {'$set': {'is_new': 0, 'm_date': datetime.datetime.now()}}))
if len(actions) == 5000:
seaweed_conn.bulk_write(actions)
count += len(actions)
actions.clear()
df.apply(lambda x: deal(x), axis=1)
if len(actions) > 0:
seaweed_conn.bulk_write(actions)
count += len(actions)
actions.clear()
log.info(f"执行新房变二手房脚本完毕,共更新{count}")
do()
+76 -72
View File
@@ -5,43 +5,67 @@ import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
from elasticsearch import helpers
import numpy as np
from concurrent.futures import ThreadPoolExecutor
import sys
from threading import Lock
mutex = Lock()
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
m_date = dateutils.iso_format()
path = "/home/airflow/data/community"
file_path = "/opt/airflow/dags/repo/dags/business/business/community/data"
avg_cost_min_max_df = pd.read_excel(file_path + '/全国各区域人均消费中位值.xlsx')
avg_cost_city_region_dict = dict()
def find_around_price(lng, lat):
# 加载各区域人均消费最大最小值
def deal_avg_cost(x):
city_now = x['city']
region_now = x['region']
if pd.notna(city_now) and pd.notna(region_now):
avg_cost_city_region_dict[city_now + region_now] = {'min': x['3'], 'max': x['97']}
elif pd.notna(city_now):
avg_cost_city_region_dict[city_now] = {'min': x['3'], 'max': x['97']}
avg_cost_min_max_df.apply(lambda x: deal_avg_cost(x), axis=1)
def find_around_price(seaweed):
"""
查找周边均价
返回取值为10~300,为所有有价格的1-99百分位
"""
if pd.isna(lng):
return None
lng = commutils.format_lat_lng(seaweed['lng'])
lat = commutils.format_lat_lng(seaweed['lat'])
city_short = seaweed['city']
region_short = seaweed['region']
city_display = areautils.get_city_display(city_short)
filter_query = [{"terms": {"tag": ["美食;中餐厅", "美食;外国餐厅"]}}]
if pd.notna(city_display):
filter_query.append({"term": {"city": {"value": city_display}}})
min_now = 10
max_now = 300
if (city_short+region_short) in avg_cost_city_region_dict:
min_now = avg_cost_city_region_dict[city_short + region_short]['min']
max_now = avg_cost_city_region_dict[city_short + region_short]['max']
elif city_short in avg_cost_city_region_dict:
min_now = avg_cost_city_region_dict[city_short]['min']
max_now = avg_cost_city_region_dict[city_short]['max']
filter_query.append({"range": {"price": {"gte": min_now, "lte": max_now}}})
filter_query.append({"geo_distance": {"distance": 1000, "location": [lng, lat]}})
ret_es = es206_client.elastic_client.search({
"query": {
"bool": {
"must": [
{
"term": {
"status": {
"value": 0
}
}
}
],
"filter": [
{
"geo_distance": {
"distance": 1000,
"location": [lng, lat]
}
}
]
"filter": filter_query
}
},
"size": 0,
@@ -58,72 +82,51 @@ def find_around_price(lng, lat):
def build_doc(item):
doc_dict = dict()
if 'avg_cost' in item and pd.notna(item['avg_cost']):
doc_dict['avg_cost'] = item['avg_cost']
if 'avg_cost' in item:
if pd.notna(item['avg_cost']) or item['avg_cost'] > 0:
doc_dict['avg_cost'] = int(item['avg_cost'])
else:
doc_dict['avg_cost'] = None
return doc_dict
def update_status(new_df):
actions = list()
find_count = 0
def deal(item):
action_doc = {
"_op_type": "update",
"_index": "bd_poi",
"_id": item['_id'],
"doc": {'status': 0}
}
actions.append(action_doc)
if len(actions) == 5000:
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
return item
new_df.apply(lambda x: deal(x), axis=1)
if len(actions) > 0:
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
def find_around_price_item(seaweed):
global find_count
find_count += 1
if find_count % 1000 == 0:
log.info("已查找" + str(find_count))
if 'lng' not in seaweed:
return
price = find_around_price(seaweed)
mutex.acquire(timeout=60)
if pd.notna(price):
seaweed['avg_cost'] = int(price)
else:
seaweed['avg_cost'] = None
mutex.release()
def sync(city=None):
log.info("开始同步消费数据...")
query_sql = "select id, city, tag, type, price from bd_poi where price > 0 and tag in('美食;中餐厅', '美食;外国餐厅') "
if pd.notna(city):
query_sql += " and city='" + areautils.get_city_display(city) + "'"
df = es206_client.scan_by_sql_to_data_frame(query_sql)
# 按百分位5-95筛选数据
cities = df['city'].unique()
for city_now in cities:
df.loc[df['city'] == city_now, 'minPrice5'] = np.percentile(a=df[df['city'] == city_now]['price'], q=5, axis=0)
df.loc[df['city'] == city_now, 'minPrice95'] = np.percentile(a=df[df['city'] == city_now]['price'], q=95, axis=0)
new_df = df[(df['price'] >= df['minPrice5']) & (df['price'] <= df['minPrice95'])]
new_df['new_mean'] = new_df.groupby(by=['city'])['price'].transform(np.mean)
# 按最新的百分位更新库中价格状态
update_status(new_df)
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
def find_around_price_item(index, item):
if index % 1000 == 0:
log.info("已查找" + str(index))
if 'lng' not in item:
return
lng = commutils.format_lat_lng(item['lng'])
lat = commutils.format_lat_lng(item['lat'])
price = find_around_price(lng, lat)
seaweed_df.loc[index, 'avg_cost'] = price
seaweed_list = list()
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
with ThreadPoolExecutor(max_workers=30) as pool:
for index, item in seaweed_df.iterrows():
pool.submit(find_around_price_item, index, item)
# for index, item in seaweed_df.iterrows():
# find_around_price_item(index, item)
for seaweed in seaweed_list:
pool.submit(find_around_price_item, seaweed)
# for seaweed in seaweed_list:
# find_around_price_item(seaweed)
seaweed_df = pd.DataFrame(seaweed_list)
actions = list()
count = 0
for index, item in seaweed_df.iterrows():
@@ -150,4 +153,5 @@ def sync(city=None):
log.info("完成同步消费数据...")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -3,10 +3,11 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -104,4 +105,5 @@ def sync(city=None):
log.info(f"完成同步商圈至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+7 -2
View File
@@ -7,6 +7,7 @@ import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
import os
import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -93,7 +94,10 @@ def sync(city=None):
'estate_charge': 1, 'finish_building_date': 1})
.batch_size(100000)))
community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}")
sql = f"select id from {index_name} "
if pd.notna(city):
sql += f" where city='{city}'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
if len(community_summary_df) > 0:
@@ -154,4 +158,5 @@ def sync(city=None):
log.info(f"完成同步小区基本属性至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -8,6 +8,7 @@ import logging
import os
from elasticsearch import helpers
import matplotlib.pyplot as plt
import sys
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import QuantileTransformer
@@ -50,9 +51,19 @@ def calc_consumption_potential(city):
np.mean)
community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.std)
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(
# 噪音词处理
community_summary_df['avg_cost_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform(
community_summary_df[['avg_cost']])
# 结合全国物业费情况,对于7以上的占比1%
community_summary_df.loc[community_summary_df['estate_charge'] > 7, 'estate_charge'] = 7
community_summary_df['estate_charge_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform(
community_summary_df[['estate_charge']])
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge_fixed'].transform(
np.mean)
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge_fixed'].transform(
np.std)
community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])[
@@ -60,8 +71,8 @@ def calc_consumption_potential(city):
community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])[
'relative_price_ranking'].transform(np.std)
community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.mean)
community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.std)
community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost_fixed'].transform(np.mean)
community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost_fixed'].transform(np.std)
# 和田等小城市均值可能为0
if community_summary_df['total_price_std'].unique()[0] != 0:
@@ -70,7 +81,7 @@ def calc_consumption_potential(city):
community_summary_df['total_price_score'] = 0
if community_summary_df['estate_charge_std'].unique()[0] != 0:
community_summary_df['estate_charge_score'] = 139.1374524 * (community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std']
community_summary_df['estate_charge_score'] = 139.1374524 * (community_summary_df['estate_charge_fixed'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std']
else:
community_summary_df['estate_charge_score'] = 0
@@ -80,7 +91,7 @@ def calc_consumption_potential(city):
community_summary_df['relative_price_ranking_score'] = 0
if community_summary_df['avg_cost_std'].unique()[0] != 0:
community_summary_df['avg_cost_score'] = 67.0195871700 * (community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std']
community_summary_df['avg_cost_score'] = 67.0195871700 * (community_summary_df['avg_cost_fixed'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std']
else:
community_summary_df['avg_cost_score'] = 0
@@ -249,5 +260,7 @@ def save_png(community_city):
# 关窗口
plt.close()
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
#dingutils.send_text("【分布图生成完毕】财力评级已完成, 请点击 https://nextcloud.fangjia.com/s/Jqs5BzXi8xLYeH9 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
@@ -3,10 +3,11 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -62,11 +63,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}")
seaweed_query = {'status': 0, 'cat': 'district'}
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -83,4 +83,5 @@ def sync(city=None):
log.info(f"完成同步教育资源至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -3,10 +3,11 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -95,4 +96,5 @@ def sync(city=None):
log.info(f"完成同步教育资源至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -4,14 +4,17 @@
from goojia_utils import ElasticSearchClient, areautils, dateutils
import pandas as pd
import logging
import sys
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
mutex = Lock()
def save(community_summary_df):
if 'estate_charge' not in community_summary_df.columns:
@@ -109,7 +112,7 @@ def sync_by_city(city_now):
log.info(f"{city_now}开始补充物业费....")
sql = "select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='" + city_now + "' and (estate_charge is null or estate_charge_flag=1)"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
with ThreadPoolExecutor(max_workers=50) as pool:
with ThreadPoolExecutor(max_workers=20) as pool:
def deal(index, item):
if index % 1000 == 0 and index > 0:
log.info(f"已查找{index}")
@@ -126,7 +129,9 @@ def sync_by_city(city_now):
# 取周边均值
estate_charge = docs['estate_charge'].mean()
if pd.notna(estate_charge):
mutex.acquire(timeout=60)
community_summary_df.loc[index, "estate_charge"] = estate_charge
mutex.release()
for index, item in community_summary_df.iterrows():
pool.submit(deal, index, item)
save(community_summary_df)
@@ -140,4 +145,5 @@ def sync(city=None):
sync_by_city(city)
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -5,6 +5,7 @@ from goojia_utils import ElasticSearchClient, nextcloudutils
import pandas as pd
import logging
import os
import sys
from elasticsearch import helpers
import matplotlib.pyplot as plt
@@ -163,6 +164,6 @@ def save_png(community_city):
plt.close()
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
#dingutils.send_text("【分布图生成完毕】物业费评级已完成, 请点击 https://nextcloud.fangjia.com/s/HbifLoEDc5wi8HR 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
+3 -1
View File
@@ -14,6 +14,7 @@ import pandas as pd
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -140,4 +141,5 @@ def sync(city=None):
log.info(f"完成同步附加标签至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+4 -2
View File
@@ -3,10 +3,11 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -96,4 +97,5 @@ def sync(city=None):
log.info(f"完成同步生活配套至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -1
View File
@@ -7,6 +7,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -83,4 +84,5 @@ def sync(city=None):
log.info(f"完成同步医院至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -2
View File
@@ -5,7 +5,7 @@ from goojia_utils import ElasticSearchClient, dateutils
import logging
import pandas as pd
from elasticsearch import helpers
import datetime
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -57,4 +57,5 @@ def sync(city=None):
log.info(f"完成同步售租比、人口数至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -0,0 +1,79 @@
"""
人口评级
"""
from goojia_utils import ElasticSearchClient
import pandas as pd
import logging
from elasticsearch import helpers
import sys
from sklearn.mixture import GaussianMixture
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
def gmm_train(total_rate, consumption_df, source, target):
# 高斯混合模型
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]])
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
return consumption_df
def calc_person_rating(city=None):
"""
计算消费等级
"""
sql = "select id, city, region, name, household_count from community_summary"
if pd.notna(city):
sql += " where city='" + city + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
consumption_aim_df = community_summary_df[['_id', 'city', 'household_count']]
consumption_aim_df = consumption_aim_df.dropna(subset=['household_count'])
cities = consumption_aim_df['city'].unique()
source = "household_count"
target = "person_rating"
actions = list()
count = 0
for city in cities:
df = consumption_aim_df[consumption_aim_df['city'] == city].copy()
if len(df) < 5:
continue
df = gmm_train(5, df, source, target)
# 批量更新
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['person_rating'] = item['person_rating']
action = {'_op_type': 'update',
'_index': index_name,
'_id': _id, # 请勿忘记ID
'doc': row}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级中: 当前已评级{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级完成,共更新: {}".format(count))
def sync(city=None):
calc_person_rating(city)
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+62 -55
View File
@@ -3,8 +3,9 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
from elasticsearch import helpers
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -37,78 +38,69 @@ def build_doc(item):
return doc_dict
def sync(city=None):
log.info(f"开始同步价格信息至206 {index_name}")
if pd.notna(city):
cities = [city]
else:
cities = None
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
def sync_by_city(city, total_seaweed_df):
seaweed_df = total_seaweed_df[total_seaweed_df['city'] == city].copy()
if len(seaweed_df) == 0:
return
seaweed_df = seaweed_df.reset_index(drop=True)
# 小区所在城市均价
log.info('开始导出小区所在城市均价')
city_price_df = commutils.get_city_price(city=city)
city_price_df = city_price_df[['city', 'avg_price']]
city_price_df = city_price_df.rename(columns={'avg_price': 'city_avg_price'})
seaweed_df = pd.merge(seaweed_df, city_price_df, how='left', on=['city'])
log.info('小区所在城市均价导出完毕')
if len(city_price_df) > 0:
city_price_df = city_price_df[['city', 'avg_price']]
city_price_df = city_price_df.rename(columns={'avg_price': 'city_avg_price'})
seaweed_df = pd.merge(seaweed_df, city_price_df, how='left', on=['city'])
# 小区所在区域均价
log.info('开始导出小区所在区域均价')
region_price_df = commutils.get_region_price(city=city)
region_price_df = region_price_df[['city', 'region', 'avg_price']]
region_price_df = region_price_df.rename(columns={'avg_price': 'region_avg_price'})
seaweed_df = pd.merge(seaweed_df, region_price_df, how='left', on=['city', 'region'])
log.info('小区所在区域均价导出完毕')
if len(region_price_df) > 0:
region_price_df = region_price_df[['city', 'region', 'avg_price']]
region_price_df = region_price_df.rename(columns={'avg_price': 'region_avg_price'})
seaweed_df = pd.merge(seaweed_df, region_price_df, how='left', on=['city', 'region'])
# 小区均价
log.info('开始导出小区均价')
avg_price_df = commutils.get_avg_price(city=city)
avg_price_df = avg_price_df[['city', 'district_name', 'region', 'avg_price']]
avg_price_df = avg_price_df.rename(columns={'district_name': 'name'})
seaweed_df = pd.merge(seaweed_df, avg_price_df, how='left', on=['city', 'region', 'name'])
log.info('小区均价导出完毕')
if len(avg_price_df) > 0:
avg_price_df = avg_price_df[['city', 'district_name', 'region', 'avg_price']]
avg_price_df = avg_price_df.rename(columns={'district_name': 'name'})
seaweed_df = pd.merge(seaweed_df, avg_price_df, how='left', on=['city', 'region', 'name'])
# 小区均价区域排名
seaweed_df['avg_price_region_ranking'] = seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
# 小区均价城市排名
seaweed_df['avg_price_ranking'] = seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
# 小区均价区域排名
seaweed_df['avg_price_region_ranking'] = \
seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
# 小区均价城市排名
seaweed_df['avg_price_ranking'] = \
seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
# 12个月均价涨跌幅
log.info('开始导出12个月均价涨跌幅')
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=cities, months=-12)
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(columns={'district_name': 'name', 'up_and_downs': 'year_on_year'})
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
log.info('12个月均价涨跌幅导出完毕')
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-12)
if len(avg_price_up_and_downs_df) > 0:
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
columns={'district_name': 'name', 'up_and_downs': 'year_on_year'})
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
# 近半年均价涨跌幅
log.info('开始导出近半年均价涨跌幅')
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=cities, months=-6)
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
columns={'district_name': 'name', 'up_and_downs': 'half_on_year'})
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
log.info('近半年均价涨跌幅导出完毕')
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-6)
if len(avg_price_up_and_downs_df) > 0:
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
columns={'district_name': 'name', 'up_and_downs': 'half_on_year'})
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
# 平米租金
log.info('开始导出平米租金')
rent_price_df = commutils.get_rent_price(city=city)
rent_price_df = rent_price_df[['city', 'region', 'district_name', 'avg_price']]
rent_price_df = rent_price_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_price'})
seaweed_df = pd.merge(seaweed_df, rent_price_df, how='left', on=['city', 'region', 'name'])
log.info('平米租金导出完毕')
if len(rent_price_df) > 0:
rent_price_df = rent_price_df[['city', 'region', 'district_name', 'avg_price']]
rent_price_df = rent_price_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_price'})
seaweed_df = pd.merge(seaweed_df, rent_price_df, how='left', on=['city', 'region', 'name'])
# 政府指导价
log.info('开始导出政府指导价')
gov_price_df = commutils.get_gov_guide_price()
gov_price_df = gov_price_df[['city', 'region', 'name', 'avg_price']]
gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'})
seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name'])
log.info('政府指导价导出完毕')
gov_price_df = commutils.get_gov_guide_price(city=city)
if len(gov_price_df) > 0:
gov_price_df = gov_price_df[['city', 'region', 'name', 'avg_price']]
gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'})
seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name'])
actions = list()
count = 0
@@ -131,7 +123,22 @@ def sync(city=None):
if len(actions) > 0:
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
def sync(city=None):
log.info(f"开始同步价格信息至206 {index_name}")
if pd.notna(city):
cities = [city]
else:
cities = areautils.get_all_cities()
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
for city in cities:
log.info(f"开始同步{city}")
sync_by_city(city, seaweed_df)
log.info(f"完成同步{city}")
log.info(f"完成同步价格信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -8,6 +8,7 @@ from goojia_utils import ElasticSearchClient
import pandas as pd
import logging
from elasticsearch import helpers
import sys
from sklearn.mixture import GaussianMixture
@@ -16,12 +17,13 @@ log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
def gmm_train(total_rate, consumption_df, source, target):
# 高斯混合模型
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
consumption_df['cluster'] = gmm.predict(consumption_df[[source]])
consumption_df['price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df[target] = consumption_df['price_mean_cluster'].rank(method='dense').astype(int)
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]])
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
return consumption_df
@@ -46,12 +48,12 @@ def calc_price_rating(city=None):
for city in cities:
df = consumption_aim_df[consumption_aim_df['city'] == city].copy()
if len(df) < 10:
if len(df) < 5:
continue
df = gmm_train(10, df, source, target)
if len(df) >= 100:
df = gmm_train(100, df, source, score_target)
df['price_score'] /= 10
df = gmm_train(5, df, source, target)
if len(df) >= 5:
df = gmm_train(5, df, source, score_target)
df['price_score'] /= 5
# 批量更新
for index, item in df.iterrows():
@@ -82,4 +84,5 @@ def sync(city=None):
calc_price_rating(city)
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -7,6 +7,8 @@ import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import sys
from threading import Lock
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -14,9 +16,10 @@ es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
mutex = Lock()
def find_around_seaweed(lng, lat):
def find_around_seaweed(city_now, lng, lat):
"""
查找周边均价
"""
@@ -27,6 +30,13 @@ def find_around_seaweed(lng, lat):
"query": {
"bool": {
"filter": [
{
"term": {
"city": {
"value": city_now
}
}
},
{
"range": {
"avg_price": {
@@ -54,6 +64,31 @@ def build_doc(item):
return doc_dict
find_count = 0
def find_around_price_item(seaweed):
global find_count
find_count += 1
if find_count % 1000 == 0:
log.info("已查找" + str(find_count))
if 'lng' not in seaweed:
return
lng = commutils.format_lat_lng(seaweed['lng'])
lat = commutils.format_lat_lng(seaweed['lat'])
if pd.notna(lng):
city_now = seaweed['city']
around_seaweed_df = find_around_seaweed(city_now, lng, lat)
mutex.acquire(timeout=60)
if len(around_seaweed_df) > 1:
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
seaweed['relative_price_ranking'] = around_seaweed_df[around_seaweed_df['id'] == seaweed['_id']].iloc[0][
'relative_price_ranking']
elif len(around_seaweed_df) == 1:
seaweed['relative_price_ranking'] = 0.5
mutex.release()
def sync(city=None):
log.info("开始同步相对价格排名数据...")
@@ -62,24 +97,17 @@ def sync(city=None):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
def find_around_price_item(index, item):
if index % 1000 == 0:
log.info("已处理" + str(index))
if 'lng' not in item:
return
lng = commutils.format_lat_lng(item['lng'])
lat = commutils.format_lat_lng(item['lat'])
if pd.notna(lng):
around_seaweed_df = find_around_seaweed(lng, lat)
if len(around_seaweed_df) > 1:
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0]
elif len(around_seaweed_df) == 1:
seaweed_df.loc[index, 'relative_price_ranking'] = 0.5
seaweed_list = list()
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
with ThreadPoolExecutor(max_workers=30) as pool:
for index, item in seaweed_df.iterrows():
pool.submit(find_around_price_item, index, item)
for seaweed in seaweed_list:
pool.submit(find_around_price_item, seaweed)
# for seaweed in seaweed_list:
# find_around_price_item(seaweed)
seaweed_df = pd.DataFrame(seaweed_list)
actions = list()
count = 0
@@ -90,17 +118,15 @@ def sync(city=None):
"_op_type": "update",
"_index": index_name,
"_id": item['_id'],
"doc": doc_info
"doc": doc_info,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
try:
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
log.info("更新:" + str(count))
except Exception as e:
log.error(e)
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
log.info("更新:" + str(count))
actions.clear()
if len(actions) > 0:
try:
@@ -112,4 +138,5 @@ def sync(city=None):
log.info("完成同步相对价格排名数据...")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+12 -5
View File
@@ -6,6 +6,8 @@ import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import sys
from threading import Lock
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -14,6 +16,7 @@ es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
mutex = Lock()
def build_doc(item):
@@ -49,12 +52,15 @@ def sync(city=None):
names_str = "'" + names_str + "'"
# 近三个月挂牌量
# sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=93)
# seaweed_df.loc[index, 'three_months_sell_num'] = sample_num
sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=93)
# 近半年挂牌量
sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=186)
seaweed_df.loc[index, 'half_on_year_sell_num'] = sample_num
sample_half_year_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=186)
mutex.acquire()
seaweed_df.loc[index, 'three_months_sell_num'] = sample_num
seaweed_df.loc[index, 'half_on_year_sell_num'] = sample_half_year_num
mutex.release()
for index, item in seaweed_df.iterrows():
pool.submit(deal, index, item)
@@ -84,4 +90,5 @@ def sync(city=None):
log.info(f"完成同步出售信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+11 -50
View File
@@ -3,10 +3,9 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import re
import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -25,59 +24,20 @@ def build_doc(item):
return doc_dict
# def avg_price_map_deal(x):
# avg_price_map[x['city'] + x['region'] + x['name']] = x['avg_price']
# return x
def deal(seaweed_df, index, item):
if index % 1000 == 0:
log.info("已处理" + str(index))
curr_city = item['city']
curr_region = item['region']
curr_name = item['name']
names = [curr_name]
if pd.notna(item['alias']) and '[]' != item['alias']:
alias_list = re.sub("\[|\]|'", '', item['alias']).split(',')
if len(alias_list) > 0:
for alias_item in alias_list:
names.append(alias_item)
# 户均总价
# if (curr_city + curr_region + curr_name) in avg_price_map:
# avg_price = avg_price_map[curr_city + curr_region + curr_name]
# if pd.notna(avg_price):
# 由于覆盖率较低,暂不过滤
# min_avg_price = 0.8 * avg_price
# max_avg_rice = 1.2 * avg_price
total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366)
seaweed_df.loc[index, 'total_price'] = total_price
def sync(city=None):
log.info(f"开始同步户均总价信息至206 {index_name}")
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
# 加载库中均价数据
# query_sql = "SELECT city, region, name, avg_price FROM community_summary "
# if pd.notna(city):
# query_sql += " WHERE city='" + city + "' "
# community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql)
# community_summary_df.apply(lambda x: avg_price_map_deal(x), axis=1)
with ThreadPoolExecutor(30) as pool:
for index, item in seaweed_df.iterrows():
pool.submit(deal, seaweed_df, index, item)
query_sql = "SELECT city, region, name, avg_price, house_avg_area FROM community_summary WHERE avg_price > 0 and house_avg_area > 0 "
if pd.notna(city):
query_sql += " AND city='" + city + "' "
community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql)
actions = list()
count = 0
for index, item in seaweed_df.iterrows():
community_summary_df['total_price'] = community_summary_df['avg_price'] * community_summary_df['house_avg_area']
community_summary_df['total_price'] = community_summary_df['total_price'].astype(int)
for index, item in community_summary_df.iterrows():
doc_info = build_doc(item)
action = {
@@ -106,4 +66,5 @@ def sync(city=None):
log.info(f"完成同步户均总价信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -5,6 +5,7 @@ import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, dateutils
from elasticsearch import helpers
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -48,19 +49,15 @@ def save(community_factor_df):
actions.clear()
def complete_by_area(community_df):
factor_df = pd.read_csv(path + '/小区户均总价面积因子已去极值.csv', sep='@', encoding='UTF-8')
community_factor_df = pd.merge(community_df, factor_df, how='left', on=['_id', 'city', 'region', 'name'])
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
return community_factor_df
def complete_by_build_year_area(community_factor_df):
build_year_area_df = pd.read_excel(path + '/全国各区各建筑年代户均面积已去极值.xlsx')
build_year_area_dict = dict()
for index, build_year_area in build_year_area_df.iterrows():
build_year_area_dict[
build_year_area['city'] + '|' + build_year_area['region'] + "|" + build_year_area['buildyear']] = build_year_area['factor']
# 修复按城市跑批数据字段可能不存在问题
if 'finish_building_date' not in community_factor_df.columns or 'total_price' not in community_factor_df.columns:
return False
for index, item in community_factor_df[
(community_factor_df['total_price'].isna()) & (community_factor_df['finish_building_date'].notna())].iterrows():
finish_building_date = item['finish_building_date']
@@ -101,18 +98,22 @@ def complete_by_region_area(community_factor_df):
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
def sync():
def sync(city=None):
log.info(f"开始补全户均总价信息至206 {index_name}")
community_df = es206_client.scan_by_sql_to_data_frame("select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null")
sql = "select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null "
if pd.notna(city):
sql += " and city='" + city + "'"
community_df = es206_client.scan_by_sql_to_data_frame(sql)
community_factor_df = complete_by_area(community_df)
complete_by_build_year_area(community_factor_df)
complete_by_region_area(community_factor_df)
save(community_factor_df)
if len(community_df) > 0:
community_df['total_price'] = None
complete_by_build_year_area(community_df)
complete_by_region_area(community_df)
save(community_df)
log.info(f"完成补全户均总价信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+13 -5
View File
@@ -3,6 +3,7 @@
"""
import logging
import pandas as pd
import sys
from goojia_utils import ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
@@ -24,14 +25,20 @@ def build_doc(item):
def sync(city=None):
log.info(f"开始同步成交信息至206 {index_name}")
seaweed_query = {'cat': 'district', 'status': 0}
cities = None
if pd.notna(city):
seaweed_query['city'] = city
cities = [city]
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
if pd.notna(city):
cities = [city]
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
deal_price_df = commutils.get_deal_avg_price(cities, 3)
if len(deal_price_df) <= 0:
log.info(f"无成交,无需同步")
return
if 'match_city' not in deal_price_df.columns or 'match_region' not in deal_price_df.columns or 'match_name' not in deal_price_df.columns:
log.info(f"无成交,无需同步")
return
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name',
'avg_price': 'three_month_trade_avg_price'})
@@ -68,4 +75,5 @@ def sync(city=None):
log.info(f"完成同步成交信息至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+3 -1
View File
@@ -7,6 +7,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
@@ -86,4 +87,5 @@ def sync(city=None):
log.info(f"完成同步交通至206 {index_name}")
sync()
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
@@ -0,0 +1,17 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
"change_new_to_old_seaweed_dag",
# 每月1日1点运行
schedule_interval="0 1 1 * *",
start_date=pendulum.datetime(2023, 2, 10, tz="Asia/Shanghai"),
catchup=True,
description="新房变二手房定时脚本"
) as dag:
t1 = BashOperator(
task_id='change_new_to_old_seaweed_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/change_new_to_old_seaweed.py',
)
t1
+46 -41
View File
@@ -10,84 +10,89 @@ with DAG(
catchup=True,
description="地址输出信息分类数据定时更新"
) as dag:
t1 = BashOperator(
comm = BashOperator(
task_id='comm',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t2 = BashOperator(
price = BashOperator(
task_id='price',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t3 = BashOperator(
sell = BashOperator(
task_id='sell',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t4 = BashOperator(
avg_cost = BashOperator(
task_id='avg_cost',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t5 = BashOperator(
trade = BashOperator(
task_id='trade',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t6 = BashOperator(
total_price = BashOperator(
task_id='total_price',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t6_1 = BashOperator(
total_price_complete = BashOperator(
task_id='total_price_complete',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t7 = BashOperator(
traffic = BashOperator(
task_id='traffic',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t7_1 = BashOperator(
business_district = BashOperator(
task_id='business_district',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t8 = BashOperator(
medical = BashOperator(
task_id='medical',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t9 = BashOperator(
life = BashOperator(
task_id='life',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t10 = BashOperator(
relative_price_ranking = BashOperator(
task_id='relative_price_ranking',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t10_1 = BashOperator(
estate_charge_complete = BashOperator(
task_id='estate_charge_complete',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t10_2 = BashOperator(
estate_charge_ranking = BashOperator(
task_id='estate_charge_ranking',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t11 = BashOperator(
consumption = BashOperator(
task_id='consumption',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t12 = BashOperator(
person_rating = BashOperator(
task_id='person_rating',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/person_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
price_rating = BashOperator(
task_id='price_rating',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t13 = BashOperator(
education = BashOperator(
task_id='education',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t14 = BashOperator(
education2 = BashOperator(
task_id='education2',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t15 = BashOperator(
extra = BashOperator(
task_id='extra',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t16 = BashOperator(
other = BashOperator(
task_id='other',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t10_1 >> t10_2 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16
# 人均消费并行,值为空
comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> [estate_charge_ranking, price_rating, person_rating] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other
+30 -214
View File
@@ -4,8 +4,6 @@
# In[56]:
import os
import tarfile
from goojia_utils import dateutils, ftputils, tarutils
import datetime
import calendar
@@ -13,217 +11,7 @@ import requests
import json
import shutil
# In[9]:
import ftplib
import os
import socket
import stat
from os.path import isdir
from numpy import unicode
import paramiko
def _check_local(local):
if not os.path.exists(local):
try:
os.mkdir(local)
except IOError as err:
print(err)
class SftpClient:
def __init__(self):
self.client = None
self.sftp = None
pass
def sftp_connect(self, username, password, host, port=22):
"""
建立连接,获取sftp句柄
:param username
:param password
:param host
:param port
:return:
"""
try:
self.client = paramiko.Transport((host, port))
except Exception as error:
print(error)
else:
try:
self.client.connect(username=username, password=password)
except Exception as error:
print(error)
else:
self.sftp = paramiko.SFTPClient.from_transport(self.client)
def get(self, remote, local):
"""
从远程机器上下载文件(夹)到本地
:param remote 远程路径
:param local 本地路径
"""
try:
result = self.sftp.stat(remote)
except IOError as err:
error = '[ERROR %s] %s: %s' % (err.errno, os.path.basename(os.path.normpath(remote)), err.strerror)
print(error)
else:
# 判断远程文件是否为目录
if stat.S_ISDIR(result.st_mode):
dirname = os.path.basename(os.path.normpath(remote))
local = os.path.join(local, dirname)
_check_local(local)
for file in self.sftp.listdir(remote):
sub_remote = os.path.join(remote, file)
sub_remote = sub_remote.replace('\\', '/')
self.get(sub_remote, local)
else:
# 拷贝文件
if os.path.isdir(local):
local = os.path.join(local, os.path.basename(remote))
try:
self.sftp.get(remote, local)
except IOError as err:
print(err)
else:
print('[get]', local, '<==', remote)
# 断开连接
def disconnect(self):
try:
self.client.close()
except Exception as error:
print(error)
error
class FtpClient:
def __init__(self):
self.ftp = ftplib.FTP()
self.ftp.set_debuglevel(2)
pass
def connect_ftp(self, host, port, timeout=60, user=None, passwd=None):
"""
连接登录ftp
:param host:
:param port:
:param user:
:param passwd:
:return:
"""
try:
self.ftp.connect(host=host, port=port, timeout=timeout)
print("*******已经成功连接服务器FTP服务!!!")
except(socket.error, socket.gaierror) as e:
print("错误:无法访问服务器FTP服务!!!")
print(e)
exit()
try:
self.ftp.login(user=user, passwd=passwd)
except ftplib.error_perm as e:
print("登录失败,请检查用户名和密码,错误代码")
print(e)
exit()
def cat_dir(self, filepath):
"""
显示ftp指定目录下的文件
:param filepath:
:return:
"""
return self.ftp.dir(filepath)
def make_dir(self, filepath):
"""
新建ftp下的目录,ftp目录下不存在此文件
:param filepath:
:return:
"""
try:
filepath.encode("utf-8")
mkd = self.ftp.mkd(filepath)
return mkd
except ftplib.error_perm as e:
print("请查看目录下是否存在同名文件,异常为")
print(e)
def remove_dir(self, dirname):
"""
删除目录
:param dirname
"""
try:
self.ftp.rmd(dirname)
except ftplib.error_perm as e:
print("异常:!!!目录已经被删除了")
print(e)
def download_file(self, localpath, remotepath, filename):
"""
从FTP指定目录上下载文件,并保存到本地指定路径
:param localpath: linux上路径
:param remotepath: 本地路径
:param filename: 文件名
"""
bufsize = 8192 # 设置缓冲区大小;
ulocalpath = unicode(localpath, "utf-8")
try:
self.ftp.cwd(remotepath)
filepath = ulocalpath + '/'
file_handler = open(filepath + filename, "wb").write # 以写模式打开本地文件,---需下载到本地的路径
# 传一个回调函数给retrbinary() 它在每接收一个二进制数据时都会被调用
self.ftp.retrbinary("RETR " % filename, file_handler, bufsize) # 接收服务器上文件并写入本地文件
except ftplib.error_perm as e:
print("请检查路径是否正确,是否有权限读取,错误代码")
print(e)
def upload(self, localpath):
"""
上传至ftp
:param localpath: linux上路径
"""
bufsize = 8192
ulocalpath = unicode(localpath, "utf-8")
try:
file_handler = open(ulocalpath, "rb")
filename = os.path.split(ulocalpath)[-1]
try:
self.ftp.storbinary("stor " % filename, file_handler, bufsize)
except ftplib.error_perm as e:
print("请检查上传路径是否正确,是否连接FTP,---错误代码")
print(e)
except e:
print("请检查本地路径是否存在····")
def lst_file(self, path):
"""
获取指定目录下文件
:param path
"""
s = []
try:
for name in self.ftp.nlst(path):
name1 = unicode(name, "utf-8")
s.append(name1)
return s
except ftplib.all_errors as e:
print("请检查目录是否正确,错误代码时-----")
print(e)
def close(self):
"""
退出ftp
"""
self.ftp.quit()
result = []
@@ -245,13 +33,18 @@ try:
path = '/home/airflow/data/rh'
if not os.path.exists(path):
os.makedirs(path)
if not os.path.exists(path + "/" + date):
os.makedirs(path + "/" + date)
client = SftpClient()
client = ftputils.SftpClient()
client.sftp_connect(username="ccb", password="ccb!@#", host="192.168.0.45", port=17495)
client.get(remote="/home/ccb_rh/output/" + date, local=path)
client.get(remote="/home/ccb_rh/output/" + date + '/000000_' + date + '_FJW.tar.gz', local=path + '/' + date)
client.get(remote="/home/ccb_rh/output/" + date + '/' + date + '.ok', local=path + '/' + date)
client.disconnect()
# In[80]:
@@ -322,6 +115,29 @@ try:
result.append('FJW_000000_' + cityid + '_BIGDATA_POSITION_FJW_ALL_' + date + '_0001.dat文件不存在')
if 'FJW_000000_' + cityid + '_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_' + cityid + '_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_GUIDE_STATISTICS_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_GUIDE_STATISTICS_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_' + date + '_0001.dat文件不存在')
if len(result) == 0:
result.append('融合检测完毕未发现异常文件')
shutil.rmtree(path + "/" + date)
Binary file not shown.
+63
View File
@@ -0,0 +1,63 @@
import os, sys
import logging
import pandas as pd
import requests
from datetime import datetime
from goojia_utils import dateutils, oracle
from sqlalchemy import create_engine
os.environ["NLS_LANG"] = "American_America.AL32UTF8"
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
log = logging.getLogger(__name__)
path = "/home/airflow/data/jh/price"
if not os.path.exists(path):
os.makedirs(path)
year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_date(datetime.utcnow(), "%Y%m"))
log.info(f"测试月份:{year_month}")
avm_df = pd.read_sql(f"select urbn_id,urbn_nm,gujia_bds_comm_num,gujiabdsvalmnycommnum from t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month} ", con=engine)
comm_df = pd.read_sql(f"select urbn_id,gujia_admndiv_nm,gujia_comm_id,comm_nm from base.t0831_bigdata_community_gj_h where gujia_bds_bds_st=0 ", con=engine)
price_df = pd.read_sql(f"select gujia_bds_id as gujia_comm_id,gujia_comm_basic_prc from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month} ", con=engine)
comm_df = comm_df.merge(avm_df, on="urbn_id", how="left")
comm_df = comm_df.merge(price_df, on="gujia_comm_id", how="left")
no_price_comm_df = comm_df[comm_df.gujia_comm_basic_prc.isna()][['urbn_id','urbn_nm','gujia_admndiv_nm','gujia_comm_id','comm_nm']]
avm_df.loc[:,'无均价数量'] = avm_df['gujia_bds_comm_num'] - avm_df['gujiabdsvalmnycommnum']
avm_df.loc[:,'num'] = 60 - avm_df['gujiabdsvalmnycommnum']
avm_df = avm_df[avm_df['无均价数量'] > 0]
avm_df = avm_df[['urbn_id','urbn_nm','num']]
file_name = f"{year_month}无价格小区统计数据.xlsx"
aim_file_name = path + '/' + file_name
with pd.ExcelWriter(aim_file_name) as writer:
no_price_comm_df.to_excel(writer, sheet_name="无价格小区列表", columns=['urbn_id','urbn_nm','gujia_admndiv_nm','gujia_comm_id','comm_nm'], header=["城市id", "城市名称","区域","小区id","小区名称"], index=False)
avm_df.to_excel(writer, sheet_name="待补充数量", columns=['urbn_id','urbn_nm','num'], header=["城市id","城市名称","待补充数量"],index=False)
log.info("无价格小区统计数据结束")
def up_to_nextcloud():
"""
上传统计信息至nextcloud
"""
from nextcloud import NextCloud
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
NEXTCLOUD_PASSWORD = "goojia123456"
nextcloud_file_name = '共享/建行/' + file_name
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD)
nxc.upload_file(aim_file_name, nextcloud_file_name)
def send_msg():
infos = "【准确度测评】无价格小区统计数据生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情"
data = {
"msgtype": "text",
"text": {
"content": infos
}
}
# 发送消息至群
requests.post(
url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
json=data)
up_to_nextcloud()
send_msg()
+74
View File
@@ -0,0 +1,74 @@
import os, sys
import logging
import pandas as pd
import requests
from datetime import datetime
from goojia_utils import dateutils, oracle
from sqlalchemy import create_engine
os.environ["NLS_LANG"] = "American_America.AL32UTF8"
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
log = logging.getLogger(__name__)
path = "/home/airflow/data/jh/price"
if not os.path.exists(path):
os.makedirs(path)
month_now = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now()), '%Y%m')
month_last = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-1), '%Y%m')
month_before = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-2), '%Y%m')
type = sys.argv[1] if len(sys.argv) > 1 else ''
log.info("输出类型" + type)
# 新增支持上一月与前月对比
if type == month_last:
month_now = month_last
month_last = month_before
log.info(f"测试月份:{month_now}")
price_df = pd.read_sql(f"select urbn_id,urbn_nm,gujia_admndiv_nm,gujia_bds_id as gujia_comm_id,comm_nm,gujia_bds_clmtd,gujia_comm_basic_prc from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={month_now} ", con=engine)
deal_df = pd.read_sql(f"select gujia_bds_mdl_id,gujia_comm_id,gujia_bds_trdtckt_prc from base.t0831_bigdata_deal_gj_h where to_char(txn_dt,'yyyyMM')='{month_last}' ", con=engine)
deal_price_df = deal_df.merge(price_df, on="gujia_comm_id", how="left")
deal_price_df["ppe"] = round((deal_price_df["gujia_comm_basic_prc"] - deal_price_df['gujia_bds_trdtckt_prc']) / deal_price_df['gujia_bds_trdtckt_prc'], 4)
deal_price_df = deal_price_df[abs(deal_price_df['ppe']) > 0.15]
# third_df = pd.read_sql(f"select gujia_trdpt_avprc_id,gujia_bds_id as gujia_comm_id,gujia_dtsr_dsc,gujia_bds_comm_avprc from base.t0831_bigdata_third_party_price_gj_h where gujiabds_avprc_dt_val={month_last} ", con=engine)
# third_price_df = third_df.merge(price_df, on="gujia_comm_id", how="left")
# third_price_df["ppe"] = round((third_price_df["gujia_comm_basic_prc"] - third_price_df['gujia_bds_comm_avprc']) / third_price_df['gujia_bds_comm_avprc'], 4)
# third_price_df = third_price_df[abs(third_price_df['ppe']) > 0.15]
file_name = f"{month_now}测试偏差较大案例统计数据.xlsx"
aim_file_name = path + '/' + file_name
with pd.ExcelWriter(aim_file_name) as writer:
deal_price_df.to_excel(writer, sheet_name="成交", columns=['urbn_id','urbn_nm','gujia_bds_mdl_id','gujia_admndiv_nm','gujia_comm_id','comm_nm','gujia_bds_clmtd','gujia_comm_basic_prc','gujia_bds_trdtckt_prc','ppe'], header=["城市id", "城市名称","成交id","区域","小区id","小区名称",'计算类型','计算价格','成交价格','PPE'], index=False)
# third_price_df.to_excel(writer, sheet_name="第三方", columns=['urbn_id','urbn_nm','gujia_trdpt_avprc_id','gujia_admndiv_nm','gujia_comm_id','comm_nm','gujia_bds_clmtd','gujia_comm_basic_prc','gujia_bds_comm_avprc','PPE','gujia_dtsr_dsc'], header=["城市id", "城市名称","第三方id","区域","小区id","小区名称",'计算类型','计算价格','第三方价格','PPE','来源'],index=False)
log.info("偏差较大案例统计数据结束")
def up_to_nextcloud():
"""
上传统计信息至nextcloud
"""
from nextcloud import NextCloud
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
NEXTCLOUD_PASSWORD = "goojia123456"
nextcloud_file_name = '共享/建行/' + file_name
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD)
nxc.upload_file(aim_file_name, nextcloud_file_name)
def send_msg():
infos = "【准确度测评】测试偏差较大案例统计数据生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情"
data = {
"msgtype": "text",
"text": {
"content": infos
}
}
# 发送消息至群
requests.post(
url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
json=data)
up_to_nextcloud()
send_msg()
+129
View File
@@ -0,0 +1,129 @@
import os, sys
import logging
import pandas as pd
import requests
from datetime import datetime
from goojia_utils import dateutils, oracle
from sqlalchemy import create_engine
os.environ["NLS_LANG"] = "American_America.AL32UTF8"
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
important_cities =["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"]
city_order = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx")
def convert_calc_type(cale_type):
if cale_type == 1:
return "指导价"
elif cale_type == 2:
return "样本"
elif cale_type == 3:
return "神经网络"
fitment_code_map = {"0601": "毛坯", "0602": "简装", "0603": "精装"}
def convert_fitment_code(fitment_code):
if pd.isna(fitment_code):
return None
fitment_code = str(fitment_code)
if fitment_code in fitment_code_map:
return fitment_code_map[fitment_code]
return None
house_aspect_code_map = {"0301": "南北", "0302": "东南", "0303": "南", "0304": "东西", "0305": "东", "0306": "西南", "0307": "西",
"0308": "北", "0309": "东北",
"0310": "西北"}
def convert_house_aspect_code(house_aspect_code):
if pd.isna(house_aspect_code):
return None
house_aspect_code = str(house_aspect_code)
if house_aspect_code in house_aspect_code_map:
return house_aspect_code_map[house_aspect_code]
return None
log = logging.getLogger(__name__)
path = "/home/airflow/data/jh/price"
if not os.path.exists(path):
os.makedirs(path)
year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_date(datetime.utcnow(), "%Y%m"))
log.info(f"测试月份:{year_month}")
avm_evaluation_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_bds_yrmo': 'year_month', 'gujia_bds_comm_num': 'community_num', 'gujiabdsvalmnycommnum': 'has_price_community_num', 'gujia_prc_cvr_rate': 'price_coverage', 'gujiabdsmnldpccommnum': 'artificial_calc_num', 'gujia_mnl_drc_prc_pct': 'artificial_calc_coverage', 'gujiabdssamp_comm_num': 'sample_calc_num', 'gujiaext_samp_clc_pct': 'sample_calc_coverage', 'gujiabdsnervetncomnum': 'ann_calc_num', 'gujianervethrunclcpct': 'ann_calc_coverage', 'gujiabds_tst_case_num': 'test_num', 'gujiabdsmdltstcommnum': 'deal_test_comm_num', 'gujiatst_comm_num_pct': 'test_comm_coverage', 'gujia_ppe15_cvr_rate': 'ppe15_coverage', 'gujia_rto_mdn': 'rate_median', 'gujia_rto_lv_san_ceft': 'coefficient', 'gujiabdsmdltstcasenum': 'deal_test_case_num', 'gujiamdltstcommnumpct': 'deal_test_comm_coverage', 'gujiamdltst_ppe15_pct': 'deal_ppe15_coverage', 'gujiamdlrtolvsan_ceft': 'deal_coefficient', 'gujia_mdl_md_bit_rto': 'deal_rate_median', 'gujiabdstrdpttcasenum': 'third_test_case_num', 'gujiabdstrdpttcommnum': 'third_test_comm_num', 'gujiatrdpttcommnumpct': 'third_test_comm_coverage', 'gujiatrdpttstppe15pct': 'third_ppe15_coverage', 'gujiatrdptrolvsanceft': 'third_coefficient', 'gujiatrdpt_md_bit_rto': 'third_rate_median', 'data_crt_tm': 'createtime', 'data_udt_tm': 'updatetime', 'gujia_src_co': 'sourcecode'}
avm_evaluation_item_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_rgon_id': 'distid', 'gujia_rgon_nm': 'distname', 'gujia_comm_id': 'commid', 'gujia_comm_nm': 'commname', 'gujiabds_clc_yrmo_val': 'year_month', 'gujia_bds_clc_avprc': 'calc_avg_price', 'gujia_bds_avprc': 'avg_price', 'gujia_bds_clc_tp': 'calc_type', 'gujia_bds_rlb': 'confidence', 'gujiabds_to_mkprc_mny': 'target_avg_price', 'mdl_dt': 'dealdate', 'gujiabds_valt_unitprc': 'unit_price', 'gujia_area': 'buildarea', 'gujia_dcrt_val': 'fitment', 'gujia_twrd_val': 'houseaspect', 'gujia_bds_floor_num': 'floornum', 'gujiabdstot_floor_num': 'floorcnt', 'gujia_data_tp': 'data_type', 'gujia_rto': 'rate', 'gujia_mns_rto': 'pe', 'data_crt_tm': 'createtime'}
avm_evaluation = pd.read_sql(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month}", con=engine)
avm_evaluation_item = pd.read_sql(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val={year_month}", con=engine)
avm_evaluation.rename(columns=avm_evaluation_column_map, inplace=True)
avm_evaluation_item.rename(columns=avm_evaluation_item_column_map, inplace=True)
community_base_price = pd.read_sql(f'''SELECT gujia_bds_id as commid,urbn_nm as cityname,gujia_admndiv_nm as distname,comm_nm as commname,
gujia_comm_basic_prc as comm_avg_price,gujia_bds_cfdc_dgr as confidence,gujia_bds_clmtd as calc_type from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month}''',
con=engine)
communities = pd.read_sql(f"select gujia_comm_id as COMMID from base.t0831_bigdata_community_gj_h", con=engine)
community_base_price = community_base_price[community_base_price.commid.isin(communities["commid"])]
avm_evaluation_item["fitment"] = avm_evaluation_item["fitment"].map(convert_fitment_code)
avm_evaluation_item["houseaspect"] = avm_evaluation_item["houseaspect"].map(convert_house_aspect_code)
avm_evaluation_item["calc_type"] = avm_evaluation_item["calc_type"].map(convert_calc_type)
community_base_price["calc_type"] = community_base_price["calc_type"].map(convert_calc_type)
avm_evaluation_item["dealdate"] = avm_evaluation_item["dealdate"].apply(lambda x : x.strftime("%Y-%m-%d") if pd.notna(x) else None)
avm_evaluation_item_third = avm_evaluation_item[avm_evaluation_item.data_type=='第三方']
avm_evaluation_item_deal = avm_evaluation_item[avm_evaluation_item.data_type=='成交']
import_avm_evaluation_item_third = avm_evaluation_item_third[avm_evaluation_item_third.cityname.isin(important_cities)]
import_avm_evaluation_item_deal = avm_evaluation_item_deal[avm_evaluation_item_deal.cityname.isin(important_cities)]
import_avm_evaluation_item_deal.loc[:, "deal_total_price"] = import_avm_evaluation_item_deal["buildarea"] * import_avm_evaluation_item_deal["target_avg_price"]
import_avm_evaluation_item_deal.loc[:, "deal_total_price"] = import_avm_evaluation_item_deal["deal_total_price"].apply(lambda x : int(x))
import_avm_evaluation_item_deal.loc[:, "total_price"] = import_avm_evaluation_item_deal["buildarea"] * import_avm_evaluation_item_deal["calc_avg_price"]
import_avm_evaluation_item_deal.loc[:, "total_price"] = import_avm_evaluation_item_deal["total_price"].apply(lambda x : int(x))
import_community_base_price = community_base_price[community_base_price.cityname.isin(important_cities)]
normal_community_base_price = community_base_price[~(community_base_price.cityname.isin(important_cities))]
import_avm_evaluation = avm_evaluation[avm_evaluation.cityname.isin(important_cities)]
avm_evaluation = avm_evaluation.merge(city_order, on="cityname", how="left")
import_community_base_price = import_community_base_price.merge(city_order, on="cityname", how="left")
normal_community_base_price = normal_community_base_price.merge(city_order, on="cityname", how="left")
import_avm_evaluation_item_deal = import_avm_evaluation_item_deal.merge(city_order, on="cityname", how="left")
import_avm_evaluation_item_third = import_avm_evaluation_item_third.merge(city_order, on="cityname", how="left")
avm_evaluation.sort_values(by="cityrank", inplace=True)
import_community_base_price.sort_values(by="cityrank", inplace=True)
normal_community_base_price.sort_values(by="cityrank", inplace=True)
import_avm_evaluation_item_deal.sort_values(by="cityrank", inplace=True)
import_avm_evaluation_item_third.sort_values(by="cityrank", inplace=True)
file_name = f"{year_month}估值模型测试.xlsx"
aim_file_name = path + '/' + file_name
with pd.ExcelWriter(aim_file_name) as writer:
avm_evaluation.to_excel(writer, sheet_name="92个城市估值概况", columns=["cityname","community_num","price_coverage","sample_calc_coverage","ann_calc_coverage","artificial_calc_coverage","deal_test_case_num","deal_ppe15_coverage","deal_test_comm_num","deal_test_comm_coverage","deal_rate_median","deal_coefficient","third_test_comm_num","third_test_comm_coverage","third_ppe15_coverage","third_rate_median","third_coefficient"], header=["城市名称","小区量","可输出均价小区占比","样本计算占比","神经网络计算占比","人工指导价占比","成交测试案例总量","成交测试PPE15占比","成交测试小区量","成交测试小区量占比","成交比率中位数","成交比率离散系数","第三方均价测试小区量","第三方均价测试小区量占比","第三方均价测试PPE15占比","第三方比率中位数","第三方比率离散系数"], index=False)
import_community_base_price.to_excel(writer, sheet_name="测试10个城市估值数据", columns=["cityname","distname","commname","comm_avg_price","confidence","calc_type"], header=["城市","区域","小区名","估值小区均价","置信度","估值方式"],index=False)
import_avm_evaluation_item_deal.to_excel(writer, sheet_name="10个城市成交测试案例", columns=['cityname','distname', 'commname', 'unit_price', 'confidence', 'calc_type','total_price', 'deal_total_price', 'target_avg_price','pe',"dealdate","fitment","houseaspect","floornum","floorcnt","buildarea"], header=["城市","区域","小区名","修正均价","置信度","估值方式","小区估价","对标成交价","对标均价","PPE","dealdate","fitment","houseaspect","floornum","floorcnt","buildarea"], index=False)
import_avm_evaluation_item_third.to_excel(writer, sheet_name="10个城市第三方均价测试", columns=["cityname","distname","commname","avg_price","confidence","calc_type","target_avg_price","pe"], header=["城市","区域","小区名","小区均价","置信度","估值方式","对标第三方均价","PPE"],index=False)
log.info("估值结果测试结束")
def up_to_nextcloud():
"""
上传统计信息至nextcloud
"""
from nextcloud import NextCloud
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
NEXTCLOUD_PASSWORD = "goojia123456"
nextcloud_file_name = '共享/建行/' + file_name
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD)
nxc.upload_file(aim_file_name, nextcloud_file_name)
def send_msg():
infos = "【准确度测评】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情"
data = {
"msgtype": "text",
"text": {
"content": infos
}
}
# 发送消息至群
requests.post(
url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
json=data)
up_to_nextcloud()
send_msg()
+11 -8
View File
@@ -7,9 +7,10 @@ import logging
import os
import sys
import requests
from sqlalchemy import create_engine
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
def simple_calc(result_df):
"""
新增是否有样本列
@@ -25,8 +26,9 @@ def simple_calc(result_df):
if len(actions) == 800:
ins = "','".join(actions)
ins = "'" + ins + "'"
simple_temp_df = oracle.scan_by_sql_to_data_frame("select distinct commid from bigdata_forsale_gj where " +
" publishtime> TO_DATE('" + month_before + "','YYYY-MM-DD') and commid in (" + ins + ")")
simple_temp_df = pd.read_sql(
"select distinct gujia_comm_id as commid from base.t0831_bigdata_forsale_gj_h where " +
" anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")", con=engine)
simple_df = pd.concat([simple_df, simple_temp_df])
actions.clear()
@@ -34,8 +36,9 @@ def simple_calc(result_df):
if len(actions) > 0:
ins = "','".join(actions)
ins = "'" + ins + "'"
simple_temp_df = oracle.scan_by_sql_to_data_frame("select distinct commid from bigdata_forsale_gj where " +
" publishtime> TO_DATE('" + month_before + "','YYYY-MM-DD') and commid in (" + ins + ")")
simple_temp_df = pd.read_sql(
"select distinct gujia_comm_id as commid from base.t0831_bigdata_forsale_gj_h where " +
" anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")", con=engine)
simple_df = pd.concat([simple_df, simple_temp_df])
actions.clear()
simple_df['has_sample'] = 1
@@ -85,8 +88,8 @@ if type == month_last:
month_now = month_last
month_last = month_before
df = oracle.scan_by_sql_to_data_frame("SELECT id, cityname, distname, commid, commname, calc_type, confidence, avg_price, year_month FROM community_base_price_gj WHERE year_month IN('"+month_now+"', '"+month_last+"') " +
" ORDER BY cityname, distname, commid, commname, year_month")
df = pd.read_sql("SELECT gujia_bss_prc_id as id, urbn_nm as cityname, gujia_admndiv_nm as distname, gujia_bds_id as commid, comm_nm as commname, gujia_bds_clmtd as calc_type, gujia_bds_cfdc_dgr as confidence, gujia_comm_basic_prc as avg_price, gujiabds_clc_yrmo_val as year_month FROM t_app_p9pha_plda.avm_community_base_price_gj_h WHERE gujiabds_clc_yrmo_val IN('"+month_now+"', '"+month_last+"') " +
" ORDER BY urbn_nm, gujia_admndiv_nm, gujia_bds_id, comm_nm, gujia_bds_clmtd", con=engine)
six_map = dict()
five_map = dict()
@@ -143,7 +146,7 @@ for index, item in city_count_df.iterrows():
city_count_df = city_count_df.rename(columns={'小区ID': '小区数量'})
city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度')
excel_writer.save()
#excel_writer.save()
excel_writer.close()
up_to_nextcloud()
+17 -4
View File
@@ -8,10 +8,23 @@ with DAG(
schedule_interval="0 10 23 * *",
start_date=pendulum.datetime(2022, 8, 23, tz="Asia/Shanghai"),
catchup=True,
description="建行每月价格异常检测脚本,默认检测本月与上月差异, 若对比上月与前月价格,设置type=\"202210\"(前月日期即可) 若输出样本存在与否,设置type=sample"
description="建行每月价格异常检测脚本,默认检测本月与上月差异, 若对比上月与前月价格,设置year_month=\"202210\"(前月日期即可) 若输出样本存在与否,设置year_month=sample"
) as dag:
t1 = BashOperator(
task_id='price_exception',
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
task_id='price_evaluation',
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_evaluation.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}',
)
t1
t2 = BashOperator(
task_id='price_exception',
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}',
)
t3 = BashOperator(
task_id='no_price_community',
bash_command='python /opt/airflow/dags/repo/dags/jh/price/no_price_community.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}',
)
t4 = BashOperator(
task_id='price_deviation',
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_deviation.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}',
)
t1 >> t2 >> t3 >> t4
+20 -34
View File
@@ -2,7 +2,7 @@
统计近三个月、近一年样本量及小区属性
"""
import logging
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils, commutils
import pandas as pd
import datetime
import sys
@@ -24,37 +24,8 @@ seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0
.batch_size(10000)))
seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'})
def sell_stat():
"""
最近一年有样本小区占比 最近3月有样本小区占比
"""
df = pd.read_excel(aim_file_name)
sell_seaweed_df = es206_client.scan_by_sql_to_data_frame(
"select city, region, district_name, p_date from latest_seawater where p_date > " + start_time + " and cat='sell' ")
# 小区别名升级为单独df
actions = list()
for index, item in seaweed_df.iterrows():
if type(item['alias']) == list:
for name in item['alias']:
actions.append({'_id': item['_id'], 'city': item['city'], 'name': name, 'region': item['region']})
alias_df = pd.DataFrame(actions)
sell_seaweed_rename_df = sell_seaweed_df[['city', 'region', 'district_name']]
sell_seaweed_rename_df = sell_seaweed_rename_df.rename(columns={'district_name': 'name'})
sell_seaweed_merge_df = pd.merge(sell_seaweed_rename_df, alias_df, how='left', on=['city', 'region', 'name'])
sell_seaweed_merge_df['count'] = sell_seaweed_merge_df[sell_seaweed_merge_df['_id'].notna()].groupby('_id')[
'_id'].transform('size')
sell_seaweed_merge_df = sell_seaweed_merge_df[['city', 'region', '_id', 'count']]
sell_seaweed_merge_df = sell_seaweed_merge_df.drop_duplicates(subset=['city', 'region', '_id'])
sell_seaweed_ok_df = sell_seaweed_merge_df[sell_seaweed_merge_df['count'] > sell_num]
sell_seaweed_ok_df['city_seaweed_count'] = sell_seaweed_ok_df.groupby('city')['city'].transform('size')
city_seaweed_count_df = sell_seaweed_ok_df[['city', 'city_seaweed_count']].drop_duplicates(
subset=['city', 'city_seaweed_count'])
city_seaweed_count_df = city_seaweed_count_df.rename(columns={'city': '城市', 'city_seaweed_count': '最近3月有样本小区占比'})
df = pd.merge(df, city_seaweed_count_df, how='left', on=['城市'])
df['最近3月有样本小区占比'] = round(df['最近3月有样本小区占比'] * 100 / df['小区量'], 2)
df.to_excel(aim_file_name, index=False)
def get_sell(df, daysBefore=365):
cities = df[['城市']].drop_duplicates(keep='first')['城市'].values.tolist()
city_one_year_count = list()
for city in cities:
@@ -79,8 +50,7 @@ def sell_stat():
names_str = "','".join(names)
names_str = "'" + names_str + "'"
try:
sql = "select count(*) from latest_seawater where city='" + city + "' and region='" + region + "' and cat='sell' and district_name in(" + names_str + ") and p_date > " + one_year_time
num = es206_client.search_by_sql(sql)['hits']['total']['value']
num = commutils.get_sample(city, region, names_str, daysBefore=daysBefore)
if num > sell_num:
count += 1
total_num += num
@@ -92,9 +62,25 @@ def sell_stat():
print(city + "已处理完, 近一年有样本小区数量" + str(count) + " 近一年的样本总量" + str(total_num))
result_df = pd.DataFrame(city_one_year_count)
return result_df
def sell_stat():
"""
最近一年有样本小区占比 最近3月有样本小区占比
"""
df = pd.read_excel(aim_file_name)
result_df = get_sell(df, 365)
result_df = result_df[['城市', '近一年有样本小区数量', '近一年的样本总量']]
df = pd.merge(df, result_df, how='left', on=['城市'])
df['最近一年有样本小区占比'] = round(df['近一年有样本小区数量'] * 100 / df['小区量'], 2)
df['近一年有样本小区占比'] = round(df['近一年有样本小区数量'] * 100 / df['小区量'], 2)
result_df = get_sell(df, 93)
result_df = result_df.rename(columns={'近一年有样本小区数量': '近三月有样本小区数量'})
result_df = result_df[['城市', '近三月有样本小区数量']]
df = pd.merge(df, result_df, how='left', on=['城市'])
df['近三月有样本小区占比'] = round(df['近三月有样本小区数量'] * 100 / df['小区量'], 2)
df.to_excel(aim_file_name, index=False)