diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index 6387264..a008cf5 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -47,11 +47,13 @@ def build_doc(item): estate_charge = commutils.format_property_fee(item['estate_charge']) if pd.notna(estate_charge): - doc_dict['estate_charge'] = float(estate_charge) - # 表示该物业费已经从库中补充 - doc_dict['estate_charge_flag'] = 0 - # 先打10分,对于其他的流程在修改分数 - doc_dict['credit_detail'] = {'estate_charge_score': 10} + # 剔除过大物业费导致ES报错 + if float(estate_charge) <= 1000: + doc_dict['estate_charge'] = float(estate_charge) + # 表示该物业费已经从库中补充 + doc_dict['estate_charge_flag'] = 0 + # 先打10分,对于其他的流程在修改分数 + doc_dict['credit_detail'] = {'estate_charge_score': 10} finish_building_date = commutils.format_finish_building_date(item['finish_building_date']) if pd.notna(finish_building_date): diff --git a/dags/business/business/community/consumption.py b/dags/business/business/community/consumption.py index cef8772..9a6d5c8 100644 --- a/dags/business/business/community/consumption.py +++ b/dags/business/business/community/consumption.py @@ -124,13 +124,15 @@ def calc_consumption_rating(community_summary_df): actions = list() count = 0 - if len(df) < 5: + # 总评级 + total_rate = 6 + if len(df) < total_rate: # 对于样本低于5的直接排名 df[target] = df['consumption_potential'].rank(method='dense', ascending=False).astype(int) else: # 极值设置 df = filter_min_max(df) - df = gmm_train(5, df, source, target) + df = gmm_train(total_rate, df, source, target) # 批量更新 for index, item in df.iterrows(): row = dict() diff --git a/dags/business/business/community/full_name.py b/dags/business/business/community/full_name.py new file mode 100644 index 0000000..867262d --- /dev/null +++ b/dags/business/business/community/full_name.py @@ -0,0 +1,69 @@ +""" +省市区简拼转全拼 +""" +from goojia_utils import ElasticSearchClient, dateutils +import logging +import pandas as pd +from elasticsearch import helpers +import sys +log = logging.getLogger(__name__) +es206_client = ElasticSearchClient(host='192.168.0.206') +es89_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') + +index_name = "community_summary" +path = "/home/airflow/data/community" +m_date = dateutils.iso_format() + +count = 0 + +short_df = es89_client.scan_by_sql_to_data_frame("select province, city, city_short, district, district_short from administrative_mapping where type='区域' ") +short_map = {} +def deal_short_map(x): + short_map[x['city_short'] + x['district_short']] = x['province'] + '|' + x['city'] + '|' + x['district'] +short_df.apply(lambda x: deal_short_map(x), axis=1) + +def deal(actions, x): + global count + doc_info = dict() + if pd.notna(x['city']) and pd.notna(x['region']): + if x['city'] + x['region'] in short_map: + short_info = short_map[x['city'] + x['region']] + doc_info['province_full'] = short_info.split('|')[0] + doc_info['city_full'] = short_info.split('|')[1] + doc_info['region_full'] = short_info.split('|')[2] + + action_doc = { + "_op_type": "update", + "_index": index_name, + "_id": x['_id'], + "doc": doc_info + } + if len(doc_info) > 0: + actions.append(action_doc) + if len(actions) == 5000: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info(f"已更新{count}") + actions.clear() + + +def sync(city=None): + log.info(f"开始转换省市区全拼至206 {index_name}") + + sql = "select city, region from community_summary" + if pd.notna(city): + sql += " where city='" + city + "'" + community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + + actions = list() + community_summary_df.apply(lambda x: deal(actions, x), axis=1) + + if len(actions) > 0: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() + + log.info(f"完成转换省市区全拼至206 {index_name}") + + +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) \ No newline at end of file diff --git a/dags/business/business/didata_sync.py b/dags/business/business/didata_sync.py index d8b8b14..b00a59d 100644 --- a/dags/business/business/didata_sync.py +++ b/dags/business/business/didata_sync.py @@ -23,10 +23,13 @@ def sync_district_office(type): for city in cities: actions = list() id_list = list() - for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0}, {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1, - 'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1, 'status': 1})\ + for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0}, + {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1, + 'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1, + 'status': 1}) \ .batch_size(10000): - updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'], 'status': doc['status']} + updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'], + 'status': doc['status']} if 'estate_type2' in doc: updateDict['estate_type'] = doc['estate_type2'] @@ -42,7 +45,7 @@ def sync_district_office(type): if 'addr_alias' in doc: updateDict['addr_alias'] = doc['addr_alias'] if 'lng' in doc and doc['lng'] is not None and str(doc['lng']) != 'nan': - updateDict['location'] = {'lon': doc['lng'] / 10**10, 'lat': doc['lat'] / 10**10} + updateDict['location'] = {'lon': doc['lng'] / 10 ** 10, 'lat': doc['lat'] / 10 ** 10} updateDict['update_date'] = update_date action = UpdateOne({'_id': doc['_id']}, {'$set': updateDict}, upsert=True) @@ -51,14 +54,16 @@ def sync_district_office(type): seaweed_didata_conn.bulk_write(actions) actions.clear() - didata_id_df = pd.DataFrame(list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000))) + didata_id_df = pd.DataFrame( + list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000))) if len(didata_id_df) > 0: didata_id_df = didata_id_df[['_id']] id_df = pd.DataFrame(id_list) deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False) for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows(): - action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, {'$set': {'status': 1, 'update_date': update_date}}, upsert=False) + action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, + {'$set': {'status': 1, 'update_date': update_date}}, upsert=False) actions.append(action) if len(actions) == 10000: seaweed_didata_conn.bulk_write(actions) @@ -69,8 +74,9 @@ def sync_district_office(type): actions.clear() log.info(city + "-" + type + "-同步完成") + if type == 'all': sync_district_office("district") sync_district_office("office") else: - sync_district_office(type) \ No newline at end of file + sync_district_office(type) diff --git a/dags/business/dag_community_summary.py b/dags/business/dag_community_summary.py index c2df35b..97500bb 100644 --- a/dags/business/dag_community_summary.py +++ b/dags/business/dag_community_summary.py @@ -110,5 +110,11 @@ with DAG( task_id='other', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) + # 统一输出为省市区全拼 + full_name = BashOperator( + task_id='full_name', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/full_name.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', + ) # 人均消费并行,值为空 - comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> city_level_and_price_rating >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other + comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> city_level_and_price_rating >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other \ + >> full_name diff --git a/dags/install.bat b/dags/install.bat index 156cf1b..b3137e8 100644 --- a/dags/install.bat +++ b/dags/install.bat @@ -1,2 +1,2 @@ -pip install -r /opt/airflow/dags/repo/dags/requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/ -pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/ \ No newline at end of file +pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/ +pip install -r /opt/airflow/dags/repo/dags/requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/ \ No newline at end of file diff --git a/dags/jh/merge/merge.py b/dags/jh/merge/merge.py index 218117d..26ab741 100644 --- a/dags/jh/merge/merge.py +++ b/dags/jh/merge/merge.py @@ -114,10 +114,10 @@ try: result.append('FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat文件不存在') if 'FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat文件不存在') - if 'FJW_000000_000000_EVALUATE_RATE_CITY_FJW_ADD_' + date + '_0001.dat' not in sub_dirs: - result.append('FJW_000000_000000_EVALUATE_RATE_CITY_FJW_ADD_' + date + '_0001.dat文件不存在') - if 'FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_' + date + '_0001.dat' not in sub_dirs: - result.append('FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_' + date + '_0001.dat文件不存在') + if 'FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_' + date + '_0001.dat文件不存在') + if 'FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_' + date + '_0001.dat文件不存在') # if 'FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: # result.append('FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在') diff --git a/dags/jh/price/price_exception.py b/dags/jh/price/price_exception.py index cfe0817..191e56b 100644 --- a/dags/jh/price/price_exception.py +++ b/dags/jh/price/price_exception.py @@ -1,13 +1,18 @@ """ 价格异常检测脚本 """ -from goojia_utils import dateutils, pgsql +from goojia_utils import dateutils, pgsql, MAIN_MONGO_CLIENT, data_utils import pandas as pd import logging import os import sys import requests +seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +# 获取独家小区 +exclusive_seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0, 'mark_map.exclusive': '是'}, {'city': 1, 'region': 1, 'name': 1, 'mark_map.exclusive': 1}).batch_size(100000))) +exclusive_seaweed_df = data_utils.flatten_json(exclusive_seaweed_df, 'mark_map') +exclusive_seaweed_df = exclusive_seaweed_df.rename(columns={'name': 'commname', 'exclusive': '独家小区'}) def simple_calc(result_df): """ @@ -111,7 +116,7 @@ for key in six_map: if abs(six_price - five_price) * 100 / five_price > 10: actions.append({'city': info[0], 'region': info[1], 'commid': info[2], 'commname': info[3], 'five_price': five_price,'five_calc_type': five_map[key].split("|")[1], - 'six_price': six_price,'six_calc_type': six_map[key].split("|")[1], + 'six_price': six_price,'six_calc_type': six_map[key].split("|")[1], '差异值': abs(six_price - five_price) * 100 / five_price, '涨跌幅': (six_price - five_price) * 100 / five_price, 'confidence': six_map[key].split("|")[2]}) if info[0] not in city_seaweed_dict: city_seaweed_dict[info[0]] = 1 @@ -132,6 +137,7 @@ excel_writer = pd.ExcelWriter(aim_file_name) if type == 'sample': result_df = simple_calc(result_df) +result_df = pd.merge(result_df, exclusive_seaweed_df, on=['city', 'region', 'commname'], how='left') result_df = result_df.drop_duplicates(subset=['commid'], keep='first') result_df = result_df.rename(columns={'city': '城市', 'region': '区域', 'commid': '小区ID', 'commname': '小区名', 'five_price': '上一月价格', 'five_calc_type': '上一月计算方式', 'six_price': '当月价格', 'six_calc_type': '当月计算方式', 'confidence': '置信度'}) diff --git a/dags/jh/sample/samples_notice.py b/dags/jh/sample/samples_notice.py index da7d9f1..d8080d2 100644 --- a/dags/jh/sample/samples_notice.py +++ b/dags/jh/sample/samples_notice.py @@ -1,4 +1,4 @@ -from goojia_utils import ElasticSearchClient,dingutils,dateutils +from goojia_utils import ElasticSearchClient, dingutils, dateutils import pandas as pd import datetime import logging @@ -7,22 +7,23 @@ log = logging.getLogger(__name__) # 获取当前日期和2个月前的日期,统计2个月样本数据 now = datetime.datetime.today() -first_day = datetime.datetime(now.year,now.month, 1) +first_day = datetime.datetime(now.year, now.month, 1) now_day = first_day.strftime('%Y%m%d') -last = dateutils.add_months(first_day,0,-2,0,0) +last = dateutils.add_months(first_day, 0, -2, 0, 0) last_day = last.strftime('%Y%m%d') log.info(f"系统当前时间 : {now}") # 区域维度,统计2个月内样本数据量 es206_client = ElasticSearchClient(host='192.168.0.206') -sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+str(last_day)+' and p_date<'+str(now_day) +sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + str( + last_day) + ' and p_date<' + str(now_day) log.info(f"城市区域维度 sql : {sql}") df = es206_client.scan_by_sql_to_data_frame(sql) -df_region = pd.DataFrame(df.groupby(by=['city','region'])['_id'].count()).reset_index() -df_region = df_region.rename(columns={'_id':'count'}) -df_region_res = df_region[df_region['count']<20] +df_region = pd.DataFrame(df.groupby(by=['city', 'region'])['_id'].count()).reset_index() +df_region = df_region.rename(columns={'_id': 'count'}) +df_region_res = df_region[df_region['count'] < 20] if df_region_res.empty == False: text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' @@ -31,7 +32,7 @@ if df_region_res.empty == False: # 当前时间 now = datetime.datetime.today() # 获取当前月份的第一天 -first_day_of_month = datetime.datetime(now.year,now.month, 1) +first_day_of_month = datetime.datetime(now.year, now.month, 1) # 获取1个月前的第一天 last_month = first_day_of_month - datetime.timedelta(days=1) last_day_of_month = last_month.replace(day=1) @@ -46,29 +47,29 @@ strat_time2 = last2_day_of_month.strftime('%Y%m%d') end_time2 = last_day_of_month.strftime('%Y%m%d') # 房源网站维度,按当月数量,和当月数量相比上月减少百分比 -sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+strat_time+' and p_date<='+end_time -sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+strat_time2+' and p_date<='+end_time2 +sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time + ' and p_date<=' + end_time +sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time2 + ' and p_date<=' + end_time2 log.info(f"房源网站维度-百分比 当月sql : {sql1}") log.info(f"房源网站维度-百分比 上月sql : {sql2}") df_root1 = es206_client.scan_by_sql_to_data_frame(sql1) df_root2 = es206_client.scan_by_sql_to_data_frame(sql2) - df_root_group1 = pd.DataFrame(df_root1.groupby(by=['root'])['_id'].count()).reset_index() df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).reset_index() -df_root_group1 = df_root_group1.rename(columns={'_id':'count'}) -df_root_group2 = df_root_group2.rename(columns={'_id':'count'}) +df_root_group1 = df_root_group1.rename(columns={'_id': 'count'}) +df_root_group2 = df_root_group2.rename(columns={'_id': 'count'}) # 房源网站,百分比维度,减少30% -df_root_merge = pd.merge(df_root_group1,df_root_group2,how='left',on=['root']) +df_root_merge = pd.merge(df_root_group1, df_root_group2, how='left', on=['root']) -count_new = 'count_'+strat_time[0:6] -count_old = 'count_'+strat_time2[0:6] +count_new = 'count_' + strat_time[0:6] +count_old = 'count_' + strat_time2[0:6] -df_root_merge = df_root_merge.rename(columns={'count_x':count_new,'count_y':count_old}) +df_root_merge = df_root_merge.rename(columns={'count_x': count_new, 'count_y': count_old}) -df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge[count_new]0.3))].index) +df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge[count_new] < df_root_merge[count_old]) & ( + (df_root_merge[count_old] - df_root_merge[count_new]) / df_root_merge[count_old] > 0.3))].index) if df_root_res.empty == False: if text == '': @@ -81,9 +82,10 @@ topCities = cities.head(70) city_list = list(topCities['cityname']) log.info(f"房源网站维度-数量 city_list : {city_list}") -df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root','city'])['_id'].count()).reset_index() -df_root_city_group1 = df_root_city_group1.rename(columns={'_id':'count'}) -df_root_city_res = df_root_city_group1[(df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count']<100)].reset_index() +df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root', 'city'])['_id'].count()).reset_index() +df_root_city_group1 = df_root_city_group1.rename(columns={'_id': 'count'}) +df_root_city_res = df_root_city_group1[ + (df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count'] < 100)].reset_index() if df_root_city_res.empty == False: if text == '': @@ -93,6 +95,7 @@ path = "/home/airflow/data/jh/price" file_name = f"{now_day}房源样本数量检测.xlsx" air_file_name = path + '/' + file_name + def up_to_nextcloud(): """ 上传统计信息至nextcloud @@ -105,21 +108,17 @@ def up_to_nextcloud(): nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD) nxc.upload_file(air_file_name, nextcloud_file_name) + if text != '': with pd.ExcelWriter(air_file_name) as writer: if df_region_res.empty == False: - df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city","region",'count'], index=False) + df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city", "region", 'count'], index=False) if df_root_city_res.empty == False: - df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root','city','count'], index=False) + df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root', 'city', 'count'], + index=False) if df_root_res.empty == False: - df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root',count_old,count_new], index=False) + df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root', count_old, count_new], + index=False) up_to_nextcloud() token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736' - dingutils.send_text(text,token) - - - - - - - + dingutils.send_text(text, token) diff --git a/dags/requirements.txt b/dags/requirements.txt index ec2f4af..712fbef 100644 --- a/dags/requirements.txt +++ b/dags/requirements.txt @@ -1,3 +1,4 @@ elasticsearch==7.11.0 transbigdata==0.4.15 -pykalman==0.9.5 \ No newline at end of file +pykalman==0.9.5 +matplotlib \ No newline at end of file diff --git a/dags/stat/cityPrice/__init__.py b/dags/stat/cityPrice/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/stat/cityPrice/increase_decrease_city_region.py b/dags/stat/cityPrice/increase_decrease_city_region.py new file mode 100644 index 0000000..36581a0 --- /dev/null +++ b/dags/stat/cityPrice/increase_decrease_city_region.py @@ -0,0 +1,249 @@ +""" +统计城市 区域 价格涨跌幅 +""" +import logging +import pandas as pd +from sqlalchemy import create_engine +from goojia_utils import dateutils +import datetime +import os +pathAll = "/home/airflow/data/cityPrice" +if not os.path.exists(pathAll): + os.makedirs(pathAll) +# 创建数据库引擎 +engine235 = create_engine("postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5432/fangjia?client_encoding=utf8", + echo=False) +pd.set_option('mode.chained_assignment', None) +year_month_day = dateutils.format_date(datetime.datetime.now()) +# 执行SQL查询获取城市价格数据 +city_sql = "SELECT DISTINCT urbn_nm FROM price_stat.avm_city_price" +city_names_set = set(pd.read_sql(city_sql, engine235)['urbn_nm']) +city_names = list(city_names_set) +# 假设这是你的pgsql数据 +all_city_data = pd.DataFrame() +# 执行SQL查询获取区域价格数据 +region_sql = "SELECT DISTINCT urbn_nm,gxda_admndiv_nm FROM price_stat.avm_district_price" +region_data = pd.read_sql(region_sql, engine235) +city_data_useinregion = pd.DataFrame() +# 假设这是你的pgsql数据 +all_region_data = pd.DataFrame() + +def up_to_nextcloud(aim_file_name,aim_file_path): + """ + 上传统计信息至nextcloud + """ + from nextcloud import NextCloud + NEXTCLOUD_URL = "https://nextcloud.fangjia.com" + NEXTCLOUD_USERNAME = "shenjian@fangjia.com" + NEXTCLOUD_PASSWORD = "goojia123456" + nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True) + nxc.upload_file(aim_file_name, aim_file_path) +def city_increase(city_names, all_city_data): + global city_data_useinregion # 声明为全局变量 + # 遍历城市列表,对每个城市进行查询 + for city in city_names: + query = f"SELECT urbn_nm as 城市, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_city_price WHERE urbn_nm = '{city}'" + city_data = pd.read_sql(query, engine235) + + if city_data.empty: + continue + + # 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变 + city_data['original_date'] = city_data['月份'] + + # 将日期列转换为日期类型 + city_data['月份'] = pd.to_datetime(city_data['月份'], format='%Y%m') + + # 对价格数据按照月份排序 + city_data.sort_values(by='月份', inplace=True) + + # 计算涨跌幅 + city_data['涨跌幅'] = city_data['估家计算均价'].diff() / city_data['估家计算均价'].shift(1) * 100 + + # 计算涨跌幅,并保留两位小数,展示为百分比形式 + price_change = city_data['估家计算均价'].pct_change() * 100 + city_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', '') + + # 将原始日期列替换为计算涨跌幅后的日期列 + city_data['月份'] = city_data['original_date'] + + # 删除临时列 + city_data.drop(columns=['original_date'], inplace=True) + + # 将城市数据添加到总的数据中 + all_city_data = pd.concat([all_city_data, city_data], ignore_index=True) + city_data_useinregion = all_city_data + return all_city_data + + +def city_missing(): + df = city_increase(city_names, all_city_data) + df['月份1'] = df['月份'] + # 将日期字符串转换为日期时间对象 + df["月份"] = pd.to_datetime(df["月份"], format="%Y%m") + + # 获取当前时间 + current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0) + + # 计算前一年到上月的时间范围 + one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1) + last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1) + + # 创建一个包含所有可能月份的日期范围 + all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS") + + # 获取每个城市缺失的月份 + missing_months = [] + for city, group in df.groupby("城市"): + available_months = group["月份"] + missing = all_months.difference(available_months) + if len(missing) > 0: + missing_str = ",".join([m.strftime("%Y%m") for m in missing]) + missing_months.append({"城市": city, "缺失月份": missing_str}) + if len(missing_months) > 0: + # 创建DataFrame来存储结果 + missing_df = pd.DataFrame(missing_months) + path = pathAll+'/城市价格数据缺失'+year_month_day+'.xlsx' + missing_df.to_excel(path, index=False) + logging.info(f"城市价格缺失数据已写入Excel文件:{path}") + + cloudPath = '共享/统计/城市区域涨跌幅/' + '城市价格数据缺失'+year_month_day+'.xlsx' + up_to_nextcloud(path, cloudPath) + logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘") + + # print(missing_df) + df['月份'] = df['月份1'] + # 删除月份1列 + df.drop(columns=['月份1'], inplace=True) + output_path = pathAll+'/城市价格涨跌幅'+year_month_day+'.xlsx' + df.to_excel(output_path, index=False) + logging.info(f"城市涨跌幅数据已写入Excel文件:{output_path}") + + cloudPath = '共享/统计/城市区域涨跌幅/' + '城市价格涨跌幅'+year_month_day+'.xlsx' + up_to_nextcloud(output_path, cloudPath) + logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘") + + +def region_increase(region_data, all_region_data): + # 遍历区域列表,对每个区域进行查询 + for index, row in region_data.iterrows(): + city = row['urbn_nm'] + region = row['gxda_admndiv_nm'] + query = f"SELECT urbn_nm as 城市, gxda_admndiv_nm as 区域, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_district_price WHERE urbn_nm = '{city}' AND gxda_admndiv_nm = '{region}'" + region_data = pd.read_sql(query, engine235) + + if region_data.empty: + continue + + # 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变 + region_data['original_date'] = region_data['月份'] + + # 将日期列转换为日期类型 + region_data['月份'] = pd.to_datetime(region_data['月份'], format='%Y%m') + + # 对价格数据按照月份排序 + region_data.sort_values(by='月份', inplace=True) + + # 计算涨跌幅 + region_data['涨跌幅'] = region_data['估家计算均价'].diff() / region_data['估家计算均价'].shift(1) * 100 + + # 计算涨跌幅,并保留两位小数,展示为百分比形式 + price_change = region_data['估家计算均价'].pct_change() * 100 + region_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', + '') + + # 将原始日期列替换为计算涨跌幅后的日期列 + region_data['月份'] = region_data['original_date'] + + # 删除临时列 + region_data.drop(columns=['original_date'], inplace=True) + + # 将区域数据添加到总的数据中 + all_region_data = pd.concat([all_region_data, region_data], ignore_index=True) + + return all_region_data + + +# 城市价个和区域价格比较 +def add_price_comparison_by_month(region_data): + # 遍历区域数据,比较城市和区域的价格,并添加比值超过3倍的情况到区域数据中 + for index, row in region_data.iterrows(): + city = row['城市'] + region = row['区域'] + region_month = row['月份'] + region_price = row['估家计算均价'] + city_data = city_data_useinregion + # 获取城市对应月份的价格 + city_price_row = city_data[(city_data['城市'] == city) & (city_data['月份'] == region_month)] + if city_price_row.empty: + continue + city_price = city_price_row['估家城市均价'].iloc[0] + + # 比较价格并计算比值 + price_ratio = max(city_price, region_price) / min(city_price, region_price) + + # 如果比值超过3倍,将结果添加到区域数据中 + if price_ratio > 3: + region_data.loc[index, '城市区域价格比值超3倍'] = price_ratio + + return region_data + + +def region_missing(): + df = region_increase(region_data, all_region_data) + df['月份1'] = df['月份'] + # 将日期字符串转换为日期时间对象 + df["月份"] = pd.to_datetime(df["月份"], format="%Y%m") + + # 获取当前时间 + current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0) + + # 计算前一年到上月的时间范围 + one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1) + last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1) + + # 创建一个包含所有可能月份的日期范围 + all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS") + + # 获取每个城市-区域组合缺失的月份 + missing_months = [] + for (city, region), group in df.groupby(["城市", "区域"]): + available_months = group["月份"] + missing = all_months.difference(available_months) + if len(missing) > 0: + missing_str = ",".join([m.strftime("%Y%m") for m in missing]) + missing_months.append({"城市": city, "区域": region, "缺失月份": missing_str}) + + if missing_months: + # 创建DataFrame来存储结果 + missing_df = pd.DataFrame(missing_months) + path = pathAll+'/区域价格数据缺失'+year_month_day+'.xlsx' + missing_df.to_excel(path, index=False) + logging.info(f"区域价格缺失数据已写入Excel文件:{path}") + cloudPath = '共享/统计/城市区域涨跌幅/' + '区域价格数据缺失' + year_month_day + '.xlsx' + up_to_nextcloud(path, cloudPath) + logging.info(f"区域价格缺失数据Excel文件已经上传到云盘") + + # 如果区域在 missing_df 中,则将 df 中的该区域数据删除 + for city, region in zip(missing_df['城市'], missing_df['区域']): + df = df[(df['城市'] != city) | (df['区域'] != region)] + + df['月份'] = df['月份1'] + # 删除月份1列 + df.drop(columns=['月份1'], inplace=True) + + # 添加价格比值超过3倍的列 + # print(city_data_useinregion) + df = add_price_comparison_by_month(df) + + output_path = pathAll+'/区域价格涨跌幅'+year_month_day+'.xlsx' + df.to_excel(output_path, index=False) + logging.info(f"区域涨跌幅数据已写入Excel文件:{output_path}") + + cloudPath = '共享/统计/城市区域涨跌幅/' + '区域价格涨跌幅'+year_month_day+'.xlsx' + up_to_nextcloud(output_path, cloudPath) + logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘") + + +city_missing() +region_missing() diff --git a/dags/stat/cityPrice/test.py b/dags/stat/cityPrice/test.py new file mode 100644 index 0000000..4e08b92 --- /dev/null +++ b/dags/stat/cityPrice/test.py @@ -0,0 +1,97 @@ +import logging +import pandas as pd +from sqlalchemy import create_engine + +# 创建数据库引擎 +engine235 = create_engine("postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5432/fangjia?client_encoding=utf8", + echo=False) + +# 执行SQL查询获取区域价格数据 +region_sql = "SELECT DISTINCT urbn_nm,gxda_admndiv_nm FROM price_stat.avm_district_price_copy1" +region_data = pd.read_sql(region_sql, engine235) + +# 假设这是你的pgsql数据 +all_region_data = pd.DataFrame() +pd.set_option('mode.chained_assignment', None) + +def region_increase(region_data, all_region_data): + # 遍历区域列表,对每个区域进行查询 + for index, row in region_data.iterrows(): + city = row['urbn_nm'] + region = row['gxda_admndiv_nm'] + query = f"SELECT urbn_nm as 城市, gxda_admndiv_nm as 区域, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_district_price_copy1 WHERE urbn_nm = '{city}' AND gxda_admndiv_nm = '{region}'" + region_data = pd.read_sql(query, engine235) + + if region_data.empty: + continue + + # 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变 + region_data['original_date'] = region_data['月份'] + + # 将日期列转换为日期类型 + region_data['月份'] = pd.to_datetime(region_data['月份'], format='%Y%m') + + # 对价格数据按照月份排序 + region_data.sort_values(by='月份', inplace=True) + + # 计算涨跌幅 + region_data['涨跌幅'] = region_data['估家计算均价'].diff() / region_data['估家计算均价'].shift(1) * 100 + + # 计算涨跌幅,并保留两位小数,展示为百分比形式 + price_change = region_data['估家计算均价'].pct_change() * 100 + region_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', '') + + # 将原始日期列替换为计算涨跌幅后的日期列 + region_data['月份'] = region_data['original_date'] + + # 删除临时列 + region_data.drop(columns=['original_date'], inplace=True) + + # 将区域数据添加到总的数据中 + all_region_data = pd.concat([all_region_data, region_data], ignore_index=True) + + return all_region_data + +def region_missing(): + df = region_increase(region_data, all_region_data) + df['月份1'] = df['月份'] + # 将日期字符串转换为日期时间对象 + df["月份"] = pd.to_datetime(df["月份"], format="%Y%m") + + # 获取当前时间 + current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0) + + # 计算前一年到上月的时间范围 + one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1) + last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1) + + # 创建一个包含所有可能月份的日期范围 + all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS") + + # 获取每个城市-区域组合缺失的月份 + missing_months = [] + for (city, region), group in df.groupby(["城市", "区域"]): + available_months = group["月份"] + missing = all_months.difference(available_months) + if len(missing) > 0: + missing_str = ",".join([m.strftime("%Y%m") for m in missing]) + missing_months.append({"城市": city, "区域": region, "缺失月份": missing_str}) + + if missing_months: + # 创建DataFrame来存储结果 + missing_df = pd.DataFrame(missing_months) + path = 'E:\\goojia\\file\\excel文件\\airflow\\missing_region.xlsx' + missing_df.to_excel(path, index=False) + + # 如果区域在 missing_df 中,则将 df 中的该区域数据删除 + for city, region in zip(missing_df['城市'], missing_df['区域']): + df = df[(df['城市'] != city) | (df['区域'] != region)] + + df['月份'] = df['月份1'] + # 删除月份1列 + df.drop(columns=['月份1'], inplace=True) + output_path = 'E:\\goojia\\file\\excel文件\\airflow\\all_region_price_with_change.xlsx' + df.to_excel(output_path, index=False) + logging.info(f"区域涨跌幅数据已写入Excel文件:{output_path}") + +region_missing() diff --git a/dags/stat/community/rate.py b/dags/stat/community/rate.py new file mode 100644 index 0000000..6f7c0ce --- /dev/null +++ b/dags/stat/community/rate.py @@ -0,0 +1,52 @@ +""" +小区周边1公里交通地铁入库业务逻辑 +""" +import datetime +import logging +import pandas as pd +from goojia_utils import MAIN_MONGO_CLIENT +from goojia_utils import dateutils +from sklearn.mixture import GaussianMixture +from pymongo import InsertOne, DeleteMany, ReplaceOne, UpdateOne + +logger = logging.getLogger(__name__) + +district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat') +district_analy_data_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_analy_data') +server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') +weekend = list(district_stat_conn.find().sort([("weekend", -1)]).limit(1))[0]['weekend'] +level_map = {"1": "S", "2": "A", "3": "B", "4": "C", "5": "D"} +logger.info(f"开始计算楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}") +for server_area in server_area_conn.find({'display': {'$ne': '全国'}}, {"display": 1}): + city = server_area["display"] + data = pd.DataFrame( + district_stat_conn.find({"city": city, "weekend": weekend}, + {"_id": 0, "region": 1, "district_name": 1, "avg_price": 1})) + if "avg_price" not in data or len(data['avg_price'].unique()) < 5: + continue + data['city'] = city + logger.info(f"开始计算{city}楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}") + # 高斯混合模型 + gmm = GaussianMixture(n_components=5, covariance_type='tied', max_iter=200, n_init=10).fit( + data[["avg_price"]]) + data.loc[:, 'cluster'] = gmm.predict(data[["avg_price"]]) + data.loc[:, 'price_mean_cluster'] = data.groupby('cluster')["avg_price"].transform('mean') + data.loc[:, "target"] = data['price_mean_cluster'].rank(method='dense', ascending=False).astype(int) + data = data.drop(columns=['cluster', 'price_mean_cluster'], axis=1) + data["community_rating"] = data['target'].apply(lambda x: level_map[str(x)]) + opts = [] + for item in data.to_dict('records'): + opts.append(UpdateOne({"city": city, "region": item["region"], "district_name": item["district_name"]}, + {"$set": {"community_rating": item["community_rating"]}}, upsert=True + )) + # 按照每 1000 条分割 updates 列表 + batch_size = 1000 + batches = [opts[i:i + batch_size] for i in range(0, len(opts), batch_size)] + # 逐个批次执行批量更新操作 + cnt = 0 + for batch in batches: + cnt = cnt + 1 + logger.info(f"{city}楼盘评级更新数据库, 进度:{cnt}/{len(batches)}") + result = district_analy_data_conn.bulk_write(batch) + print(f"当前批次更新的文档数量: {result.modified_count}") +logger.info(f"楼盘评级计算结束{dateutils.format_date(datetime.datetime.utcnow(), '%Y-%m-%d')}") diff --git a/dags/stat/dag_community_rating.py b/dags/stat/dag_community_rating.py new file mode 100644 index 0000000..b5dc78b --- /dev/null +++ b/dags/stat/dag_community_rating.py @@ -0,0 +1,17 @@ +from airflow import DAG +from airflow.operators.bash import BashOperator +import pendulum + +with DAG( + dag_id="community_rating_dag", + # 每月的最后一天运行 + schedule_interval="0 20 * * 6", + start_date=pendulum.datetime(2024, 7, 12, tz="Asia/Shanghai"), + description="小区楼盘评级计算", + catchup=True +) as dag: + t1 = BashOperator( + task_id='community_rating', + bash_command='python /opt/airflow/dags/repo/dags/stat/community/rate.py' + ) + t1 \ No newline at end of file diff --git a/dags/stat/dag_increase_decrease_city_region.py b/dags/stat/dag_increase_decrease_city_region.py new file mode 100644 index 0000000..ab63f58 --- /dev/null +++ b/dags/stat/dag_increase_decrease_city_region.py @@ -0,0 +1,17 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + dag_id="increase_decrease_city_region_dag", + # 每月的最后一天运行 + schedule_interval="0 10 L * *", + start_date=pendulum.datetime(2024, 5, 30, tz="Asia/Shanghai"), + description="城市区域涨跌幅数据统计", + catchup=True +) as dag: + t1 = BashOperator( + task_id='increase_decrease_city_region', + bash_command='python /opt/airflow/dags/repo/dags/stat/cityPrice/increase_decrease_city_region.py', + ) + t1