From 3cda5cd2110d24d2155cbf17fe59b9d364e40ecd Mon Sep 17 00:00:00 2001 From: muliangshu <283299495@qq.com> Date: Mon, 22 Jul 2024 17:13:48 +0800 Subject: [PATCH] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E5=9D=87=E4=BB=B7?= =?UTF-8?q?=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/configs/citymap.py | 109 +++++++++++++++++++++++++++++++ dags/jh/price/mongo_client.py | 25 +++++++ dags/jh/price/mongodb_connect.py | 81 +++++++++++++++++++++++ dags/jh/price/pgsql_client.py | 42 ++++++++++++ dags/jh/price/pqsql_connect.py | 100 ++++++++++++++++++++++++++++ dags/jh/price/price_third_gxd.py | 49 ++++++++++++++ 6 files changed, 406 insertions(+) create mode 100644 dags/configs/citymap.py create mode 100644 dags/jh/price/mongo_client.py create mode 100644 dags/jh/price/mongodb_connect.py create mode 100644 dags/jh/price/pgsql_client.py create mode 100644 dags/jh/price/pqsql_connect.py create mode 100644 dags/jh/price/price_third_gxd.py diff --git a/dags/configs/citymap.py b/dags/configs/citymap.py new file mode 100644 index 0000000..c58febd --- /dev/null +++ b/dags/configs/citymap.py @@ -0,0 +1,109 @@ +import pandas as pd +city_name_id_map = {'七台河': '230900', '万宁': '469006', '三亚': '460200', '东莞': '441900', '三明': '350400', '三沙': '460300', '三门峡': '411200', + '上海': '310000', '上饶': '361100', '东方': '469007', '中山': '442000', '东营': '370500', '中卫': '640500', '临夏': '622900', + '临汾': '141000', '临沂': '371300', '临沧': '530900', '临高': '469024', '丹东': '210600', '丽水': '331100', '丽江': '530700', + '乌兰察布': '150900', '乌海': '150300', '乌鲁木齐': '650100', '乐东': '469027', '乐山': '511100', '九江': '360400', + '云浮': '445300', '五家渠': '659004', '五指山': '469001', '亳州': '341600', '仙桃': '429004', '伊春': '230700', + '伊犁': '654000', '佛山': '440600', '佳木斯': '230800', '保亭': '469029', '保定': '130600', '保山': '530500', '信阳': '411500', + '儋州': '460400', '克州': '653000', '克拉玛依': '650200', '六安': '341500', '六盘水': '520200', '兰州': '620100', + '兴安': '152200', '内江': '511000', '凉山': '513400', '包头': '150200', '北京': '110000', '北屯': '659005', '北海': '450500', + '十堰': '420300', '南京': '320100', '南充': '511300', '南宁': '450100', '南平': '350700', '南昌': '360100', '南通': '320600', + '南阳': '411300', '博州': '652700', '厦门': '350200', '双河': '659007', '双鸭山': '230500', '可克达拉': '659008', + '台州': '331000', '合肥': '340100', '吉安': '360800', '吉林': '220200', '吐鲁番': '650400', '吕梁': '141100', + '吴忠': '640300', '周口': '411600', '呼伦贝尔': '150700', '呼和浩特': '150100', '和田': '653200', '咸宁': '421200', + '咸阳': '610400', '哈密': '650500', '哈尔滨': '230100', '唐山': '130200', '商丘': '411400', '商洛': '611000', + '喀什': '653100', '嘉兴': '330400', '嘉峪关': '620200', '四平': '220300', '固原': '640400', '图木舒克': '659003', + '塔城': '654200', '大兴安岭': '232700', '大同': '140200', '大庆': '230600', '大理': '532900', '大连': '210200', + '天水': '620500', '天津': '120000', '天门': '429006', '太原': '140100', '威海': '371000', '娄底': '431300', '孝感': '420900', + '宁德': '350900', '宁波': '330200', '安庆': '340800', '安康': '610900', '安阳': '410500', '安顺': '520400', '定安': '469021', + '定西': '621100', '宜宾': '511500', '宜昌': '420500', '宜春': '360900', '宝鸡': '610300', '宣城': '341800', '宿州': '341300', + '宿迁': '321300', '屯昌': '469022', '山南': '540500', '岳阳': '430600', '崇左': '451400', '巴中': '511900', + '巴彦淖尔': '150800', '巴州': '652800', '常州': '320400', '常德': '430700', '平凉': '620800', '平顶山': '410400', + '广元': '510800', '广安': '511600', '广州': '440100', '庆阳': '621000', '廊坊': '131000', '延安': '610600', + '延边': '222400', '开封': '410200', '张家口': '130700', '张家界': '430800', '张掖': '620700', '徐州': '320300', + '德宏': '533100', '德州': '371400', '德阳': '510600', '忻州': '140900', '怀化': '431200', '怒江': '533300', '恩施': '422800', + '惠州': '441300', '成都': '510100', '扬州': '321000', '承德': '130800', '抚州': '361000', '抚顺': '210400', '拉萨': '540100', + '揭阳': '445200', '攀枝花': '510400', '文山': '532600', '文昌': '469005', '新乡': '410700', '新余': '360500', + '无锡': '320200', '日喀则': '540200', '日照': '371100', '昆明': '530100', '昆玉': '659009', '昌吉': '652300', + '昌江': '469026', '昌都': '540300', '昭通': '530600', '晋中': '140700', '晋城': '140500', '普洱': '530800', '景德镇': '360200', + '曲靖': '530300', '朔州': '140600', '朝阳': '211300', '本溪': '210500', '来宾': '451300', '杭州': '330100', '松原': '220700', + '林芝': '540400', '果洛': '632600', '枣庄': '370400', '柳州': '450200', '株洲': '430200', '桂林': '450300', '梅州': '441400', + '梧州': '450400', '楚雄': '532300', '榆林': '610800', '武威': '620600', '武汉': '420100', '毕节': '520500', '永州': '431100', + '汉中': '610700', '汕头': '440500', '汕尾': '441500', '江门': '440700', '池州': '341700', '沈阳': '210100', '沧州': '130900', + '河池': '451200', '河源': '441600', '泉州': '350500', '泰安': '370900', '泰州': '321200', '泸州': '510500', '洛阳': '410300', + '济南': '370100', '济宁': '370800', '济源': '419001', '海东': '630200', '海北': '632200', '海南州': '632500', + '海口': '460100', '海西': '632800', '淄博': '370300', '淮北': '340600', '淮南': '340400', '淮安': '320800', '深圳': '440300', + '清远': '441800', '温州': '330300', '渭南': '610500', '湖州': '330500', '湘潭': '430300', '湘西': '433100', '湛江': '440800', + '滁州': '341100', '滨州': '371600', '漯河': '411100', '漳州': '350600', '潍坊': '370700', '潜江': '429005', '潮州': '445100', + '澄迈': '469023', '濮阳': '410900', '烟台': '370600', '焦作': '410800', '牡丹江': '231000', '玉林': '450900', + '玉树': '632700', '玉溪': '530400', '珠海': '440400', '琼中': '469030', '琼海': '469002', '甘南': '623000', '甘孜': '513300', + '白城': '220800', '白山': '220600', '白沙': '469025', '白银': '620400', '百色': '451000', '益阳': '430900', '盐城': '320900', + '盘锦': '211100', '眉山': '511400', '石嘴山': '640200', '石家庄': '130100', '石河子': '659001', '神农架': '429021', + '福州': '350100', '秦皇岛': '130300', '红河': '532500', '绍兴': '330600', '绥化': '231200', '绵阳': '510700', + '聊城': '371500', '肇庆': '441200', '自贡': '510300', '舟山': '330900', '芜湖': '340200', '苏州': '320500', '茂名': '440900', + '荆州': '421000', '荆门': '420800', '菏泽': '371700', '莆田': '350300', '萍乡': '360300', '营口': '210800', + '葫芦岛': '211400', '蚌埠': '340300', '衡水': '131100', '衡阳': '430400', '衢州': '330800', '襄阳': '420600', '西双版纳': '532800', + '西宁': '630100', '西安': '610100', '许昌': '411000', '贵港': '450800', '贵阳': '520100', '贺州': '451100', '资阳': '512000', + '赣州': '360700', '赤峰': '150400', '辽源': '220400', '辽阳': '211000', '达州': '511700', '运城': '140800', '连云港': '320700', + '迪庆': '533400', '通化': '220500', '通辽': '150500', '遂宁': '510900', '遵义': '520300', '邢台': '130500', '那曲': '540600', + '邯郸': '130400', '邵阳': '430500', '郑州': '410100', '郴州': '431000', '鄂尔多斯': '150600', '鄂州': '420700', '酒泉': '620900', + '重庆': '500000', '金华': '330700', '金昌': '620300', '钦州': '450700', '铁岭': '211200', '铁门关': '659006', '铜仁': '520600', + '铜川': '610200', '铜陵': '340700', '银川': '640100', '锡林郭勒': '152500', '锦州': '210700', '镇江': '321100', '长春': '220100', + '长沙': '430100', '长治': '140400', '阜新': '210900', '阜阳': '341200', '防城港': '450600', '阳江': '441700', '阳泉': '140300', + '阿克苏': '652900', '阿勒泰': '654300', '阿坝': '513200', '阿拉善': '152900', '阿拉尔': '659002', '阿里': '542500', '陇南': '621200', + '陵水': '469028', '随州': '421300', '雅安': '511800', '青岛': '370200', '鞍山': '210300', '韶关': '440200', + '驻马店': '411700', '鸡西': '230300', '鹤壁': '410600', '鹤岗': '230400', '鹰潭': '360600', '黄冈': '421100', '黄南': '632300', + '黄山': '341000', '黄石': '420200', '黑河': '231100', '黔东南': '522600', '黔南': '522700', '黔西南': '522300', '齐齐哈尔': '230200', + '龙岩': '350800', "马鞍山": "340500"} +third_data_source_id_map = {'01': '诸葛找房','02': '安居客','03': '房天下','04': '链家','05': '城市居家', + '06': '麦田','07': '贝壳','08': '淘房356','09': '中原','10': '兔博士', + '11': 'Q房','12': '房探','13': '房多多','14': '我爱我家','15': '禧泰数据', + '16': '吉屋网','17': '乐居','18': '21世纪','19': '房产超市','20': '58同城'} +city_id_name_map = {k: v for v, k in city_name_id_map.items()} + + +def get_city_name(city_id): + """ + 通过城市ID获取城市名 + :param city_id: + :return: + """ + return city_id_name_map[city_id] if city_id in city_id_name_map else None + + +def get_city_id(city_name): + """ + 通过城市名获取城市ID + :param city_name: + :return: + """ + return city_name_id_map[city_name] if city_name in city_name_id_map else None + + +def is_city_id(city): + """ + 判断是不是合法的cityid + :param city: + :return: + """ + return city in city_id_name_map + + +def is_city_name(city): + """ + 判断是不是合法的城市名 + :param city: + :return: + """ + return city in city_name_id_map + + +def get_city_data(): + """ + 返回所有的城市数据DataFrame[cityid,cityname] + :return: + """ + data = list() + for city_id in city_id_name_map.keys(): + data.append({"cityid": city_id, "cityname": city_id_name_map[city_id]}) + return pd.DataFrame(data) diff --git a/dags/jh/price/mongo_client.py b/dags/jh/price/mongo_client.py new file mode 100644 index 0000000..a1e3819 --- /dev/null +++ b/dags/jh/price/mongo_client.py @@ -0,0 +1,25 @@ +from pymongo import MongoClient +def getMongo(*args): + if len(args) == 6: + con = MongoClient(args[0], args[1]) # 获取链接\ + db_auth = con.fangjia_craw + if args[2] =="vip_site": + db_auth.authenticate(args[4], args[5],"fangjia_craw") + else: + db_auth.authenticate(args[4], args[5],"admin") + db = con[args[2]] # 关联数据库 + collection = db[args[3]] # 关联数据表 + elif len(args) == 4: + con = MongoClient(args[0], args[1]) # 获取链接 + db = con[args[2]] # 关联数据库 + collection = db[args[3]] # 关联数据表 + return collection +#线上小区库 +seaweed_on = getMongo("192.168.0.43",28017,"fangjia","seaweed","developer","G00IDCDEV12385") +sz_price_on = getMongo("192.168.0.43",28017,"fangjia","sz_price","developer","G00IDCDEV12385") +district_stat_on = getMongo("192.168.0.43",28017,"fangjia_stat","district_stat","developer","G00IDCDEV12385") +seaweed_map = getMongo("192.168.0.235",27018,"fangjia","seaweedmap") +web_community_check = getMongo("192.168.0.235",27018,"fangjia","web_community_check") +web_invalid_comm = getMongo("192.168.0.235",27018,"fangjia","web_invalid_comm") +addr_nine_level = getMongo("192.168.0.206",27020,"etl","nine_level") +gxd_price = getMongo("192.168.0.43",28017,"fangjia","confirm_price_data","developer","G00IDCDEV12385") \ No newline at end of file diff --git a/dags/jh/price/mongodb_connect.py b/dags/jh/price/mongodb_connect.py new file mode 100644 index 0000000..621e0c5 --- /dev/null +++ b/dags/jh/price/mongodb_connect.py @@ -0,0 +1,81 @@ +from mongo_client import seaweed_on,seaweed_map,addr_nine_level,district_stat_on,gxd_price +import pandas as pd +import datetime +import pytz +pd.set_option ('display.max_colwidth',5) + +def parse_date(time, fm='%Y-%m-%d'): + pacific = pytz.timezone('Asia/Shanghai') + return pacific.localize(datetime.datetime.strptime(time, fm)) +def get_last_weekend(): + sunday = datetime.datetime.utcnow() + sunday = sunday.replace(hour=0, minute=0, second=0, microsecond=0) + one_day = datetime.timedelta(days=1) + while sunday.weekday() != 6: + sunday -= one_day + pacific = pytz.timezone('Asia/Shanghai') + return pacific.localize(sunday) + +confirm_date = parse_date("2024-05-20") +def get_seaweed(city=None,status=0): + seaweed_fields = {"city": 1, "region": 1, "name": 1,'estate_type2':1,'cat':1,"address":1,"alias":1,"lat":1,"lng":1,"addr_alias":1,"quality":1,"name_cat":1,"mapping_alias":1,"m_date":1,"is_new":1} + # seaweed_fields = {"_id": 0} + if city: + if city in ['阿坝','玉树','甘孜','德宏','迪庆','海北','海西','黄南','海南','怒江','文山','西双版纳','凉山','果洛']: + city = city+'州' + elif city in ['锡林郭勒','兴安','阿拉善']: + city = city+'盟' + elif city=='克州': + city = '克孜勒苏' + elif city=='博州': + city = '博尔塔拉' + elif city=='巴州': + city = '巴音郭楞' + seaweed_condition = {"cat": "district", "status": status, 'city':city} + else: + seaweed_condition = {"cat": "district", "status": status} + seaweed_data = seaweed_on.find(seaweed_condition, seaweed_fields) + seaweed_df = pd.DataFrame(seaweed_data) + seaweed_df = seaweed_df.rename(columns={'estate_type2': 'property_type'}) + return seaweed_df + +def get_map(city=None): + # seaweed_fields = {"city": 1, "region": 1, "name": 1,"address":1,"alias":1,"lat":1,"lng":1,"estate_type2":1,"addr_alias":1} + seaweed_fields = {"_id": 0} + if city: + seaweed_condition = {'city':city} + else: + seaweed_condition = {} + seaweed_map_data = seaweed_map.find(seaweed_condition, seaweed_fields) + seaweed_map_df = pd.DataFrame(seaweed_map_data) + return seaweed_map_df + +def get_addr_nine_level(count): + _fields = {"address":1,"province":1,"city": 1, "district": 1,"township": 1, "name": 1,"":1} + _condition = {'address':{"$exists": True, "$ne": None},"remark":{"$exists": False}} + addr_nine_data = addr_nine_level.find(_condition,_fields).limit(count) + addr_nine_df = pd.DataFrame(addr_nine_data) + return addr_nine_df + +def get_district_stat_on(city=None): + price_date = get_last_weekend() + stat_fields = {"_id": 0, "city": 1, "region": 1, "district_name": 1, "avg_price": 1,"avg_price_2b":1} + if city: + stat_condition = {'city':city,"weekend": price_date} + else: + stat_condition = {"weekend": price_date} + stat_data = district_stat_on.find(stat_condition, stat_fields) + stat_df = pd.DataFrame(stat_data) + stat_df = stat_df.rename(columns={'district_name': 'name'}) + return stat_df + +def get_gxd_price(): + gxd_fields = {"_id": 0} + gxd_condition = {'source':'GXD'} + gxd_price_data = gxd_price.find(gxd_condition, gxd_fields) + gxd_price_df = pd.DataFrame(gxd_price_data) + return gxd_price_df +if __name__ == '__main__': + city = '厦门' + df = get_gxd_price() + print(df.head()) \ No newline at end of file diff --git a/dags/jh/price/pgsql_client.py b/dags/jh/price/pgsql_client.py new file mode 100644 index 0000000..beca0ec --- /dev/null +++ b/dags/jh/price/pgsql_client.py @@ -0,0 +1,42 @@ +import os +import pandas as pd +from pandas import DataFrame +from retrying import retry +from sqlalchemy import create_engine + +# os.environ["NLS_LANG"] = "AMERICAN_AMERICA.AL32UTF8" + + +class PgsqlDb: + """ + PgsqlDb Database Base Class + """ + + def __init__(self,db): + if db == 'warehouse': + url=f"postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/{db}" + elif db == 'fangjia': + url = f"postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5432/{db}" + self.engine = create_engine(url, echo=False,client_encoding='utf-8') + @retry(stop_max_attempt_number=3, wait_random_min=1000, wait_random_max=3000) + def read(self, sql) -> DataFrame: + """ + 读取数据 + :param sql: + :return: + """ + data_df = pd.read_sql(sql, con=self.engine) + return data_df + + @retry(stop_max_attempt_number=3, wait_random_min=1000, wait_random_max=3000) + def count(self, sql) -> int: + """ + 执行统计类SQL + :param sql: eg:select count(1) from t + :return: + """ + return pd.read_sql_query(sql, con=self.engine).iat[0, 0] + def sql_execute(self,sql): + self.engine.execute(sql) +if __name__ == '__main__': + pgdb = PgsqlDb() \ No newline at end of file diff --git a/dags/jh/price/pqsql_connect.py b/dags/jh/price/pqsql_connect.py new file mode 100644 index 0000000..3a3ae7f --- /dev/null +++ b/dags/jh/price/pqsql_connect.py @@ -0,0 +1,100 @@ +import pandas as pd +import os +from pgsql_client import PgsqlDb +os.environ['NLS_LANG'] = 'SAMPLIFIED CHINESE_CHINA.UTF8' +pgdb = PgsqlDb('warehouse') +pgdb_new = PgsqlDb('fangjia') +def get_community_base_price(cityid,yearmonth): + """ + 查询提供建行小区数据 + :param cr: + :return: + """ + sql = f"select gujia_admndiv_nm,gujia_bds_id,urbn_nm,comm_nm,gujiabds_clc_yrmo_val,gujia_comm_basic_prc,gujia_bds_clmtd from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='{cityid}' and gujiabds_clc_yrmo_val={yearmonth}" + data_df = pgdb.read(sql) + data_df = data_df.rename(columns={'gujia_admndiv_nm':'district','gujia_bds_id': '_id','urbn_nm':'city','comm_nm':'name','gujiabds_clc_yrmo_val':'yearmonth','gujia_comm_basic_prc':'price','gujia_bds_clmtd':'type'}) + return data_df + +def get_community_base_price_new(cityid,yearmonth): + """ + 查询提供建行小区数据 + :param cr: + :return: + """ + sql = f"select gujia_bds_id,urbn_nm,comm_nm,gujiabds_clc_yrmo_val,gujia_comm_basic_prc,gujia_bds_clmtd from price_stat.avm_community_base_price where urbn_id='{cityid}' and gujiabds_clc_yrmo_val={yearmonth}" + data_df = pgdb_new.read(sql) + data_df = data_df.rename(columns={'gujia_bds_id': '_id','urbn_nm':'city_copy','comm_nm':'name_copy','gujiabds_clc_yrmo_val':'yearmonth','gujia_comm_basic_prc':'price_new','gujia_bds_clmtd':'type_new'}) + return data_df +def get_district_polyline(cityid=None,cr=None): + """ + 查询提供建行小区数据 + :param cr: + :return: + """ + if cityid: + sql = f"select * from bigdata_district_polyline_gd where cityid='{cityid}'" + else: + sql = f"select distinct city,distname from bigdata_district_polyline_gd" + queryse = cr.execute(sql) + columns = [i[0].lower() for i in queryse.description] + rs = cr.fetchall() + data_df = pd.DataFrame(rs,columns=columns) + return data_df +def get_community_ccb(cityid): + sql = f"select gujia_comm_id,gujia_admndiv_nm,comm_nm from base.t0831_bigdata_community_gj_h where urbn_id='{cityid}' and gujia_bds_bds_st=0" + data_df = pgdb.read(sql) + data_df = data_df.rename(columns={'gujia_comm_id': '_id','gujia_admndiv_nm':'region','comm_nm':'name'}) + return data_df + +def get_bak_community(cityid,cr): + """ + 查询小区数据 + :param cr: + :return: + """ + sql = f"select a.*,b.longitude,b.latitude,c.distname from bigdata_community_bak a left join (select cityid, objectid, LONGITUDE, LATITUDE from bigdata_position_bak where objecttype = '5' and positiontype = '1') b on a.cityid = b.cityid and a.commid = b.objectid left join bigdata_district_bak c on a.cityid=c.cityid and a.distid=c.distid where a.cityid = '{cityid}'" + queryse = cr.execute(sql) + columns = [i[0].lower() for i in queryse.description] + rs = cr.fetchall() + data_df = pd.DataFrame(rs,columns=columns) + return data_df +def avm_price_copy(cityid,yearmonth): + sql = f"insert into t_app_p9pha_plda.avm_community_base_price_gj_h_copy select * from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='{cityid}' and gujiabds_clc_yrmo_val={yearmonth}" + pgdb.sql_execute(sql) + +def get_sample(cityid): + sql = f"select urbn_id,gujia_comm_id,comm_nm,gujia_bds_unitprc,guiabconstructionarea from base.t0831_bigdata_forsale_gj_h where urbn_id='{cityid}' and gujia_hs_src_dtsrc in ('1','4') and anc_tm >= TO_DATE('2023-04-01','YYYY-MM-DD')" + data_df = pgdb.read(sql) + return data_df + +def get_deal(cityid): + sql = f"select urbn_id,gujia_comm_id,comm_nm,guiabconstructionarea,gujia_bds_trdtckt_prc from base.t0831_bigdata_deal_gj_h where urbn_id='{cityid}' and txn_dt >= TO_DATE('2024-03-01','YYYY-MM-DD') and txn_dt < TO_DATE('2024-04-01','YYYY-MM-DD')" + data_df = pgdb.read(sql) + data_df = data_df.rename(columns={'urbn_id': 'cityid','gujia_comm_id': '_id','comm_nm': 'name', 'guiabconstructionarea': 'area','gujia_bds_trdtckt_prc': 'deal_price'}) + return data_df + +def get_deal_data(cityid): + sql = f"select * from base.t0831_bigdata_deal_gj_h where urbn_id='{cityid}' and txn_dt >= TO_DATE('2024-03-01','YYYY-MM-DD') and txn_dt < TO_DATE('2024-04-01','YYYY-MM-DD')" + data_df = pgdb.read(sql) + return data_df + +def get_region_price(cityid,yearmonth): + sql = f"select urbn_id,gujia_admndiv_id,gxda_admndiv_nm,gujia_bds_urbn_avprc from t_app_p9pha_plda.avm_district_price_gj_h where urbn_id='{cityid}' and gujiabds_clc_yrmo_val={yearmonth}" + data_df = pgdb.read(sql) + data_df = data_df.rename(columns={'urbn_id': 'cityid','gujia_admndiv_id': 'district_id','gxda_admndiv_nm': 'district_name', 'gujia_bds_urbn_avprc': 'district_price'}) + return data_df +def get_evaluation_third_price(cityid,yearmonth,type='第三方'): + sql = f"select urbn_id,gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where urbn_id='{cityid}' and gujiabds_clc_yrmo_val={yearmonth} and gujia_data_tp='{type}' " + data_df = pgdb.read(sql) + data_df = data_df.rename(columns={'urbn_id': 'cityid','gujiabds_to_mkprc_mny': 'third_price','gujia_comm_id':'_id'}) + return data_df + +def get_third_price(cityid,yearmonth): + sql = f"select urbn_id,gujia_bds_id,comm_nm,gujia_dtsr_dsc,gujia_bds_comm_avprc from base.t0831_bigdata_third_party_price_gj_h where urbn_id='{cityid}' and gujiabds_avprc_dt_val={yearmonth} and gujia_dtsr_dsc in ('链家','安居客') " + data_df = pgdb.read(sql) + data_df = data_df.rename(columns={'urbn_id': 'cityid','gujia_bds_id': '_id','gujia_dtsr_dsc': 'source','gujia_bds_comm_avprc': 'price'}) + return data_df +if __name__ == '__main__': + city='常州' + yearmonth=202302 + get_community_base_price(city=city,yearmonth=yearmonth) \ No newline at end of file diff --git a/dags/jh/price/price_third_gxd.py b/dags/jh/price/price_third_gxd.py new file mode 100644 index 0000000..a7f5ec5 --- /dev/null +++ b/dags/jh/price/price_third_gxd.py @@ -0,0 +1,49 @@ +import pandas as pd +from dags.configs.citymap import city_name_id_map +from pqsql_connect import get_community_base_price,get_evaluation_third_price,get_third_price +from mongodb_connect import get_district_stat_on,get_seaweed,get_gxd_price +yearmonth='202404' +latest_year_month = '202403' +df_concat = pd.DataFrame() +for city in city_name_id_map: + print(city) + city_id = city_name_id_map[city] + base_price = get_community_base_price(city_id, yearmonth=yearmonth) + base_price.rename(columns={'price': '当月均价', 'type': '计算方式'}, inplace=True) + base_price_last = get_community_base_price(city_id, yearmonth=latest_year_month) + base_price_last.rename(columns={'price': '上月均价', 'type': '上月计算方式'}, inplace=True) + get_evaluation_third_price_df = get_evaluation_third_price(city_id, yearmonth=yearmonth,type='第三方') + get_evaluation_third_price_df.rename(columns={'third_price': '第三方对标价格'}, inplace=True) + deal_price_df = get_evaluation_third_price(city_id, yearmonth=yearmonth,type='成交') + deal_id = deal_price_df['_id'].tolist() + get_evaluation_third_price_df = get_evaluation_third_price_df[~(get_evaluation_third_price_df['_id'].isin(deal_id))] + third_price = get_third_price(cityid=city_id, yearmonth=latest_year_month) + lianjia_price =third_price[third_price['source']=='链家'].drop_duplicates(subset=["_id"], keep="first") + lianjia_price.rename(columns={'price': '链家价格'}, inplace=True) + anjuke_price =third_price[third_price['source']=='安居客'].drop_duplicates(subset=["_id"], keep="first") + anjuke_price.rename(columns={'price': '安居客价格'}, inplace=True) + base_price_gj = get_district_stat_on(city) + seaweed_df = get_seaweed(city) + seaweed_df['_id'] = seaweed_df['_id'].astype('str') + if base_price_gj.empty: + continue + base_price_gj = base_price_gj.merge(seaweed_df[['_id', 'region', 'name']], on=['region', 'name'], how='left') + base_price_gj.rename(columns={'avg_price': '房价网2C价格', 'avg_price_2b': '房价网2B价格'}, inplace=True) + base_price_gxd = get_gxd_price() + base_price_gxd.rename(columns={'district_name': 'name', 'price': '国信达价格'}, inplace=True) + data_df_temp = seaweed_df[['_id','city', 'region', 'name']].merge(base_price_gxd[['city', 'region', 'name', '国信达价格', 'c_date']], on=['city', 'region', 'name']) + data_df_temp = data_df_temp.sort_values(by="c_date", ascending=False) + data_df_temp = data_df_temp.drop_duplicates(subset=["city", "region", "name"], keep="first") + if base_price.empty or base_price_last.empty or get_evaluation_third_price_df.empty: + continue + price_merge = base_price.merge(base_price_last[['_id', '上月均价', '上月计算方式']], on='_id', how='left') + price_merge = price_merge.merge(get_evaluation_third_price_df[['_id','第三方对标价格']],on='_id') + price_merge = price_merge.merge(lianjia_price[['_id','链家价格']],on='_id',how='left') + price_merge = price_merge.merge(anjuke_price[['_id','安居客价格']],on='_id',how='left') + price_merge = price_merge.merge(base_price_gj[['_id','房价网2C价格','房价网2B价格']], on='_id', how='left') + price_merge = price_merge.merge(data_df_temp[['_id','国信达价格']], on='_id', how='left') + price_merge.rename(columns={'city': '城市', 'district': '区域', '_id': '小区ID', 'name': '小区名','yearmonth':'月份'}, inplace=True) + df_concat = pd.concat([df_concat,price_merge]) +df_concat.to_excel('小区第三方均价输出.xlsx', encoding='utf-8', index=False) + +