第三方均价输出
This commit is contained in:
@@ -0,0 +1,109 @@
|
||||
import pandas as pd
|
||||
city_name_id_map = {'七台河': '230900', '万宁': '469006', '三亚': '460200', '东莞': '441900', '三明': '350400', '三沙': '460300', '三门峡': '411200',
|
||||
'上海': '310000', '上饶': '361100', '东方': '469007', '中山': '442000', '东营': '370500', '中卫': '640500', '临夏': '622900',
|
||||
'临汾': '141000', '临沂': '371300', '临沧': '530900', '临高': '469024', '丹东': '210600', '丽水': '331100', '丽江': '530700',
|
||||
'乌兰察布': '150900', '乌海': '150300', '乌鲁木齐': '650100', '乐东': '469027', '乐山': '511100', '九江': '360400',
|
||||
'云浮': '445300', '五家渠': '659004', '五指山': '469001', '亳州': '341600', '仙桃': '429004', '伊春': '230700',
|
||||
'伊犁': '654000', '佛山': '440600', '佳木斯': '230800', '保亭': '469029', '保定': '130600', '保山': '530500', '信阳': '411500',
|
||||
'儋州': '460400', '克州': '653000', '克拉玛依': '650200', '六安': '341500', '六盘水': '520200', '兰州': '620100',
|
||||
'兴安': '152200', '内江': '511000', '凉山': '513400', '包头': '150200', '北京': '110000', '北屯': '659005', '北海': '450500',
|
||||
'十堰': '420300', '南京': '320100', '南充': '511300', '南宁': '450100', '南平': '350700', '南昌': '360100', '南通': '320600',
|
||||
'南阳': '411300', '博州': '652700', '厦门': '350200', '双河': '659007', '双鸭山': '230500', '可克达拉': '659008',
|
||||
'台州': '331000', '合肥': '340100', '吉安': '360800', '吉林': '220200', '吐鲁番': '650400', '吕梁': '141100',
|
||||
'吴忠': '640300', '周口': '411600', '呼伦贝尔': '150700', '呼和浩特': '150100', '和田': '653200', '咸宁': '421200',
|
||||
'咸阳': '610400', '哈密': '650500', '哈尔滨': '230100', '唐山': '130200', '商丘': '411400', '商洛': '611000',
|
||||
'喀什': '653100', '嘉兴': '330400', '嘉峪关': '620200', '四平': '220300', '固原': '640400', '图木舒克': '659003',
|
||||
'塔城': '654200', '大兴安岭': '232700', '大同': '140200', '大庆': '230600', '大理': '532900', '大连': '210200',
|
||||
'天水': '620500', '天津': '120000', '天门': '429006', '太原': '140100', '威海': '371000', '娄底': '431300', '孝感': '420900',
|
||||
'宁德': '350900', '宁波': '330200', '安庆': '340800', '安康': '610900', '安阳': '410500', '安顺': '520400', '定安': '469021',
|
||||
'定西': '621100', '宜宾': '511500', '宜昌': '420500', '宜春': '360900', '宝鸡': '610300', '宣城': '341800', '宿州': '341300',
|
||||
'宿迁': '321300', '屯昌': '469022', '山南': '540500', '岳阳': '430600', '崇左': '451400', '巴中': '511900',
|
||||
'巴彦淖尔': '150800', '巴州': '652800', '常州': '320400', '常德': '430700', '平凉': '620800', '平顶山': '410400',
|
||||
'广元': '510800', '广安': '511600', '广州': '440100', '庆阳': '621000', '廊坊': '131000', '延安': '610600',
|
||||
'延边': '222400', '开封': '410200', '张家口': '130700', '张家界': '430800', '张掖': '620700', '徐州': '320300',
|
||||
'德宏': '533100', '德州': '371400', '德阳': '510600', '忻州': '140900', '怀化': '431200', '怒江': '533300', '恩施': '422800',
|
||||
'惠州': '441300', '成都': '510100', '扬州': '321000', '承德': '130800', '抚州': '361000', '抚顺': '210400', '拉萨': '540100',
|
||||
'揭阳': '445200', '攀枝花': '510400', '文山': '532600', '文昌': '469005', '新乡': '410700', '新余': '360500',
|
||||
'无锡': '320200', '日喀则': '540200', '日照': '371100', '昆明': '530100', '昆玉': '659009', '昌吉': '652300',
|
||||
'昌江': '469026', '昌都': '540300', '昭通': '530600', '晋中': '140700', '晋城': '140500', '普洱': '530800', '景德镇': '360200',
|
||||
'曲靖': '530300', '朔州': '140600', '朝阳': '211300', '本溪': '210500', '来宾': '451300', '杭州': '330100', '松原': '220700',
|
||||
'林芝': '540400', '果洛': '632600', '枣庄': '370400', '柳州': '450200', '株洲': '430200', '桂林': '450300', '梅州': '441400',
|
||||
'梧州': '450400', '楚雄': '532300', '榆林': '610800', '武威': '620600', '武汉': '420100', '毕节': '520500', '永州': '431100',
|
||||
'汉中': '610700', '汕头': '440500', '汕尾': '441500', '江门': '440700', '池州': '341700', '沈阳': '210100', '沧州': '130900',
|
||||
'河池': '451200', '河源': '441600', '泉州': '350500', '泰安': '370900', '泰州': '321200', '泸州': '510500', '洛阳': '410300',
|
||||
'济南': '370100', '济宁': '370800', '济源': '419001', '海东': '630200', '海北': '632200', '海南州': '632500',
|
||||
'海口': '460100', '海西': '632800', '淄博': '370300', '淮北': '340600', '淮南': '340400', '淮安': '320800', '深圳': '440300',
|
||||
'清远': '441800', '温州': '330300', '渭南': '610500', '湖州': '330500', '湘潭': '430300', '湘西': '433100', '湛江': '440800',
|
||||
'滁州': '341100', '滨州': '371600', '漯河': '411100', '漳州': '350600', '潍坊': '370700', '潜江': '429005', '潮州': '445100',
|
||||
'澄迈': '469023', '濮阳': '410900', '烟台': '370600', '焦作': '410800', '牡丹江': '231000', '玉林': '450900',
|
||||
'玉树': '632700', '玉溪': '530400', '珠海': '440400', '琼中': '469030', '琼海': '469002', '甘南': '623000', '甘孜': '513300',
|
||||
'白城': '220800', '白山': '220600', '白沙': '469025', '白银': '620400', '百色': '451000', '益阳': '430900', '盐城': '320900',
|
||||
'盘锦': '211100', '眉山': '511400', '石嘴山': '640200', '石家庄': '130100', '石河子': '659001', '神农架': '429021',
|
||||
'福州': '350100', '秦皇岛': '130300', '红河': '532500', '绍兴': '330600', '绥化': '231200', '绵阳': '510700',
|
||||
'聊城': '371500', '肇庆': '441200', '自贡': '510300', '舟山': '330900', '芜湖': '340200', '苏州': '320500', '茂名': '440900',
|
||||
'荆州': '421000', '荆门': '420800', '菏泽': '371700', '莆田': '350300', '萍乡': '360300', '营口': '210800',
|
||||
'葫芦岛': '211400', '蚌埠': '340300', '衡水': '131100', '衡阳': '430400', '衢州': '330800', '襄阳': '420600', '西双版纳': '532800',
|
||||
'西宁': '630100', '西安': '610100', '许昌': '411000', '贵港': '450800', '贵阳': '520100', '贺州': '451100', '资阳': '512000',
|
||||
'赣州': '360700', '赤峰': '150400', '辽源': '220400', '辽阳': '211000', '达州': '511700', '运城': '140800', '连云港': '320700',
|
||||
'迪庆': '533400', '通化': '220500', '通辽': '150500', '遂宁': '510900', '遵义': '520300', '邢台': '130500', '那曲': '540600',
|
||||
'邯郸': '130400', '邵阳': '430500', '郑州': '410100', '郴州': '431000', '鄂尔多斯': '150600', '鄂州': '420700', '酒泉': '620900',
|
||||
'重庆': '500000', '金华': '330700', '金昌': '620300', '钦州': '450700', '铁岭': '211200', '铁门关': '659006', '铜仁': '520600',
|
||||
'铜川': '610200', '铜陵': '340700', '银川': '640100', '锡林郭勒': '152500', '锦州': '210700', '镇江': '321100', '长春': '220100',
|
||||
'长沙': '430100', '长治': '140400', '阜新': '210900', '阜阳': '341200', '防城港': '450600', '阳江': '441700', '阳泉': '140300',
|
||||
'阿克苏': '652900', '阿勒泰': '654300', '阿坝': '513200', '阿拉善': '152900', '阿拉尔': '659002', '阿里': '542500', '陇南': '621200',
|
||||
'陵水': '469028', '随州': '421300', '雅安': '511800', '青岛': '370200', '鞍山': '210300', '韶关': '440200',
|
||||
'驻马店': '411700', '鸡西': '230300', '鹤壁': '410600', '鹤岗': '230400', '鹰潭': '360600', '黄冈': '421100', '黄南': '632300',
|
||||
'黄山': '341000', '黄石': '420200', '黑河': '231100', '黔东南': '522600', '黔南': '522700', '黔西南': '522300', '齐齐哈尔': '230200',
|
||||
'龙岩': '350800', "马鞍山": "340500"}
|
||||
third_data_source_id_map = {'01': '诸葛找房','02': '安居客','03': '房天下','04': '链家','05': '城市居家',
|
||||
'06': '麦田','07': '贝壳','08': '淘房356','09': '中原','10': '兔博士',
|
||||
'11': 'Q房','12': '房探','13': '房多多','14': '我爱我家','15': '禧泰数据',
|
||||
'16': '吉屋网','17': '乐居','18': '21世纪','19': '房产超市','20': '58同城'}
|
||||
city_id_name_map = {k: v for v, k in city_name_id_map.items()}
|
||||
|
||||
|
||||
def get_city_name(city_id):
|
||||
"""
|
||||
通过城市ID获取城市名
|
||||
:param city_id:
|
||||
:return:
|
||||
"""
|
||||
return city_id_name_map[city_id] if city_id in city_id_name_map else None
|
||||
|
||||
|
||||
def get_city_id(city_name):
|
||||
"""
|
||||
通过城市名获取城市ID
|
||||
:param city_name:
|
||||
:return:
|
||||
"""
|
||||
return city_name_id_map[city_name] if city_name in city_name_id_map else None
|
||||
|
||||
|
||||
def is_city_id(city):
|
||||
"""
|
||||
判断是不是合法的cityid
|
||||
:param city:
|
||||
:return:
|
||||
"""
|
||||
return city in city_id_name_map
|
||||
|
||||
|
||||
def is_city_name(city):
|
||||
"""
|
||||
判断是不是合法的城市名
|
||||
:param city:
|
||||
:return:
|
||||
"""
|
||||
return city in city_name_id_map
|
||||
|
||||
|
||||
def get_city_data():
|
||||
"""
|
||||
返回所有的城市数据DataFrame[cityid,cityname]
|
||||
:return:
|
||||
"""
|
||||
data = list()
|
||||
for city_id in city_id_name_map.keys():
|
||||
data.append({"cityid": city_id, "cityname": city_id_name_map[city_id]})
|
||||
return pd.DataFrame(data)
|
||||
@@ -0,0 +1,25 @@
|
||||
from pymongo import MongoClient
|
||||
def getMongo(*args):
|
||||
if len(args) == 6:
|
||||
con = MongoClient(args[0], args[1]) # 获取链接\
|
||||
db_auth = con.fangjia_craw
|
||||
if args[2] =="vip_site":
|
||||
db_auth.authenticate(args[4], args[5],"fangjia_craw")
|
||||
else:
|
||||
db_auth.authenticate(args[4], args[5],"admin")
|
||||
db = con[args[2]] # 关联数据库
|
||||
collection = db[args[3]] # 关联数据表
|
||||
elif len(args) == 4:
|
||||
con = MongoClient(args[0], args[1]) # 获取链接
|
||||
db = con[args[2]] # 关联数据库
|
||||
collection = db[args[3]] # 关联数据表
|
||||
return collection
|
||||
#线上小区库
|
||||
seaweed_on = getMongo("192.168.0.43",28017,"fangjia","seaweed","developer","G00IDCDEV12385")
|
||||
sz_price_on = getMongo("192.168.0.43",28017,"fangjia","sz_price","developer","G00IDCDEV12385")
|
||||
district_stat_on = getMongo("192.168.0.43",28017,"fangjia_stat","district_stat","developer","G00IDCDEV12385")
|
||||
seaweed_map = getMongo("192.168.0.235",27018,"fangjia","seaweedmap")
|
||||
web_community_check = getMongo("192.168.0.235",27018,"fangjia","web_community_check")
|
||||
web_invalid_comm = getMongo("192.168.0.235",27018,"fangjia","web_invalid_comm")
|
||||
addr_nine_level = getMongo("192.168.0.206",27020,"etl","nine_level")
|
||||
gxd_price = getMongo("192.168.0.43",28017,"fangjia","confirm_price_data","developer","G00IDCDEV12385")
|
||||
@@ -0,0 +1,81 @@
|
||||
from mongo_client import seaweed_on,seaweed_map,addr_nine_level,district_stat_on,gxd_price
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import pytz
|
||||
pd.set_option ('display.max_colwidth',5)
|
||||
|
||||
def parse_date(time, fm='%Y-%m-%d'):
|
||||
pacific = pytz.timezone('Asia/Shanghai')
|
||||
return pacific.localize(datetime.datetime.strptime(time, fm))
|
||||
def get_last_weekend():
|
||||
sunday = datetime.datetime.utcnow()
|
||||
sunday = sunday.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
one_day = datetime.timedelta(days=1)
|
||||
while sunday.weekday() != 6:
|
||||
sunday -= one_day
|
||||
pacific = pytz.timezone('Asia/Shanghai')
|
||||
return pacific.localize(sunday)
|
||||
|
||||
confirm_date = parse_date("2024-05-20")
|
||||
def get_seaweed(city=None,status=0):
|
||||
seaweed_fields = {"city": 1, "region": 1, "name": 1,'estate_type2':1,'cat':1,"address":1,"alias":1,"lat":1,"lng":1,"addr_alias":1,"quality":1,"name_cat":1,"mapping_alias":1,"m_date":1,"is_new":1}
|
||||
# seaweed_fields = {"_id": 0}
|
||||
if city:
|
||||
if city in ['阿坝','玉树','甘孜','德宏','迪庆','海北','海西','黄南','海南','怒江','文山','西双版纳','凉山','果洛']:
|
||||
city = city+'州'
|
||||
elif city in ['锡林郭勒','兴安','阿拉善']:
|
||||
city = city+'盟'
|
||||
elif city=='克州':
|
||||
city = '克孜勒苏'
|
||||
elif city=='博州':
|
||||
city = '博尔塔拉'
|
||||
elif city=='巴州':
|
||||
city = '巴音郭楞'
|
||||
seaweed_condition = {"cat": "district", "status": status, 'city':city}
|
||||
else:
|
||||
seaweed_condition = {"cat": "district", "status": status}
|
||||
seaweed_data = seaweed_on.find(seaweed_condition, seaweed_fields)
|
||||
seaweed_df = pd.DataFrame(seaweed_data)
|
||||
seaweed_df = seaweed_df.rename(columns={'estate_type2': 'property_type'})
|
||||
return seaweed_df
|
||||
|
||||
def get_map(city=None):
|
||||
# seaweed_fields = {"city": 1, "region": 1, "name": 1,"address":1,"alias":1,"lat":1,"lng":1,"estate_type2":1,"addr_alias":1}
|
||||
seaweed_fields = {"_id": 0}
|
||||
if city:
|
||||
seaweed_condition = {'city':city}
|
||||
else:
|
||||
seaweed_condition = {}
|
||||
seaweed_map_data = seaweed_map.find(seaweed_condition, seaweed_fields)
|
||||
seaweed_map_df = pd.DataFrame(seaweed_map_data)
|
||||
return seaweed_map_df
|
||||
|
||||
def get_addr_nine_level(count):
|
||||
_fields = {"address":1,"province":1,"city": 1, "district": 1,"township": 1, "name": 1,"":1}
|
||||
_condition = {'address':{"$exists": True, "$ne": None},"remark":{"$exists": False}}
|
||||
addr_nine_data = addr_nine_level.find(_condition,_fields).limit(count)
|
||||
addr_nine_df = pd.DataFrame(addr_nine_data)
|
||||
return addr_nine_df
|
||||
|
||||
def get_district_stat_on(city=None):
|
||||
price_date = get_last_weekend()
|
||||
stat_fields = {"_id": 0, "city": 1, "region": 1, "district_name": 1, "avg_price": 1,"avg_price_2b":1}
|
||||
if city:
|
||||
stat_condition = {'city':city,"weekend": price_date}
|
||||
else:
|
||||
stat_condition = {"weekend": price_date}
|
||||
stat_data = district_stat_on.find(stat_condition, stat_fields)
|
||||
stat_df = pd.DataFrame(stat_data)
|
||||
stat_df = stat_df.rename(columns={'district_name': 'name'})
|
||||
return stat_df
|
||||
|
||||
def get_gxd_price():
|
||||
gxd_fields = {"_id": 0}
|
||||
gxd_condition = {'source':'GXD'}
|
||||
gxd_price_data = gxd_price.find(gxd_condition, gxd_fields)
|
||||
gxd_price_df = pd.DataFrame(gxd_price_data)
|
||||
return gxd_price_df
|
||||
if __name__ == '__main__':
|
||||
city = '厦门'
|
||||
df = get_gxd_price()
|
||||
print(df.head())
|
||||
@@ -0,0 +1,42 @@
|
||||
import os
|
||||
import pandas as pd
|
||||
from pandas import DataFrame
|
||||
from retrying import retry
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
# os.environ["NLS_LANG"] = "AMERICAN_AMERICA.AL32UTF8"
|
||||
|
||||
|
||||
class PgsqlDb:
|
||||
"""
|
||||
PgsqlDb Database Base Class
|
||||
"""
|
||||
|
||||
def __init__(self,db):
|
||||
if db == 'warehouse':
|
||||
url=f"postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/{db}"
|
||||
elif db == 'fangjia':
|
||||
url = f"postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5432/{db}"
|
||||
self.engine = create_engine(url, echo=False,client_encoding='utf-8')
|
||||
@retry(stop_max_attempt_number=3, wait_random_min=1000, wait_random_max=3000)
|
||||
def read(self, sql) -> DataFrame:
|
||||
"""
|
||||
读取数据
|
||||
:param sql:
|
||||
:return:
|
||||
"""
|
||||
data_df = pd.read_sql(sql, con=self.engine)
|
||||
return data_df
|
||||
|
||||
@retry(stop_max_attempt_number=3, wait_random_min=1000, wait_random_max=3000)
|
||||
def count(self, sql) -> int:
|
||||
"""
|
||||
执行统计类SQL
|
||||
:param sql: eg:select count(1) from t
|
||||
:return:
|
||||
"""
|
||||
return pd.read_sql_query(sql, con=self.engine).iat[0, 0]
|
||||
def sql_execute(self,sql):
|
||||
self.engine.execute(sql)
|
||||
if __name__ == '__main__':
|
||||
pgdb = PgsqlDb()
|
||||
@@ -0,0 +1,100 @@
|
||||
import pandas as pd
|
||||
import os
|
||||
from pgsql_client import PgsqlDb
|
||||
os.environ['NLS_LANG'] = 'SAMPLIFIED CHINESE_CHINA.UTF8'
|
||||
pgdb = PgsqlDb('warehouse')
|
||||
pgdb_new = PgsqlDb('fangjia')
|
||||
def get_community_base_price(cityid,yearmonth):
|
||||
"""
|
||||
查询提供建行小区数据
|
||||
:param cr:
|
||||
:return:
|
||||
"""
|
||||
sql = f"select gujia_admndiv_nm,gujia_bds_id,urbn_nm,comm_nm,gujiabds_clc_yrmo_val,gujia_comm_basic_prc,gujia_bds_clmtd from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='{cityid}' and gujiabds_clc_yrmo_val={yearmonth}"
|
||||
data_df = pgdb.read(sql)
|
||||
data_df = data_df.rename(columns={'gujia_admndiv_nm':'district','gujia_bds_id': '_id','urbn_nm':'city','comm_nm':'name','gujiabds_clc_yrmo_val':'yearmonth','gujia_comm_basic_prc':'price','gujia_bds_clmtd':'type'})
|
||||
return data_df
|
||||
|
||||
def get_community_base_price_new(cityid,yearmonth):
|
||||
"""
|
||||
查询提供建行小区数据
|
||||
:param cr:
|
||||
:return:
|
||||
"""
|
||||
sql = f"select gujia_bds_id,urbn_nm,comm_nm,gujiabds_clc_yrmo_val,gujia_comm_basic_prc,gujia_bds_clmtd from price_stat.avm_community_base_price where urbn_id='{cityid}' and gujiabds_clc_yrmo_val={yearmonth}"
|
||||
data_df = pgdb_new.read(sql)
|
||||
data_df = data_df.rename(columns={'gujia_bds_id': '_id','urbn_nm':'city_copy','comm_nm':'name_copy','gujiabds_clc_yrmo_val':'yearmonth','gujia_comm_basic_prc':'price_new','gujia_bds_clmtd':'type_new'})
|
||||
return data_df
|
||||
def get_district_polyline(cityid=None,cr=None):
|
||||
"""
|
||||
查询提供建行小区数据
|
||||
:param cr:
|
||||
:return:
|
||||
"""
|
||||
if cityid:
|
||||
sql = f"select * from bigdata_district_polyline_gd where cityid='{cityid}'"
|
||||
else:
|
||||
sql = f"select distinct city,distname from bigdata_district_polyline_gd"
|
||||
queryse = cr.execute(sql)
|
||||
columns = [i[0].lower() for i in queryse.description]
|
||||
rs = cr.fetchall()
|
||||
data_df = pd.DataFrame(rs,columns=columns)
|
||||
return data_df
|
||||
def get_community_ccb(cityid):
|
||||
sql = f"select gujia_comm_id,gujia_admndiv_nm,comm_nm from base.t0831_bigdata_community_gj_h where urbn_id='{cityid}' and gujia_bds_bds_st=0"
|
||||
data_df = pgdb.read(sql)
|
||||
data_df = data_df.rename(columns={'gujia_comm_id': '_id','gujia_admndiv_nm':'region','comm_nm':'name'})
|
||||
return data_df
|
||||
|
||||
def get_bak_community(cityid,cr):
|
||||
"""
|
||||
查询小区数据
|
||||
:param cr:
|
||||
:return:
|
||||
"""
|
||||
sql = f"select a.*,b.longitude,b.latitude,c.distname from bigdata_community_bak a left join (select cityid, objectid, LONGITUDE, LATITUDE from bigdata_position_bak where objecttype = '5' and positiontype = '1') b on a.cityid = b.cityid and a.commid = b.objectid left join bigdata_district_bak c on a.cityid=c.cityid and a.distid=c.distid where a.cityid = '{cityid}'"
|
||||
queryse = cr.execute(sql)
|
||||
columns = [i[0].lower() for i in queryse.description]
|
||||
rs = cr.fetchall()
|
||||
data_df = pd.DataFrame(rs,columns=columns)
|
||||
return data_df
|
||||
def avm_price_copy(cityid,yearmonth):
|
||||
sql = f"insert into t_app_p9pha_plda.avm_community_base_price_gj_h_copy select * from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='{cityid}' and gujiabds_clc_yrmo_val={yearmonth}"
|
||||
pgdb.sql_execute(sql)
|
||||
|
||||
def get_sample(cityid):
|
||||
sql = f"select urbn_id,gujia_comm_id,comm_nm,gujia_bds_unitprc,guiabconstructionarea from base.t0831_bigdata_forsale_gj_h where urbn_id='{cityid}' and gujia_hs_src_dtsrc in ('1','4') and anc_tm >= TO_DATE('2023-04-01','YYYY-MM-DD')"
|
||||
data_df = pgdb.read(sql)
|
||||
return data_df
|
||||
|
||||
def get_deal(cityid):
|
||||
sql = f"select urbn_id,gujia_comm_id,comm_nm,guiabconstructionarea,gujia_bds_trdtckt_prc from base.t0831_bigdata_deal_gj_h where urbn_id='{cityid}' and txn_dt >= TO_DATE('2024-03-01','YYYY-MM-DD') and txn_dt < TO_DATE('2024-04-01','YYYY-MM-DD')"
|
||||
data_df = pgdb.read(sql)
|
||||
data_df = data_df.rename(columns={'urbn_id': 'cityid','gujia_comm_id': '_id','comm_nm': 'name', 'guiabconstructionarea': 'area','gujia_bds_trdtckt_prc': 'deal_price'})
|
||||
return data_df
|
||||
|
||||
def get_deal_data(cityid):
|
||||
sql = f"select * from base.t0831_bigdata_deal_gj_h where urbn_id='{cityid}' and txn_dt >= TO_DATE('2024-03-01','YYYY-MM-DD') and txn_dt < TO_DATE('2024-04-01','YYYY-MM-DD')"
|
||||
data_df = pgdb.read(sql)
|
||||
return data_df
|
||||
|
||||
def get_region_price(cityid,yearmonth):
|
||||
sql = f"select urbn_id,gujia_admndiv_id,gxda_admndiv_nm,gujia_bds_urbn_avprc from t_app_p9pha_plda.avm_district_price_gj_h where urbn_id='{cityid}' and gujiabds_clc_yrmo_val={yearmonth}"
|
||||
data_df = pgdb.read(sql)
|
||||
data_df = data_df.rename(columns={'urbn_id': 'cityid','gujia_admndiv_id': 'district_id','gxda_admndiv_nm': 'district_name', 'gujia_bds_urbn_avprc': 'district_price'})
|
||||
return data_df
|
||||
def get_evaluation_third_price(cityid,yearmonth,type='第三方'):
|
||||
sql = f"select urbn_id,gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where urbn_id='{cityid}' and gujiabds_clc_yrmo_val={yearmonth} and gujia_data_tp='{type}' "
|
||||
data_df = pgdb.read(sql)
|
||||
data_df = data_df.rename(columns={'urbn_id': 'cityid','gujiabds_to_mkprc_mny': 'third_price','gujia_comm_id':'_id'})
|
||||
return data_df
|
||||
|
||||
def get_third_price(cityid,yearmonth):
|
||||
sql = f"select urbn_id,gujia_bds_id,comm_nm,gujia_dtsr_dsc,gujia_bds_comm_avprc from base.t0831_bigdata_third_party_price_gj_h where urbn_id='{cityid}' and gujiabds_avprc_dt_val={yearmonth} and gujia_dtsr_dsc in ('链家','安居客') "
|
||||
data_df = pgdb.read(sql)
|
||||
data_df = data_df.rename(columns={'urbn_id': 'cityid','gujia_bds_id': '_id','gujia_dtsr_dsc': 'source','gujia_bds_comm_avprc': 'price'})
|
||||
return data_df
|
||||
if __name__ == '__main__':
|
||||
city='常州'
|
||||
yearmonth=202302
|
||||
get_community_base_price(city=city,yearmonth=yearmonth)
|
||||
@@ -0,0 +1,49 @@
|
||||
import pandas as pd
|
||||
from dags.configs.citymap import city_name_id_map
|
||||
from pqsql_connect import get_community_base_price,get_evaluation_third_price,get_third_price
|
||||
from mongodb_connect import get_district_stat_on,get_seaweed,get_gxd_price
|
||||
yearmonth='202404'
|
||||
latest_year_month = '202403'
|
||||
df_concat = pd.DataFrame()
|
||||
for city in city_name_id_map:
|
||||
print(city)
|
||||
city_id = city_name_id_map[city]
|
||||
base_price = get_community_base_price(city_id, yearmonth=yearmonth)
|
||||
base_price.rename(columns={'price': '当月均价', 'type': '计算方式'}, inplace=True)
|
||||
base_price_last = get_community_base_price(city_id, yearmonth=latest_year_month)
|
||||
base_price_last.rename(columns={'price': '上月均价', 'type': '上月计算方式'}, inplace=True)
|
||||
get_evaluation_third_price_df = get_evaluation_third_price(city_id, yearmonth=yearmonth,type='第三方')
|
||||
get_evaluation_third_price_df.rename(columns={'third_price': '第三方对标价格'}, inplace=True)
|
||||
deal_price_df = get_evaluation_third_price(city_id, yearmonth=yearmonth,type='成交')
|
||||
deal_id = deal_price_df['_id'].tolist()
|
||||
get_evaluation_third_price_df = get_evaluation_third_price_df[~(get_evaluation_third_price_df['_id'].isin(deal_id))]
|
||||
third_price = get_third_price(cityid=city_id, yearmonth=latest_year_month)
|
||||
lianjia_price =third_price[third_price['source']=='链家'].drop_duplicates(subset=["_id"], keep="first")
|
||||
lianjia_price.rename(columns={'price': '链家价格'}, inplace=True)
|
||||
anjuke_price =third_price[third_price['source']=='安居客'].drop_duplicates(subset=["_id"], keep="first")
|
||||
anjuke_price.rename(columns={'price': '安居客价格'}, inplace=True)
|
||||
base_price_gj = get_district_stat_on(city)
|
||||
seaweed_df = get_seaweed(city)
|
||||
seaweed_df['_id'] = seaweed_df['_id'].astype('str')
|
||||
if base_price_gj.empty:
|
||||
continue
|
||||
base_price_gj = base_price_gj.merge(seaweed_df[['_id', 'region', 'name']], on=['region', 'name'], how='left')
|
||||
base_price_gj.rename(columns={'avg_price': '房价网2C价格', 'avg_price_2b': '房价网2B价格'}, inplace=True)
|
||||
base_price_gxd = get_gxd_price()
|
||||
base_price_gxd.rename(columns={'district_name': 'name', 'price': '国信达价格'}, inplace=True)
|
||||
data_df_temp = seaweed_df[['_id','city', 'region', 'name']].merge(base_price_gxd[['city', 'region', 'name', '国信达价格', 'c_date']], on=['city', 'region', 'name'])
|
||||
data_df_temp = data_df_temp.sort_values(by="c_date", ascending=False)
|
||||
data_df_temp = data_df_temp.drop_duplicates(subset=["city", "region", "name"], keep="first")
|
||||
if base_price.empty or base_price_last.empty or get_evaluation_third_price_df.empty:
|
||||
continue
|
||||
price_merge = base_price.merge(base_price_last[['_id', '上月均价', '上月计算方式']], on='_id', how='left')
|
||||
price_merge = price_merge.merge(get_evaluation_third_price_df[['_id','第三方对标价格']],on='_id')
|
||||
price_merge = price_merge.merge(lianjia_price[['_id','链家价格']],on='_id',how='left')
|
||||
price_merge = price_merge.merge(anjuke_price[['_id','安居客价格']],on='_id',how='left')
|
||||
price_merge = price_merge.merge(base_price_gj[['_id','房价网2C价格','房价网2B价格']], on='_id', how='left')
|
||||
price_merge = price_merge.merge(data_df_temp[['_id','国信达价格']], on='_id', how='left')
|
||||
price_merge.rename(columns={'city': '城市', 'district': '区域', '_id': '小区ID', 'name': '小区名','yearmonth':'月份'}, inplace=True)
|
||||
df_concat = pd.concat([df_concat,price_merge])
|
||||
df_concat.to_excel('小区第三方均价输出.xlsx', encoding='utf-8', index=False)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user