Files
airflow/dags/jh/price/mongodb_connect.py
T
2024-07-22 17:13:48 +08:00

81 lines
3.3 KiB
Python

from mongo_client import seaweed_on,seaweed_map,addr_nine_level,district_stat_on,gxd_price
import pandas as pd
import datetime
import pytz
pd.set_option ('display.max_colwidth',5)
def parse_date(time, fm='%Y-%m-%d'):
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(datetime.datetime.strptime(time, fm))
def get_last_weekend():
sunday = datetime.datetime.utcnow()
sunday = sunday.replace(hour=0, minute=0, second=0, microsecond=0)
one_day = datetime.timedelta(days=1)
while sunday.weekday() != 6:
sunday -= one_day
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(sunday)
confirm_date = parse_date("2024-05-20")
def get_seaweed(city=None,status=0):
seaweed_fields = {"city": 1, "region": 1, "name": 1,'estate_type2':1,'cat':1,"address":1,"alias":1,"lat":1,"lng":1,"addr_alias":1,"quality":1,"name_cat":1,"mapping_alias":1,"m_date":1,"is_new":1}
# seaweed_fields = {"_id": 0}
if city:
if city in ['阿坝','玉树','甘孜','德宏','迪庆','海北','海西','黄南','海南','怒江','文山','西双版纳','凉山','果洛']:
city = city+'州'
elif city in ['锡林郭勒','兴安','阿拉善']:
city = city+'盟'
elif city=='克州':
city = '克孜勒苏'
elif city=='博州':
city = '博尔塔拉'
elif city=='巴州':
city = '巴音郭楞'
seaweed_condition = {"cat": "district", "status": status, 'city':city}
else:
seaweed_condition = {"cat": "district", "status": status}
seaweed_data = seaweed_on.find(seaweed_condition, seaweed_fields)
seaweed_df = pd.DataFrame(seaweed_data)
seaweed_df = seaweed_df.rename(columns={'estate_type2': 'property_type'})
return seaweed_df
def get_map(city=None):
# seaweed_fields = {"city": 1, "region": 1, "name": 1,"address":1,"alias":1,"lat":1,"lng":1,"estate_type2":1,"addr_alias":1}
seaweed_fields = {"_id": 0}
if city:
seaweed_condition = {'city':city}
else:
seaweed_condition = {}
seaweed_map_data = seaweed_map.find(seaweed_condition, seaweed_fields)
seaweed_map_df = pd.DataFrame(seaweed_map_data)
return seaweed_map_df
def get_addr_nine_level(count):
_fields = {"address":1,"province":1,"city": 1, "district": 1,"township": 1, "name": 1,"":1}
_condition = {'address':{"$exists": True, "$ne": None},"remark":{"$exists": False}}
addr_nine_data = addr_nine_level.find(_condition,_fields).limit(count)
addr_nine_df = pd.DataFrame(addr_nine_data)
return addr_nine_df
def get_district_stat_on(city=None):
price_date = get_last_weekend()
stat_fields = {"_id": 0, "city": 1, "region": 1, "district_name": 1, "avg_price": 1,"avg_price_2b":1}
if city:
stat_condition = {'city':city,"weekend": price_date}
else:
stat_condition = {"weekend": price_date}
stat_data = district_stat_on.find(stat_condition, stat_fields)
stat_df = pd.DataFrame(stat_data)
stat_df = stat_df.rename(columns={'district_name': 'name'})
return stat_df
def get_gxd_price():
gxd_fields = {"_id": 0}
gxd_condition = {'source':'GXD'}
gxd_price_data = gxd_price.find(gxd_condition, gxd_fields)
gxd_price_df = pd.DataFrame(gxd_price_data)
return gxd_price_df
if __name__ == '__main__':
city = '厦门'
df = get_gxd_price()
print(df.head())