Uploading New Exposures to Existing Dataset and Updating the Risk Dataset#

Use this notebook to append new exposure data to an existing upload and update an existing risk dataset.

import datetime as dt
from pathlib import Path

from tqdm import tqdm

from bayesline.apiclient import BayeslineApiClient
from bayesline.api.equity import (
    ExposureSettings, 
    IndustrySettings, 
    RegionSettings,
    UniverseSettings,
)

bln = BayeslineApiClient.new_client(
    endpoint="https://[ENDPOINT]",
    api_key="[API-KEY]",
)

Updating New Exposure Files#

exposure_dir = Path("PATH/TO/EXPOSURES")

exposure_dataset_name = "My-Exposures"

Below gets the exposure uploader for the chosen dataset name My-Exposures. This dataset is assummed to be already created since we demonstrate a catch up upload. See the Uploaders Tutorial for a deep dive into the Uploaders API.

exposure_uploader = bln.equity.uploaders.get_data_type("exposures")
uploader = exposure_uploader.get_dataset(dataset=exposure_dataset_name)

Below we list the existing files in the provided folder and filter out all dates for which we already processed files in a previous run.

# list all csv files and group them by year
# expects file pattern "*_YYYY-MM-DD.csv"

files = list(exposure_dir.glob("*.csv"))
file_date_strs = [file.name.split("_")[-1].replace("-", "")[:8] for file in files]

available_dates = [
    dt.date(int(d[:4]), int(d[4:6]), int(d[6:8])) for d in file_date_strs
]

existing_dates = (
    uploader.get_data(columns=["date"], unique=True).collect().to_series().to_list()
)

new_dates = sorted(set(available_dates) - set(existing_dates))

print(f"Got {len(new_dates)} new dates")

Got 24 new dates

files_by_date = {
    dt.date(int(d[:4]), int(d[4:6]), int(d[6:8])): f for d, f in zip(file_date_strs, files)
}

As a next step we iterate over each csv file and stage it. Note that for large amounts of files it’s much more performant to upload zip files instead. See this recipe for details.

See the Uploaders Tutorial for more details on the staging and commit concepts.

for date in tqdm(new_dates):
    file = files_by_date[date]
    result = uploader.stage_file(file)
    assert result.success

uploader.commit(mode="append")

UploadCommitResult(version=2, committed_names=['exposures_2025-06-11', 'exposures_2025-06-01', 'exposures_2025-06-09', 'exposures_2025-06-02', 'exposures_2025-06-16', 'exposures_2025-06-10', 'exposures_2025-06-03', 'exposures_2025-06-15', 'exposures_2025-06-07', 'exposures_2025-06-12', 'exposures_2025-06-21', 'exposures_2025-06-14', 'exposures_2025-06-22', 'exposures_2025-06-17', 'exposures_2025-06-04', 'exposures_2025-06-20', 'exposures_2025-06-24', 'exposures_2025-06-05', 'exposures_2025-06-18', 'exposures_2025-06-13', 'exposures_2025-06-23', 'exposures_2025-06-06', 'exposures_2025-06-08', 'exposures_2025-06-19'])

uploader.get_data_detail_summary()

shape: (55, 7)

date	n_assets	min_exposure	max_exposure	mean_exposure	median_exposure	std_exposure
date	u32	f32	f32	f32	f32	f32
2025-05-01	35804	-4.0625	4.09375	0.241719	0.49707	1.013115
2025-05-02	35798	-4.0625	4.09375	0.241691	0.497559	1.013026
2025-05-03	35793	-4.0625	4.09375	0.2417	0.497559	1.012994
2025-05-04	35793	-4.0625	4.09375	0.241718	0.497559	1.013005
2025-05-05	35796	-4.0625	4.09375	0.241675	0.49707	1.012967
…	…	…	…	…	…	…
2025-06-20	35686	-4.042969	4.113281	0.243499	0.494873	1.011643
2025-06-21	35650	-4.042969	4.113281	0.243514	0.495117	1.011587
2025-06-22	35650	-4.042969	4.113281	0.243508	0.495117	1.011592
2025-06-23	35656	-4.042969	4.113281	0.243441	0.495361	1.011482
2025-06-24	35552	-4.042969	4.113281	0.243571	0.496338	1.011318

Updating the Risk Dataset#

To bring the newly uploaded exposures into the pre-existing dataset we need to update it. This pulls the most recent version for all referenced datasets and re-creates the risk dataset.

risk_dataset_name = "My-Risk-Dataset"

risk_datasets = bln.equity.riskdatasets
risk_dataset = risk_datasets.load(risk_dataset_name)

risk_dataset.update()

RiskDatasetUpdateResult()

risk_dataset.describe()

RiskDatasetProperties(calendar_settings_menu=CalendarSettingsMenu(exchanges=['ARCX', 'BVCA', 'BVMF', 'DIFX', 'DSMD', 'ETFP', 'FRAB', 'HSTC', 'JBUL', 'PFTS', 'ROCO', 'SHSC', 'SZSC', 'WBDM', 'XADS', 'XAMM', 'XAMS', 'XASE', 'XASX', 'XATH', 'XBAH', 'XBEL', 'XBEY', 'XBKF', 'XBKK', 'XBOG', 'XBOM', 'XBOS', 'XBRA', 'XBRU', 'XBRV', 'XBUD', 'XBUE', 'XCAI', 'XCAN', 'XCAS', 'XCSE', 'XCYS', 'XDUB', 'XEQY', 'XETB', 'XHEL', 'XHKG', 'XHNX', 'XICE', 'XIDX', 'XJAM', 'XJAS', 'XJSE', 'XKAR', 'XKLS', 'XKOS', 'XKRX', 'XKUW', 'XLIM', 'XLIS', 'XLIT', 'XLJU', 'XLON', 'XLUX', 'XMAD', 'XMAL', 'XMAU', 'XMEX', 'XMUS', 'XNAI', 'XNAM', 'XNAS', 'XNCM', 'XNSA', 'XNSE', 'XNYS', 'XNZE', 'XOSL', 'XPAE', 'XPAR', 'XPHS', 'XPRM', 'XPSX', 'XQUI', 'XRIS', 'XSAU', 'XSEC', 'XSES', 'XSGO', 'XSHE', 'XSHG', 'XSSC', 'XSTC', 'XSTO', 'XSWX', 'XTAE', 'XTAI', 'XTAL', 'XTKS', 'XTSE', 'XTSX', 'XTUN', 'XWAR', 'XZAG', 'XZIM']), universe_settings_menu=UniverseSettingsMenu(id_types=['bayesid'], exchanges=['ARCX', 'BVCA', 'BVMF', 'DIFX', 'DSMD', 'ETFP', 'FRAB', 'HSTC', 'JBUL', 'PFTS', 'ROCO', 'SHSC', 'SZSC', 'WBDM', 'XADS', 'XAMM', 'XAMS', 'XASE', 'XASX', 'XATH', 'XBAH', 'XBEL', 'XBEY', 'XBKF', 'XBKK', 'XBOG', 'XBOM', 'XBOS', 'XBRA', 'XBRU', 'XBRV', 'XBUD', 'XBUE', 'XCAI', 'XCAN', 'XCAS', 'XCSE', 'XCYS', 'XDUB', 'XEQY', 'XETB', 'XHEL', 'XHKG', 'XHNX', 'XICE', 'XIDX', 'XJAM', 'XJAS', 'XJSE', 'XKAR', 'XKLS', 'XKOS', 'XKRX', 'XKUW', 'XLIM', 'XLIS', 'XLIT', 'XLJU', 'XLON', 'XLUX', 'XMAD', 'XMAL', 'XMAU', 'XMEX', 'XMUS', 'XNAI', 'XNAM', 'XNAS', 'XNCM', 'XNSA', 'XNSE', 'XNYS', 'XNZE', 'XOSL', 'XPAE', 'XPAR', 'XPHS', 'XPRM', 'XPSX', 'XQUI', 'XRIS', 'XSAU', 'XSEC', 'XSES', 'XSGO', 'XSHE', 'XSHG', 'XSSC', 'XSTC', 'XSTO', 'XSWX', 'XTAE', 'XTAI', 'XTAL', 'XTKS', 'XTSE', 'XTSX', 'XTUN', 'XWAR', 'XZAG', 'XZIM'], industry={'industry': {'ALL': ['industry.Academic & Educational Services', 'industry.Basic Materials', 'industry.Consumer Cyclicals', 'industry.Consumer Non-Cyclicals', 'industry.Energy', 'industry.Financials', 'industry.Government Activity', 'industry.Healthcare', 'industry.Industrials', 'industry.Institutions, Associations & Organizations', 'industry.Real Estate', 'industry.Technology', 'industry.Utilities']}}, region={'region': {'ALL': ['region.Australia', 'region.Austria', 'region.Belgium', 'region.Bermuda', 'region.Brazil', 'region.British Virgin Islands', 'region.Bulgaria', 'region.Canada', 'region.Cayman Islands', 'region.China', 'region.Croatia', 'region.Cyprus', 'region.Czechia', 'region.Denmark', 'region.Estonia', 'region.Finland', 'region.France', 'region.Germany', 'region.Greece', 'region.Guernsey', 'region.Hong Kong', 'region.Hungary', 'region.Iceland', 'region.India', 'region.Indonesia', 'region.Ireland', 'region.Israel', 'region.Italy', 'region.Japan', 'region.Jersey', 'region.Kazakhstan', 'region.Latvia', 'region.Lithuania', 'region.Luxembourg', 'region.Macau', 'region.Malaysia', 'region.Mexico', 'region.Netherlands', 'region.New Zealand', 'region.Norway', 'region.Pakistan', 'region.Papua New Guinea', 'region.Philippines', 'region.Poland', 'region.Portugal', 'region.Romania', 'region.Serbia', 'region.Singapore', 'region.Slovakia', 'region.Slovenia', 'region.South Korea', 'region.Spain', 'region.Sweden', 'region.Switzerland', 'region.Taiwan', 'region.Thailand', 'region.Türkiye', 'region.Ukraine', 'region.United Arab Emirates', 'region.United Kingdom', 'region.United States', 'region.Vietnam']}}, industry_labels={'industry': {'ALL': 'ALL', 'industry.Academic & Educational Services': 'industry.Academic & Educational Services', 'industry.Basic Materials': 'industry.Basic Materials', 'industry.Consumer Cyclicals': 'industry.Consumer Cyclicals', 'industry.Consumer Non-Cyclicals': 'industry.Consumer Non-Cyclicals', 'industry.Energy': 'industry.Energy', 'industry.Financials': 'industry.Financials', 'industry.Government Activity': 'industry.Government Activity', 'industry.Healthcare': 'industry.Healthcare', 'industry.Industrials': 'industry.Industrials', 'industry.Institutions, Associations & Organizations': 'industry.Institutions, Associations & Organizations', 'industry.Real Estate': 'industry.Real Estate', 'industry.Technology': 'industry.Technology', 'industry.Utilities': 'industry.Utilities'}}, region_labels={'region': {'ALL': 'ALL', 'region.Australia': 'region.Australia', 'region.Austria': 'region.Austria', 'region.Belgium': 'region.Belgium', 'region.Bermuda': 'region.Bermuda', 'region.Brazil': 'region.Brazil', 'region.British Virgin Islands': 'region.British Virgin Islands', 'region.Bulgaria': 'region.Bulgaria', 'region.Canada': 'region.Canada', 'region.Cayman Islands': 'region.Cayman Islands', 'region.China': 'region.China', 'region.Croatia': 'region.Croatia', 'region.Cyprus': 'region.Cyprus', 'region.Czechia': 'region.Czechia', 'region.Denmark': 'region.Denmark', 'region.Estonia': 'region.Estonia', 'region.Finland': 'region.Finland', 'region.France': 'region.France', 'region.Germany': 'region.Germany', 'region.Greece': 'region.Greece', 'region.Guernsey': 'region.Guernsey', 'region.Hong Kong': 'region.Hong Kong', 'region.Hungary': 'region.Hungary', 'region.Iceland': 'region.Iceland', 'region.India': 'region.India', 'region.Indonesia': 'region.Indonesia', 'region.Ireland': 'region.Ireland', 'region.Israel': 'region.Israel', 'region.Italy': 'region.Italy', 'region.Japan': 'region.Japan', 'region.Jersey': 'region.Jersey', 'region.Kazakhstan': 'region.Kazakhstan', 'region.Latvia': 'region.Latvia', 'region.Lithuania': 'region.Lithuania', 'region.Luxembourg': 'region.Luxembourg', 'region.Macau': 'region.Macau', 'region.Malaysia': 'region.Malaysia', 'region.Mexico': 'region.Mexico', 'region.Netherlands': 'region.Netherlands', 'region.New Zealand': 'region.New Zealand', 'region.Norway': 'region.Norway', 'region.Pakistan': 'region.Pakistan', 'region.Papua New Guinea': 'region.Papua New Guinea', 'region.Philippines': 'region.Philippines', 'region.Poland': 'region.Poland', 'region.Portugal': 'region.Portugal', 'region.Romania': 'region.Romania', 'region.Serbia': 'region.Serbia', 'region.Singapore': 'region.Singapore', 'region.Slovakia': 'region.Slovakia', 'region.Slovenia': 'region.Slovenia', 'region.South Korea': 'region.South Korea', 'region.Spain': 'region.Spain', 'region.Sweden': 'region.Sweden', 'region.Switzerland': 'region.Switzerland', 'region.Taiwan': 'region.Taiwan', 'region.Thailand': 'region.Thailand', 'region.Türkiye': 'region.Türkiye', 'region.Ukraine': 'region.Ukraine', 'region.United Arab Emirates': 'region.United Arab Emirates', 'region.United Kingdom': 'region.United Kingdom', 'region.United States': 'region.United States', 'region.Vietnam': 'region.Vietnam'}}), exposure_settings_menu=ExposureSettingsMenu(market='market.Market', styles={'style.Dividend-all': ['style.Dividend'], 'style.Growth-all': ['style.Growth'], 'style.Leverage-all': ['style.Leverage'], 'style.Momentum-all': ['style.Momentum'], 'style.Size-all': ['style.Size'], 'style.Value-all': ['style.Value'], 'style.Volatility-all': ['style.Volatility']}, industries={'industry': {'ALL': ['industry.Academic & Educational Services', 'industry.Basic Materials', 'industry.Consumer Cyclicals', 'industry.Consumer Non-Cyclicals', 'industry.Energy', 'industry.Financials', 'industry.Government Activity', 'industry.Healthcare', 'industry.Industrials', 'industry.Institutions, Associations & Organizations', 'industry.Real Estate', 'industry.Technology', 'industry.Utilities']}}, regions={'region': {'ALL': ['region.Australia', 'region.Austria', 'region.Belgium', 'region.Bermuda', 'region.Brazil', 'region.British Virgin Islands', 'region.Bulgaria', 'region.Canada', 'region.Cayman Islands', 'region.China', 'region.Croatia', 'region.Cyprus', 'region.Czechia', 'region.Denmark', 'region.Estonia', 'region.Finland', 'region.France', 'region.Germany', 'region.Greece', 'region.Guernsey', 'region.Hong Kong', 'region.Hungary', 'region.Iceland', 'region.India', 'region.Indonesia', 'region.Ireland', 'region.Israel', 'region.Italy', 'region.Japan', 'region.Jersey', 'region.Kazakhstan', 'region.Latvia', 'region.Lithuania', 'region.Luxembourg', 'region.Macau', 'region.Malaysia', 'region.Mexico', 'region.Netherlands', 'region.New Zealand', 'region.Norway', 'region.Pakistan', 'region.Papua New Guinea', 'region.Philippines', 'region.Poland', 'region.Portugal', 'region.Romania', 'region.Serbia', 'region.Singapore', 'region.Slovakia', 'region.Slovenia', 'region.South Korea', 'region.Spain', 'region.Sweden', 'region.Switzerland', 'region.Taiwan', 'region.Thailand', 'region.Türkiye', 'region.Ukraine', 'region.United Arab Emirates', 'region.United Kingdom', 'region.United States', 'region.Vietnam']}}, other=set(), market_labels={'market.Market': 'market.Market'}, style_labels={'style.Dividend-all': 'style.Dividend', 'style.Growth-all': 'style.Growth', 'style.Leverage-all': 'style.Leverage', 'style.Momentum-all': 'style.Momentum', 'style.Size-all': 'style.Size', 'style.Value-all': 'style.Value', 'style.Volatility-all': 'style.Volatility', 'style.Dividend': 'style.Dividend', 'style.Growth': 'style.Growth', 'style.Leverage': 'style.Leverage', 'style.Momentum': 'style.Momentum', 'style.Size': 'style.Size', 'style.Value': 'style.Value', 'style.Volatility': 'style.Volatility'}, industry_labels={'industry': {'ALL': 'ALL', 'industry.Academic & Educational Services': 'industry.Academic & Educational Services', 'industry.Basic Materials': 'industry.Basic Materials', 'industry.Consumer Cyclicals': 'industry.Consumer Cyclicals', 'industry.Consumer Non-Cyclicals': 'industry.Consumer Non-Cyclicals', 'industry.Energy': 'industry.Energy', 'industry.Financials': 'industry.Financials', 'industry.Government Activity': 'industry.Government Activity', 'industry.Healthcare': 'industry.Healthcare', 'industry.Industrials': 'industry.Industrials', 'industry.Institutions, Associations & Organizations': 'industry.Institutions, Associations & Organizations', 'industry.Real Estate': 'industry.Real Estate', 'industry.Technology': 'industry.Technology', 'industry.Utilities': 'industry.Utilities'}}, region_labels={'region': {'ALL': 'ALL', 'region.Australia': 'region.Australia', 'region.Austria': 'region.Austria', 'region.Belgium': 'region.Belgium', 'region.Bermuda': 'region.Bermuda', 'region.Brazil': 'region.Brazil', 'region.British Virgin Islands': 'region.British Virgin Islands', 'region.Bulgaria': 'region.Bulgaria', 'region.Canada': 'region.Canada', 'region.Cayman Islands': 'region.Cayman Islands', 'region.China': 'region.China', 'region.Croatia': 'region.Croatia', 'region.Cyprus': 'region.Cyprus', 'region.Czechia': 'region.Czechia', 'region.Denmark': 'region.Denmark', 'region.Estonia': 'region.Estonia', 'region.Finland': 'region.Finland', 'region.France': 'region.France', 'region.Germany': 'region.Germany', 'region.Greece': 'region.Greece', 'region.Guernsey': 'region.Guernsey', 'region.Hong Kong': 'region.Hong Kong', 'region.Hungary': 'region.Hungary', 'region.Iceland': 'region.Iceland', 'region.India': 'region.India', 'region.Indonesia': 'region.Indonesia', 'region.Ireland': 'region.Ireland', 'region.Israel': 'region.Israel', 'region.Italy': 'region.Italy', 'region.Japan': 'region.Japan', 'region.Jersey': 'region.Jersey', 'region.Kazakhstan': 'region.Kazakhstan', 'region.Latvia': 'region.Latvia', 'region.Lithuania': 'region.Lithuania', 'region.Luxembourg': 'region.Luxembourg', 'region.Macau': 'region.Macau', 'region.Malaysia': 'region.Malaysia', 'region.Mexico': 'region.Mexico', 'region.Netherlands': 'region.Netherlands', 'region.New Zealand': 'region.New Zealand', 'region.Norway': 'region.Norway', 'region.Pakistan': 'region.Pakistan', 'region.Papua New Guinea': 'region.Papua New Guinea', 'region.Philippines': 'region.Philippines', 'region.Poland': 'region.Poland', 'region.Portugal': 'region.Portugal', 'region.Romania': 'region.Romania', 'region.Serbia': 'region.Serbia', 'region.Singapore': 'region.Singapore', 'region.Slovakia': 'region.Slovakia', 'region.Slovenia': 'region.Slovenia', 'region.South Korea': 'region.South Korea', 'region.Spain': 'region.Spain', 'region.Sweden': 'region.Sweden', 'region.Switzerland': 'region.Switzerland', 'region.Taiwan': 'region.Taiwan', 'region.Thailand': 'region.Thailand', 'region.Türkiye': 'region.Türkiye', 'region.Ukraine': 'region.Ukraine', 'region.United Arab Emirates': 'region.United Arab Emirates', 'region.United Kingdom': 'region.United Kingdom', 'region.United States': 'region.United States', 'region.Vietnam': 'region.Vietnam'}}, other_labels={}), modelconstruction_settings_menu=ModelConstructionSettingsMenu(weights=['SqrtCap', 'InvIdioVar'], known_factors={'risk_free_rate': 'The 3 Month US Treasury Bill Rate', 'international_return': 'The return on the international portfolio'}))

exposures_api = bln.equity.exposures.load(
    ExposureSettings(
        industries=None,
        regions=None,
    )
)

# note that the industry and region hierarchy names tie out with the factor groups we specified above

df = exposures_api.get(
    UniverseSettings(
        dataset=risk_dataset_name, 
        industry=IndustrySettings(hierarchy="industry", include="All"),
        region=RegionSettings(hierarchy="region", include="All")
    )
)

df.tail()

shape: (5, 10)

date	bayesid	market.market.Market	style.style.Dividend	style.style.Growth	style.style.Leverage	style.style.Momentum	style.style.Size	style.style.Value	style.style.Volatility
date	str	f32	f32	f32	f32	f32	f32	f32	f32
2025-06-24	"ZSPC"	1.0	-0.597168	0.366211	-0.072388	-0.144531	-1.766602	-1.714844	1.740234
2025-06-24	"ZTR"	1.0	1.834961	-0.287354	-0.330566	1.40918	-0.745605	1.161133	-1.368164
2025-06-24	"ZUMZ"	1.0	-1.166016	-2.351562	-0.849609	-1.360352	-0.789551	1.0625	0.803711
2025-06-24	"ZVIA"	1.0	-0.050262	-1.299805	-1.982422	1.37793	-1.433594	-0.694824	1.260742
2025-06-24	"ZVVT"	1.0	-0.501465	-0.906738	1.451172	0.598145	-1.37207	-1.271484	2.162109

Uploading New Exposures to Existing Dataset and Updating the Risk Dataset

Contents

Uploading New Exposures to Existing Dataset and Updating the Risk Dataset#

Updating New Exposure Files#

Updating the Risk Dataset#