diff --git a/EIVPackage/EIVGeneral/coverage_plotting.py b/EIVPackage/EIVGeneral/coverage_collect.py similarity index 51% rename from EIVPackage/EIVGeneral/coverage_plotting.py rename to EIVPackage/EIVGeneral/coverage_collect.py index ad944b557d6223f89add11e6dd2d9fcb25008f1a..6a3dbb7a0164f8804606c4eaa0262d1977995254 100644 --- a/EIVPackage/EIVGeneral/coverage_plotting.py +++ b/EIVPackage/EIVGeneral/coverage_collect.py @@ -1,30 +1,9 @@ """ -Get the numerical vs the theoretical coverage for different coverage factors -and plot them. This module contains -- The function `get_coverages`, that returns two numpy arrays containing - the numerical and theoretical coverage. -- The function `get_coverages_with_uncertainties`, that runs through several - networks and collects the results of `get_coverages`. -- The function `plot_coverages`, that plots the results of - `get_coverage_distribution`. +Collect coverages for various coverage factors, networks and dataloaders. """ -import importlib -import os -import argparse -import json - import numpy as np import torch import torch.backends.cudnn -from torch.utils.data import DataLoader -from tqdm import tqdm - -from EIVArchitectures import Networks -from EIVTrainingRoutines import train_and_store -from EIVGeneral.coverage_metrics import epistemic_coverage, normalized_std -from EIVData.repeated_sampling import repeated_sampling - -import matplotlib.pyplot as plt def get_coverages(not_averaged_predictions, y,\ q_range=np.linspace(0.1,0.9,num=30)): @@ -96,85 +75,10 @@ def get_coverage_distribution(net_iterator, dataloader_iterator, y_collection = torch.concat(y_collection, dim=0) numerical_coverage, theoretical_coverage = get_coverages( not_averaged_predictions=not_av_pred_collection, - y=y_collection) + y=y_collection, q_range=q_range) num_coverage_collection.append(numerical_coverage) th_coverage_collection.append(theoretical_coverage) if stack: num_coverage_collection = np.stack(num_coverage_collection, axis=-1) th_coverage_collection = np.stack(th_coverage_collection, axis=-1) return num_coverage_collection, th_coverage_collection - - - - - - - -####### - -# data = 'linear' -# # load hyperparameters from JSON file -# with open(os.path.join('/home/martin09/san/Projects/journal_eiv/Experiments/configurations',f'eiv_{data}.json'),'r') as conf_file: -# eiv_conf_dict = json.load(conf_file) -# with open(os.path.join('/home/martin09/san/Projects/journal_eiv/Experiments/configurations',f'noneiv_{data}.json'),'r') as conf_file: -# noneiv_conf_dict = json.load(conf_file) -# seed = 0 -# long_dataname = eiv_conf_dict["long_dataname"] -# short_dataname = eiv_conf_dict["short_dataname"] -# load_data = importlib.import_module(f'EIVData.{long_dataname}').load_data -# train, test = load_data() -# test_dataloader = DataLoader(test, batch_size=1000) -# device = torch.device('cuda:0') -# x,y = next(iter(test_dataloader)) -# if len(y.shape) <= 1: -# y = y.view((-1,1)) -# if len(x.shape) <= 1: -# x = x.view((-1,1)) -# x,y = x.to(device), y.to(device) -# input_dim = x.shape[1] -# output_dim = y.shape[1] -# init_std_y = noneiv_conf_dict["init_std_y_list"][0] -# unscaled_reg = noneiv_conf_dict["unscaled_reg"] -# p = noneiv_conf_dict["p"] -# hidden_layers = noneiv_conf_dict["hidden_layers"] -# saved_file = os.path.join('saved_networks', -# f'noneiv_{short_dataname}'\ -# f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\ -# f'_p_{p:.2f}_seed_{seed}.pkl') -# noneiv_net = Networks.FNNBer(p=p, init_std_y=init_std_y, -# h=[input_dim, *hidden_layers, output_dim]).to(device) -# train_and_store.open_stored_training(saved_file=saved_file, -# net=noneiv_net, device=device) -# # EiV -# init_std_y = eiv_conf_dict["init_std_y_list"][0] -# unscaled_reg = eiv_conf_dict["unscaled_reg"] -# p = eiv_conf_dict["p"] -# hidden_layers = eiv_conf_dict["hidden_layers"] -# fixed_std_x = eiv_conf_dict["fixed_std_x"] -# saved_file = os.path.join('saved_networks', -# f'eiv_{short_dataname}'\ -# f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\ -# f'_p_{p:.2f}_fixed_std_x_{fixed_std_x:.3f}'\ -# f'_seed_{seed}.pkl') -# eiv_net = Networks.FNNEIV(p=p, init_std_y=init_std_y, -# h=[input_dim, *hidden_layers, output_dim], -# fixed_std_x=fixed_std_x).to(device) -# def eiv_net_iterator(seed_range=range(0,10)): - -# train_and_store.open_stored_training(saved_file=saved_file, -# net=eiv_net, device=device) - -# noneiv_not_averaged_predictions = noneiv_net.predict(x,\ -# number_of_draws=100, -# take_average_of_prediction=False) -# noneiv_num_cov, noneiv_th_cov = get_coverages(noneiv_not_averaged_predictions, y,\ -# q_range=np.linspace(0.01,0.99,num=30)) -# eiv_not_averaged_predictions = eiv_net.predict(x,\ -# number_of_draws=[100,5], -# take_average_of_prediction=False) -# eiv_num_cov, eiv_th_cov = get_coverages(eiv_not_averaged_predictions, y,\ -# q_range=np.linspace(0.01,0.99,num=30)) -# lin_x = np.linspace(np.min(noneiv_th_cov), np.max(noneiv_th_cov)) -# plt.plot(lin_x, lin_x) -# plt.plot(noneiv_th_cov, noneiv_num_cov) -# plt.plot(eiv_th_cov, eiv_num_cov) diff --git a/Experiments/plot_coverage.py b/Experiments/plot_coverage.py index 81b3f146815a07d66aebb93f0ccbb0721bdc500f..cac67c28a61983e1149652971a749856b422b8b2 100644 --- a/Experiments/plot_coverage.py +++ b/Experiments/plot_coverage.py @@ -4,29 +4,22 @@ Results will be stored in the results folder """ import importlib import os -import argparse import json import numpy as np import torch import torch.backends.cudnn from torch.utils.data import DataLoader -from tqdm import tqdm import matplotlib.pyplot as plt from EIVArchitectures import Networks from EIVTrainingRoutines import train_and_store -from EIVGeneral.coverage_plotting import get_coverage_distribution +from EIVGeneral.coverage_collect import get_coverage_distribution from EIVGeneral.manipulate_datasets import VerticalCut # read in data via --data option -parser = argparse.ArgumentParser() -parser.add_argument("--data", help="Loads data", default='naval') -parser.add_argument("--no-autoindent", help="", - action="store_true") # to avoid conflics in IPython -args = parser.parse_args() -data = args.data +data = 'linear' # load hyperparameters from JSON file with open(os.path.join('configurations',f'eiv_{data}.json'),'r') as conf_file: @@ -114,7 +107,7 @@ def net_iterator(eiv=True, seed_list=seed_list): # dataloaders def dataloader_iterator(seed_list=seed_list, use_ground_truth=False, - batch_size = 1000): + batch_size = 100): for seed in seed_list: if not use_ground_truth: train_data, test_data = load_data(seed=seed)