diff --git a/EIVPackage/EIVGeneral/coverage_collect.py b/EIVPackage/EIVGeneral/coverage_collect.py new file mode 100644 index 0000000000000000000000000000000000000000..642315ebb0935322c83e5b6b8686b44011a38434 --- /dev/null +++ b/EIVPackage/EIVGeneral/coverage_collect.py @@ -0,0 +1,96 @@ +""" +Collect coverages for various coverage factors, networks and dataloaders. +""" +import numpy as np +import torch +import torch.backends.cudnn +from EIVGeneral.coverage_metrics import epistemic_coverage + +def get_coverages(not_averaged_predictions, y,\ + q_range=np.linspace(0.1,0.9,num=30), + noisy_y=True): + """ + Compute the numerical and theoretical coverage for various coverage + factors, computed from the `q` in `q_range`, using + `EIVGeneral.epistemic_coverage` + :param not_averaged_predictions: A tuple of tensors as in the output of + `FNNEIV.predict` with `take_average_of_prediction` set to `False`, i.e.: + the predictions of the neural net not averaged over the first dimension + (the repetition dimension in `FNNEIV.predict`) and + the aleatoric uncertainty with a batch dimension and a feature dimension. + :param y: A `torch.tensor` of the same shape then the second components + of `not_averaged_predictions`. If the feature dimension is missing, it is added. + :param q_range: An iterator through floats between 0 and 1. + :param noisy_y: Boolean. If True (the default), `y` is treated as noisy and + the total uncertainty is considered. If False, `y` is treated as the + unnoisy ground truth. + :returns: numerical_coverage, theoretical_coverage (numpy arrays) + """ + numerical_coverage_collection = [] + theoretical_coverage_collection = [] + for q in q_range: + numerical_coverage, theoretical_coverage =\ + epistemic_coverage(\ + not_averaged_predictions=not_averaged_predictions, + y=y, q=q, noisy_y=noisy_y) + numerical_coverage_collection.append(float(numerical_coverage)) + theoretical_coverage_collection.append(float(theoretical_coverage)) + return np.array(numerical_coverage_collection),\ + np.array(theoretical_coverage_collection) + + +def get_coverage_distribution(net_iterator, dataloader_iterator, + device, number_of_draws, q_range=np.linspace(0.1,0.9,num=30), + number_of_test_samples=10, stack=True, + noisy_y=True): + """ + Returns the numerical and theoretical coverage for all nets in + `net_iterator` with dataloader from `dataloader_iterator` + :param net_iterator: Iterator yielding `torch.nn.Module` + :param dataloader_iterator: Iterator yielding `torch.utils.data.DataLoader`. + Should return the same number of elements than `net_iterator`. + :param device: The `torch.device` to be used. + :param number_of_draws: Integer, to be used as keyword for all networks + returned by `net_iterator`. Defaults to 10. + :param q_range: Range of floats between 0 and 1, as in + `EIVGeneral.coverage_plot.get_coverages`. + :param number_of_test_samples: An integer, number of samples from the + dataloaders in dataloader_iterator to be used. + :param stack: Boolean. If True (default) the results will be stacked along + the last dimension. If False a list will be returned. + :param noisy_y: Boolean. If True (the default), `y` is treated as noisy and + the total uncertainty is considered. If False, `y` is treated as the + unnoisy ground truth. + :returns: num_coverage_collection, th_coverage_collection + """ + num_coverage_collection, th_coverage_collection = [],[] + for net, dataloader in\ + zip(net_iterator, dataloader_iterator): + not_av_pred_collection_out, not_av_pred_collection_sigma,\ + y_collection = [], [], [] + for i, (x,y) in enumerate(dataloader): + if i>= number_of_test_samples: + break + x, y = x.to(device), y.to(device) + not_averaged_predictions = net.predict(x, + take_average_of_prediction=False, + number_of_draws=number_of_draws) + not_av_pred_collection_out.append( + not_averaged_predictions[0]) + not_av_pred_collection_sigma.append( + not_averaged_predictions[1]) + y_collection.append(y) + not_av_pred_collection = [ + torch.concat(not_av_pred_collection_out, dim=0), + torch.concat(not_av_pred_collection_sigma, dim=0)] + y_collection = torch.concat(y_collection, dim=0) + numerical_coverage, theoretical_coverage = get_coverages( + not_averaged_predictions=not_av_pred_collection, + y=y_collection, q_range=q_range, + noisy_y=noisy_y) + num_coverage_collection.append(numerical_coverage) + th_coverage_collection.append(theoretical_coverage) + if stack: + num_coverage_collection = np.stack(num_coverage_collection, axis=-1) + th_coverage_collection = np.stack(th_coverage_collection, axis=-1) + return num_coverage_collection, th_coverage_collection diff --git a/EIVPackage/EIVGeneral/manipulate_datasets.py b/EIVPackage/EIVGeneral/manipulate_datasets.py new file mode 100644 index 0000000000000000000000000000000000000000..f04280e3a84f374cef1705a3e53a6607a6d0e394 --- /dev/null +++ b/EIVPackage/EIVGeneral/manipulate_datasets.py @@ -0,0 +1,21 @@ +from operator import itemgetter +from torch.utils.data import Dataset + + +class VerticalCut(Dataset): + """ + Only include certain elements of the dataset `dataset`, namely those with + index in `components_to_pick`. + :param dataset: A torch.utils.data.Dataset + :param components_to_pick: List, defaults to [0,]. + """ + def __init__(self, dataset, components_to_pick=[0,]): + super(VerticalCut, self).__init__() + self.dataset = dataset + self.components_to_pick = components_to_pick + + def __getitem__(self, i): + return itemgetter(*self.components_to_pick)(self.dataset.__getitem__(i)) + + def __len__(self): + return len(self.dataset) diff --git a/Experiments/create_tabular.py b/Experiments/create_tabular.py index 728b6e5228a3e1849210ff5a4552d4a35873de65..8b8373de4c5735c380cde7b9079b39e224a0af53 100644 --- a/Experiments/create_tabular.py +++ b/Experiments/create_tabular.py @@ -1,9 +1,8 @@ import os import glob -import argparse import json -metrics_to_display = ['rmse','logdens','bias','true_coverage_numerical','avg_bias'] +metrics_to_display = ['rmse','coverage_theory','coverage_numerical','true_coverage_numerical','avg_bias'] show_incomplete = True list_of_result_files = glob.glob(os.path.join('results','*.json')) diff --git a/Experiments/evaluate_metrics.py b/Experiments/evaluate_metrics.py index 22d4e66495a8e87cbc6230c825d728629cb53ae2..bf052e872d2e42d830e827526e52649da4d67be4 100644 --- a/Experiments/evaluate_metrics.py +++ b/Experiments/evaluate_metrics.py @@ -20,7 +20,7 @@ from EIVData.repeated_sampling import repeated_sampling # read in data via --data option parser = argparse.ArgumentParser() -parser.add_argument("--data", help="Loads data", default='replin') +parser.add_argument("--data", help="Loads data", default='linear') parser.add_argument("--no-autoindent", help="", action="store_true") # to avoid conflics in IPython args = parser.parse_args() @@ -285,7 +285,8 @@ def collect_full_seed_range_metrics(load_data, hidden_layers = noneiv_conf_dict["hidden_layers"] saved_file = os.path.join('saved_networks', f'noneiv_{short_dataname}'\ - f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\ + f'_init_std_y_{init_std_y:.3f}'\ + f'_ureg_{unscaled_reg:.1f}'\ f'_p_{p:.2f}_seed_{seed}.pkl') net = Networks.FNNBer(p=p, init_std_y=init_std_y, h=[input_dim, *hidden_layers, output_dim]).to(device) @@ -455,13 +456,7 @@ seed_list = range(noneiv_conf_dict["seed_range"][0], - - - - - - -max_batch_number = 2 +number_of_test_samples = 2 for seed in tqdm(seed_list): try: train_data, test_data, true_train_data, true_test_data \ @@ -478,7 +473,7 @@ for seed in tqdm(seed_list): batch_size=int(np.min((len(true_test_data), 800))), shuffle=True) for i in tqdm(range(num_test_epochs)): for j, x_y_pairs in enumerate(test_dataloader): - if j > max_batch_number: + if j > number_of_test_samples: break # fill in ground truth with None, if not existent if true_test_data is None: diff --git a/Experiments/plot_coverage.py b/Experiments/plot_coverage.py new file mode 100644 index 0000000000000000000000000000000000000000..b7a511645c34f2dd62f693e037db36ec05111696 --- /dev/null +++ b/Experiments/plot_coverage.py @@ -0,0 +1,442 @@ +""" +Compute the coverage for various coverage factors and compare them with +with the corresponding q or theoretical coverage. Results will be stored +in various plots in the results/figures folder. +""" +import importlib +import os +import json + +import numpy as np +import torch +import torch.backends.cudnn +from torch.utils.data import DataLoader +from matplotlib.pyplot import cm +import matplotlib.pyplot as plt + +from EIVArchitectures import Networks +from EIVTrainingRoutines import train_and_store +from EIVGeneral.coverage_collect import get_coverage_distribution +from EIVGeneral.manipulate_datasets import VerticalCut + +# coverages to consider +q_range = np.linspace(0.1, 0.95) + +# load hyperparameters from JSON file +def compute_coverages(data, eiv, number_of_draws, + use_ground_truth=False): + """ + Create network and dataloader iterators for `data` (short dataname) and + feed them into `get_coverage_distribution`. + :data: String, short dataname + :eiv: Boolean. If True an EiV model is used, else an non-EiV model. + :number_of_draws: Number of draws to use for prediction. Take an int for + non-EiV models and a two-element list for EiV models. + :use_ground_truth: If True, unnoisy `y` are considered when computing the + coverage. If there is no ground truth None,None is returned + :returns: numerical_coverage, theoretical_coverage + """ + # load configuration file + if eiv: + with open(os.path.join('configurations',f'eiv_{data}.json'),'r') as\ + conf_file: + conf_dict = json.load(conf_file) + else: + with open(os.path.join('configurations',f'noneiv_{data}.json'),'r') as\ + conf_file: + conf_dict = json.load(conf_file) + + long_dataname = conf_dict["long_dataname"] + short_dataname = conf_dict["short_dataname"] + + + load_data = importlib.import_module(f'EIVData.{long_dataname}').load_data + + # switch to gpu, if possible + try: + gpu_number = conf_dict["gpu_number"] + device = torch.device(f'cuda:{gpu_number}') + try: + torch.tensor([0.0]).to(device) + except RuntimeError: + if torch.cuda.is_available(): + print('Switched to GPU 0') + device = torch.device('cuda:0') + else: + print('No cuda available, using CPU') + device = torch.device('cpu') + except KeyError: + device = torch.device('cpu') + + + if use_ground_truth: + # test whether there is a ground truth + try: + train_data, _, _,_ \ + = load_data(seed=0, return_ground_truth=True) + except TypeError: + # if not, end function + return None,None + else: + train_data, _ = load_data() + + print(f"Computing {'EiV' if eiv else 'non-EiV'} coverage for {long_dataname}") + + # train_data only used for finding dimensions + input_dim = train_data[0][0].numel() + output_dim = train_data[0][1].numel() + + ## Create iterators for get_coverage_distribution + seed_list = range(conf_dict["seed_range"][0], + conf_dict["seed_range"][1]) + + # iterator for networks + def net_iterator(eiv=eiv, seed_list=seed_list): + """ + Yields EiV models (if `eiv`) or + non-EiV models (if not `eiv`) for the seeds in + `seed_list` and `data`. + """ + if eiv: + # load parameters + init_std_y = conf_dict["init_std_y_list"][0] + unscaled_reg = conf_dict["unscaled_reg"] + p = conf_dict["p"] + hidden_layers = conf_dict["hidden_layers"] + fixed_std_x = conf_dict["fixed_std_x"] + net = Networks.FNNEIV(p=p, init_std_y=init_std_y, + h=[input_dim, *hidden_layers, output_dim], + fixed_std_x=fixed_std_x).to(device) + for seed in seed_list: + # load network paramaters + saved_file = os.path.join('saved_networks', + f'eiv_{short_dataname}'\ + f'_init_std_y_{init_std_y:.3f}'\ + f'_ureg_{unscaled_reg:.1f}'\ + f'_p_{p:.2f}_fixed_std_x_{fixed_std_x:.3f}'\ + f'_seed_{seed}.pkl') + train_and_store.open_stored_training(saved_file=saved_file, + net=net, device=device) + yield net + else: + # load parameters + init_std_y = conf_dict["init_std_y_list"][0] + unscaled_reg = conf_dict["unscaled_reg"] + p = conf_dict["p"] + hidden_layers = conf_dict["hidden_layers"] + net = Networks.FNNBer(p=p, init_std_y=init_std_y, + h=[input_dim, *hidden_layers, output_dim]).to(device) + for seed in seed_list: + saved_file = os.path.join('saved_networks', + f'noneiv_{short_dataname}'\ + f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\ + f'_p_{p:.2f}_seed_{seed}.pkl') + # load network paramaters + train_and_store.open_stored_training(saved_file=saved_file, + net=net, device=device) + yield net + + # iterator for dataloaders + def dataloader_iterator(seed_list=seed_list,use_ground_truth=use_ground_truth, + batch_size = 100): + """ + Yields dataloaders for `data`, according to the seeds in `seed_list`. + If `use_ground_truth` the data is cut to contain noisy x and unnoisy y. + """ + for seed in seed_list: + if not use_ground_truth: + _, test_data = load_data(seed=seed) + test_dataloader = DataLoader(test_data, + batch_size=batch_size, + shuffle=True) + yield test_dataloader + else: + _, _, _, true_test =\ + load_data(seed=seed, return_ground_truth=True) + # take noisy x but unnoisy y + cut_true_test = VerticalCut(true_test, + components_to_pick=[2,1]) + test_dataloader = DataLoader(cut_true_test, + batch_size=batch_size, + shuffle=True) + yield test_dataloader + + + # Compute coverages + numerical_coverage, theoretical_coverage = get_coverage_distribution( + net_iterator=net_iterator(eiv=eiv), + dataloader_iterator=dataloader_iterator(), + device=device, + number_of_draws=number_of_draws, + q_range=q_range, + noisy_y = not use_ground_truth) + return numerical_coverage, theoretical_coverage + + + +#### + +def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color, + against_theoretical = False, label = '', mean_error=True): + """ + Plot numerical coverages against q (used coverage value), if + against_theoretical is False, or the theoretical coverage, if + `against_theoretical` is True. + :param eiv_coverages: The output of `compute_coverages` with `eiv=True` + :param noneiv_coverages: The output of `compute_coverages` with `eiv=False` + :param color: String, denoting the color. + :param against_theoretical: Boolean, see above. + :param label: String. Will be included as label in the plot. + :param mean_error: Boolean. If True the standard deviation is divided + by the square root of the number of elements, to display the error + of the mean (and not the dispersion). + """ + eiv_numerical_coverage, eiv_theoretical_coverage = eiv_coverages + noneiv_numerical_coverage, noneiv_theoretical_coverage = noneiv_coverages + assert (len(eiv_numerical_coverage.shape)) == 2 + assert (len(noneiv_numerical_coverage.shape)) == 2 + # EiV + # take mean/std over seed dimension + mean_eiv_numerical_coverage = np.mean(eiv_numerical_coverage, axis=-1) + std_eiv_numerical_coverage = np.std(eiv_numerical_coverage, axis=-1) + if mean_error: + std_eiv_numerical_coverage /= np.sqrt(eiv_numerical_coverage.shape[1]) + if against_theoretical: + # show theoretical coverage on x-axis + x_values = np.mean(eiv_theoretical_coverage, axis=-1) + else: + # show q-range on x-axis + x_values = np.array(q_range) + # plot mean + plt.plot(x_values, mean_eiv_numerical_coverage, + color=color, linestyle='solid', label=label) + # plot std + plt.fill_between(x_values, + mean_eiv_numerical_coverage - std_eiv_numerical_coverage, + mean_eiv_numerical_coverage + std_eiv_numerical_coverage, + color=color, alpha=0.5) + # non-EiV + # take mean/std over seed dimension + mean_noneiv_numerical_coverage = np.mean(noneiv_numerical_coverage, axis=-1) + std_noneiv_numerical_coverage = np.std(noneiv_numerical_coverage, axis=-1) + if mean_error: + std_noneiv_numerical_coverage /= \ + np.sqrt(noneiv_numerical_coverage.shape[1]) + if against_theoretical: + # show theoretical coverage on x-axis + x_values = np.mean(noneiv_theoretical_coverage, axis=-1) + else: + # show q-range on x-axis + x_values = np.array(q_range) + # plot mean + plt.plot(x_values, mean_noneiv_numerical_coverage, + color=color, linestyle='dashed') + # plot std + plt.fill_between(x_values, + mean_noneiv_numerical_coverage - std_noneiv_numerical_coverage, + mean_noneiv_numerical_coverage + std_noneiv_numerical_coverage, + color=color, alpha=0.3) + +def coverage_residual_plot(eiv_coverages, noneiv_coverages, color, + absolute_values=True, against_theoretical=False, label='', + mean_error=True): + """ + Plot deviation of numerical coverages to q (used coverage value), if + against_theoretical is False, or to the theoretical coverage, if + `against_theoretical` is True. On the x axis q is plotted (in contrast to + coverage_diagonal_plot). + :param eiv_coverages: The output of `compute_coverages` with `eiv=True` + :param noneiv_coverages: The output of `compute_coverages` with `eiv=False` + :param color: String, denoting the color. + :param absolute_values:Boolean. If True (default) the absolute value of the + differences will be plotted. + :param against_theoretical: Boolean, see above. + :param label: String. Will be included as label in the plot. + :param mean_error: Boolean. If True the standard deviation is divided + by the square root of the number of elements, to display the error + of the mean (and not the dispersion). + """ + if not against_theoretical: + eiv_coverages = eiv_coverages[0] + noneiv_coverages = noneiv_coverages[0] + assert len(eiv_coverages.shape) == 2 and\ + eiv_coverages.shape[0] == len(q_range) + assert len(eiv_coverages.shape) == 2 and\ + eiv_coverages.shape[0] == len(q_range) + assert len(noneiv_coverages.shape) == 2 + eiv_cov_res = eiv_coverages - q_range.reshape((-1,1)) + noneiv_cov_res = noneiv_coverages - q_range.reshape((-1,1)) + else: + eiv_cov_res = eiv_coverages[0] - eiv_coverages[1] + noneiv_cov_res = noneiv_coverages[0] - noneiv_coverages[1] + + if absolute_values: + eiv_cov_res = np.abs(eiv_cov_res) + noneiv_cov_res = np.abs(noneiv_cov_res) + mean_eiv_cov_res = np.mean(eiv_cov_res, axis=-1) + std_eiv_cov_res = np.std(eiv_cov_res, axis=-1) + if mean_error: + std_eiv_cov_res /= np.sqrt(eiv_cov_res.shape[1]) + mean_noneiv_cov_res = np.mean(noneiv_cov_res, axis=-1) + std_noneiv_cov_res = np.std(noneiv_cov_res, axis=-1) + if mean_error: + std_noneiv_cov_res /= np.sqrt(noneiv_cov_res.shape[1]) + plt.plot(q_range, mean_eiv_cov_res, + color=color, linestyle='solid', label=label) + plt.fill_between(q_range, + mean_eiv_cov_res - std_eiv_cov_res, + mean_eiv_cov_res + std_eiv_cov_res, + color=color, alpha=0.5) + plt.plot(q_range, mean_noneiv_cov_res, + color=color, linestyle='dashed') + plt.fill_between(q_range, + mean_noneiv_cov_res - std_noneiv_cov_res, + mean_noneiv_cov_res + std_noneiv_cov_res, + color=color, alpha=0.3) + + +def coverage_difference_plot(eiv_coverages, noneiv_coverages, color, label='', + mean_error=True): + """ + Plot difference of numerical coverages from EiV to the one of non-EiV. + On the x axis q is plotted. + :param eiv_coverages: The output of `compute_coverages` with `eiv=True` + :param noneiv_coverages: The output of `compute_coverages` with `eiv=False` + :param color: String, denoting the color. + :param label: String. Will be included as label in the plot. + :param mean_error: Boolean. If True the standard deviation is divided + by the square root of the number of elements, to display the error + of the mean (and not the dispersion). + """ + cov_diff = eiv_coverages[0] - noneiv_coverages[0] + + mean_cov_diff = np.mean(cov_diff, axis=-1) + std_cov_diff = np.std(cov_diff, axis=-1) + if mean_error: + std_cov_diff /= np.sqrt(cov_diff.shape[1]) + plt.plot(q_range, mean_cov_diff, + color=color, linestyle='solid', label=label) + plt.fill_between(q_range, mean_cov_diff - std_cov_diff, + mean_cov_diff + std_cov_diff, + color=color, alpha=0.5) + + + + + +# create figures, together with title and axis labels +plt.figure(1) +plt.clf() +plt.title('Coverage for datasets with ground truth') +plt.xlabel('q') +plt.ylabel('coverage') +plt.figure(2) +plt.clf() +plt.title('Coverage of noisy labels (vs. theory)') +plt.xlabel('th. coverage') +plt.ylabel('coverage') +plt.figure(3) +plt.clf() +plt.title('Coverage of noisy labels (vs. q)') +plt.xlabel('q') +plt.ylabel('coverage') +plt.figure(4) +plt.clf() +plt.title('q Deviation of coverage of noisy labels') +plt.xlabel('q') +plt.ylabel('coverage') +plt.figure(5) +plt.clf() +plt.title('q Deviation of coverage for datasets with ground truth') +plt.xlabel('q') +plt.ylabel('deviation cov from q') +plt.figure(6) +plt.clf() +plt.title('Theory deviation of noisy coverage') +plt.xlabel('q') +plt.ylabel('deviation cov from theor. cov') +plt.figure(7) +plt.clf() +plt.title('Difference coverages') +plt.xlabel('q') +plt.ylabel('EiV cov - nonEiV-cov') + +# datasets to plot and their coloring +datasets = ['linear', 'quadratic','yacht','wine','power', + 'protein','concrete','california','energy','kin8nm','msd','naval'] +colors = cm.rainbow(np.linspace(0,1,len(datasets))) + +# loop through data +for use_ground_truth in [False, True]: + for data, color in zip(datasets, colors): + # compute coverages + eiv_coverages = compute_coverages(data=data, eiv=True, + number_of_draws=[100,5], + use_ground_truth=use_ground_truth) + noneiv_coverages = compute_coverages(data=data, eiv=False, + number_of_draws=100, + use_ground_truth=use_ground_truth) + # If no result was produced (no ground truth although required), skip + if eiv_coverages[0] is None and eiv_coverages[1] is None: + assert noneiv_coverages[0] is None and noneiv_coverages[1] is None + break + + # create plots + if use_ground_truth: + plt.figure(1) + coverage_diagonal_plot(eiv_coverages, noneiv_coverages, + color=color, against_theoretical=False, label=data) + plt.figure(5) + coverage_residual_plot(eiv_coverages, noneiv_coverages, + color=color, against_theoretical=False, label=data) + else: + plt.figure(2) + coverage_diagonal_plot(eiv_coverages, noneiv_coverages, + color=color, against_theoretical=True, label=data) + plt.figure(3) + coverage_diagonal_plot(eiv_coverages, noneiv_coverages, + color=color, against_theoretical=False, label=data) + plt.figure(4) + coverage_residual_plot(eiv_coverages, noneiv_coverages, + color=color, against_theoretical=False, label=data) + plt.figure(6) + coverage_residual_plot(eiv_coverages, noneiv_coverages, + color=color, against_theoretical=True, label=data) + plt.figure(7) + coverage_difference_plot(eiv_coverages, noneiv_coverages, + color=color, label=data) + +# add diagonals, where meaningful +plt.figure(1) +x_diag = np.linspace(0.0, 1.0) +plt.plot(x_diag, x_diag, color='k', linestyle='dotted' ) +plt.figure(2) +x_diag = np.linspace(0.0, 1.0) +plt.plot(x_diag, x_diag, color='k', linestyle='dotted' ) +plt.figure(3) +x_diag = np.linspace(0.0, 1.0) +plt.plot(x_diag, x_diag, color='k', linestyle='dotted' ) + + +# add legends +for fig_nr in range(1,8): + plt.figure(fig_nr) + plt.legend() + +plt.figure(1) +plt.savefig('results/figures/summary_coverage_ground_truth.pdf') +plt.figure(2) +plt.savefig('results/figures/summary_coverage_noisy_vs_th.pdf') +plt.figure(3) +plt.savefig('results/figures/summary_coverage_noisy_vs_q.pdf') +plt.figure(4) +plt.savefig('results/figures/summary_q_deviation_coverage_noisy.pdf') +plt.figure(5) +plt.savefig('results/figures/summary_q_deviation_coverage_ground_truth.pdf') +plt.figure(6) +plt.savefig('results/figures/summary_th_deviation_coverage_ground_truth.pdf') +plt.figure(7) +plt.savefig('results/figures/summary_difference_coverage.pdf') + +# show plots +plt.show() diff --git a/README.md b/README.md index 7547a9ddf61f21fe3abacba7584910cedd0dfe34..11bb840651f4a26d5b4fc09d78555994dc92f5b5 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ pip install EIVPackage/ Installing this package will make 3 modules available to the python environment: `EIVArchitectures` (for building EiV Models), `EIVTrainingRoutines` (containing a general training framework), `EIVGeneral` (containing a single module needed for repeated sampling). ## Missing -+ Tell to create folders, like `Experiments/results` ++ Tell to create folders, like `Experiments/results`, `Experiments/results/figures/` ## Contributing