diff --git a/EIVPackage/EIVGeneral/coverage_collect.py b/EIVPackage/EIVGeneral/coverage_collect.py
index 934eaac864beb0425447fa3be2b5442abe3a04a3..642315ebb0935322c83e5b6b8686b44011a38434 100644
--- a/EIVPackage/EIVGeneral/coverage_collect.py
+++ b/EIVPackage/EIVGeneral/coverage_collect.py
@@ -7,7 +7,8 @@ import torch.backends.cudnn
from EIVGeneral.coverage_metrics import epistemic_coverage
def get_coverages(not_averaged_predictions, y,\
- q_range=np.linspace(0.1,0.9,num=30)):
+ q_range=np.linspace(0.1,0.9,num=30),
+ noisy_y=True):
"""
Compute the numerical and theoretical coverage for various coverage
factors, computed from the `q` in `q_range`, using
@@ -20,6 +21,9 @@ def get_coverages(not_averaged_predictions, y,\
:param y: A `torch.tensor` of the same shape then the second components
of `not_averaged_predictions`. If the feature dimension is missing, it is added.
:param q_range: An iterator through floats between 0 and 1.
+ :param noisy_y: Boolean. If True (the default), `y` is treated as noisy and
+ the total uncertainty is considered. If False, `y` is treated as the
+ unnoisy ground truth.
:returns: numerical_coverage, theoretical_coverage (numpy arrays)
"""
numerical_coverage_collection = []
@@ -28,7 +32,7 @@ def get_coverages(not_averaged_predictions, y,\
numerical_coverage, theoretical_coverage =\
epistemic_coverage(\
not_averaged_predictions=not_averaged_predictions,
- y=y, q=q)
+ y=y, q=q, noisy_y=noisy_y)
numerical_coverage_collection.append(float(numerical_coverage))
theoretical_coverage_collection.append(float(theoretical_coverage))
return np.array(numerical_coverage_collection),\
@@ -37,7 +41,8 @@ def get_coverages(not_averaged_predictions, y,\
def get_coverage_distribution(net_iterator, dataloader_iterator,
device, number_of_draws, q_range=np.linspace(0.1,0.9,num=30),
- number_of_test_samples=10, stack=True):
+ number_of_test_samples=10, stack=True,
+ noisy_y=True):
"""
Returns the numerical and theoretical coverage for all nets in
`net_iterator` with dataloader from `dataloader_iterator`
@@ -53,6 +58,9 @@ def get_coverage_distribution(net_iterator, dataloader_iterator,
dataloaders in dataloader_iterator to be used.
:param stack: Boolean. If True (default) the results will be stacked along
the last dimension. If False a list will be returned.
+ :param noisy_y: Boolean. If True (the default), `y` is treated as noisy and
+ the total uncertainty is considered. If False, `y` is treated as the
+ unnoisy ground truth.
:returns: num_coverage_collection, th_coverage_collection
"""
num_coverage_collection, th_coverage_collection = [],[]
@@ -78,7 +86,8 @@ def get_coverage_distribution(net_iterator, dataloader_iterator,
y_collection = torch.concat(y_collection, dim=0)
numerical_coverage, theoretical_coverage = get_coverages(
not_averaged_predictions=not_av_pred_collection,
- y=y_collection, q_range=q_range)
+ y=y_collection, q_range=q_range,
+ noisy_y=noisy_y)
num_coverage_collection.append(numerical_coverage)
th_coverage_collection.append(theoretical_coverage)
if stack:
diff --git a/Experiments/plot_coverage.py b/Experiments/plot_coverage.py
index 2b625eb6e1c9ea34bf3464af4cc4b5e1bf53055f..f9d3f22e81cc77f0533a9b975d811372bd8ea5c9 100644
--- a/Experiments/plot_coverage.py
+++ b/Experiments/plot_coverage.py
@@ -6,6 +6,7 @@ import importlib
import os
import json
+import numpy as np
import torch
import torch.backends.cudnn
from torch.utils.data import DataLoader
@@ -16,9 +17,12 @@ from EIVTrainingRoutines import train_and_store
from EIVGeneral.coverage_collect import get_coverage_distribution
from EIVGeneral.manipulate_datasets import VerticalCut
+# coverages to consider
+q_range = np.linspace(0.1, 0.95)
# load hyperparameters from JSON file
-def get_coverages(data, eiv, number_of_draws, use_ground_truth=False):
+def compute_coverages(data, eiv, number_of_draws,
+ use_ground_truth=False, noisy_y=True):
"""
Create network and dataloader iterators for `data` (short dataname) and
feed them into `get_coverage_distribution`.
@@ -43,7 +47,6 @@ def get_coverages(data, eiv, number_of_draws, use_ground_truth=False):
long_dataname = conf_dict["long_dataname"]
short_dataname = conf_dict["short_dataname"]
- print(f"Plotting coverage for {long_dataname}")
load_data = importlib.import_module(f'EIVData.{long_dataname}').load_data
@@ -75,6 +78,7 @@ def get_coverages(data, eiv, number_of_draws, use_ground_truth=False):
else:
train_data, _ = load_data()
+ print(f"Computing {'EiV' if eiv else 'non-EiV'} coverage for {long_dataname}")
# train_data only used for finding dimensions
input_dim = train_data[0][0].numel()
@@ -161,14 +165,62 @@ def get_coverages(data, eiv, number_of_draws, use_ground_truth=False):
net_iterator=net_iterator(eiv=eiv),
dataloader_iterator=dataloader_iterator(),
device=device,
- number_of_draws=number_of_draws)
+ number_of_draws=number_of_draws,
+ q_range=q_range,
+ noisy_y = not use_ground_truth)
return numerical_coverage, theoretical_coverage
-#####
-# numerical_coverage, theoretical_coverage =\
-# get_coverages(data='quadratic', eiv=True, number_of_draws=100,
-# use_ground_truth=True)
-# print(numerical_coverage)
-# print(theoretical_coverage)
+####
+
+def plot_data(eiv_coverages, noneiv_coverages, color='r',
+ against_theoretical = False):
+ eiv_numerical_coverage, eiv_theoretical_coverage = eiv_coverages
+ noneiv_numerical_coverage, noneiv_theoretical_coverage = noneiv_coverages
+ if eiv_numerical_coverage is not None and eiv_theoretical_coverage is not None:
+ mean_eiv_numerical_coverage = np.mean(eiv_numerical_coverage, axis=-1)
+ std_eiv_numerical_coverage = np.std(eiv_numerical_coverage, axis=-1)
+ if against_theoretical:
+ x_values = np.mean(eiv_theoretical_coverage, axis=-1)
+ else:
+ x_values = np.array(q_range)
+ plt.plot(x_values, mean_eiv_numerical_coverage,
+ color=color, linestyle='solid')
+ plt.fill_between(x_values,
+ mean_eiv_numerical_coverage - std_eiv_numerical_coverage,
+ mean_eiv_numerical_coverage + std_eiv_numerical_coverage,
+ color=color, alpha=0.5)
+ if noneiv_numerical_coverage is not None and noneiv_theoretical_coverage is not None:
+ mean_noneiv_numerical_coverage = np.mean(noneiv_numerical_coverage, axis=-1)
+ std_noneiv_numerical_coverage = np.std(noneiv_numerical_coverage, axis=-1)
+ if against_theoretical:
+ x_values = np.mean(noneiv_theoretical_coverage, axis=-1)
+ else:
+ x_values = np.array(q_range)
+ plt.plot(x_values, mean_noneiv_numerical_coverage,
+ color=color, linestyle='dashed')
+ plt.fill_between(x_values,
+ mean_noneiv_numerical_coverage - std_noneiv_numerical_coverage,
+ mean_noneiv_numerical_coverage + std_noneiv_numerical_coverage,
+ color=color, alpha=0.3)
+
+
+plt.figure(1)
+plt.clf()
+datasets = ['energy', 'kin8nm', 'yacht', 'wine']
+colors = ['red', 'blue', 'green', 'purple']
+use_ground_truth = False
+against_theoretical = True
+for data, color in zip(datasets, colors):
+ eiv_coverages = compute_coverages(data=data, eiv=True,
+ number_of_draws=[100,5],
+ use_ground_truth=use_ground_truth)
+ noneiv_coverages = compute_coverages(data=data, eiv=False,
+ number_of_draws=100,
+ use_ground_truth=use_ground_truth)
+ plot_data(eiv_coverages, noneiv_coverages, color=color,
+ against_theoretical=against_theoretical)
+x_diag = np.linspace(0.0, 1.0)
+plt.plot(x_diag, x_diag, color='k', linestyle='dotted' )
+plt.show()