missing noisy_y included

EIVPackage/EIVGeneral/coverage_collect.py

@@ -7,7 +7,8 @@ import torch.backends.cudnn
 from EIVGeneral.coverage_metrics import epistemic_coverage
 
 def get_coverages(not_averaged_predictions, y,\
-        q_range=np.linspace(0.1,0.9,num=30)):
+        q_range=np.linspace(0.1,0.9,num=30),
+        noisy_y=True):
     """
     Compute the numerical and theoretical coverage for various coverage
     factors, computed from the `q` in `q_range`, using
@@ -20,6 +21,9 @@ def get_coverages(not_averaged_predictions, y,\
     :param y: A `torch.tensor` of the same shape then the second components
     of `not_averaged_predictions`. If the feature dimension is missing, it is added.
     :param q_range: An iterator through floats between 0 and 1.
+    :param noisy_y: Boolean. If True (the default), `y` is treated as noisy and
+    the total uncertainty is considered. If False, `y` is treated as the
+    unnoisy ground truth.
     :returns: numerical_coverage, theoretical_coverage (numpy arrays)
     """
     numerical_coverage_collection = []
@@ -28,7 +32,7 @@ def get_coverages(not_averaged_predictions, y,\
         numerical_coverage, theoretical_coverage =\
                 epistemic_coverage(\
                 not_averaged_predictions=not_averaged_predictions,
-                y=y, q=q)
+                y=y, q=q, noisy_y=noisy_y)
         numerical_coverage_collection.append(float(numerical_coverage))
         theoretical_coverage_collection.append(float(theoretical_coverage))
     return np.array(numerical_coverage_collection),\
@@ -37,7 +41,8 @@ def get_coverages(not_averaged_predictions, y,\
 
 def get_coverage_distribution(net_iterator, dataloader_iterator,
         device, number_of_draws, q_range=np.linspace(0.1,0.9,num=30),
-        number_of_test_samples=10, stack=True):
+        number_of_test_samples=10, stack=True,
+        noisy_y=True):
     """
     Returns the numerical and theoretical coverage for all nets in
     `net_iterator` with dataloader from `dataloader_iterator`
@@ -53,6 +58,9 @@ def get_coverage_distribution(net_iterator, dataloader_iterator,
     dataloaders in dataloader_iterator to be used.
     :param stack: Boolean. If True (default) the results will be stacked along 
     the last dimension. If False a list will be returned.
+    :param noisy_y: Boolean. If True (the default), `y` is treated as noisy and
+    the total uncertainty is considered. If False, `y` is treated as the
+    unnoisy ground truth.
     :returns: num_coverage_collection, th_coverage_collection
     """
     num_coverage_collection, th_coverage_collection = [],[]
@@ -78,7 +86,8 @@ def get_coverage_distribution(net_iterator, dataloader_iterator,
                 y_collection = torch.concat(y_collection, dim=0)
                 numerical_coverage, theoretical_coverage = get_coverages(
                         not_averaged_predictions=not_av_pred_collection,
-                        y=y_collection, q_range=q_range)
+                        y=y_collection, q_range=q_range,
+                        noisy_y=noisy_y)
                 num_coverage_collection.append(numerical_coverage)
                 th_coverage_collection.append(theoretical_coverage)
     if stack:

Experiments/plot_coverage.py

@@ -6,6 +6,7 @@ import importlib
 import os
 import json
 
+import numpy as np
 import torch
 import torch.backends.cudnn
 from torch.utils.data import DataLoader
@@ -16,9 +17,12 @@ from EIVTrainingRoutines import train_and_store
 from EIVGeneral.coverage_collect import get_coverage_distribution
 from EIVGeneral.manipulate_datasets import VerticalCut
 
+# coverages to consider
+q_range = np.linspace(0.1, 0.95)
 
 # load hyperparameters from JSON file
-def get_coverages(data, eiv, number_of_draws, use_ground_truth=False):
+def compute_coverages(data, eiv, number_of_draws,
+        use_ground_truth=False, noisy_y=True):
     """
     Create network and dataloader iterators for `data` (short dataname) and
     feed them into `get_coverage_distribution`.
@@ -43,7 +47,6 @@ def get_coverages(data, eiv, number_of_draws, use_ground_truth=False):
     long_dataname = conf_dict["long_dataname"]
     short_dataname = conf_dict["short_dataname"]
 
-    print(f"Plotting coverage for {long_dataname}")
 
     load_data = importlib.import_module(f'EIVData.{long_dataname}').load_data
 
@@ -75,6 +78,7 @@ def get_coverages(data, eiv, number_of_draws, use_ground_truth=False):
     else:
         train_data, _ = load_data()
 
+    print(f"Computing {'EiV' if eiv else 'non-EiV'} coverage for {long_dataname}")
 
     # train_data only used for finding dimensions 
     input_dim = train_data[0][0].numel()
@@ -161,14 +165,62 @@ def get_coverages(data, eiv, number_of_draws, use_ground_truth=False):
         net_iterator=net_iterator(eiv=eiv),
         dataloader_iterator=dataloader_iterator(),
         device=device,
-        number_of_draws=number_of_draws)
+        number_of_draws=number_of_draws,
+        q_range=q_range,
+        noisy_y = not use_ground_truth)
     return numerical_coverage, theoretical_coverage
 
 
 
-#####
-# numerical_coverage, theoretical_coverage =\
-#     get_coverages(data='quadratic', eiv=True, number_of_draws=100,
-#             use_ground_truth=True)
-# print(numerical_coverage)
-# print(theoretical_coverage)
+####
+
+def plot_data(eiv_coverages, noneiv_coverages, color='r',
+        against_theoretical = False):
+    eiv_numerical_coverage, eiv_theoretical_coverage = eiv_coverages
+    noneiv_numerical_coverage, noneiv_theoretical_coverage = noneiv_coverages
+    if eiv_numerical_coverage is not None and eiv_theoretical_coverage is not None:
+        mean_eiv_numerical_coverage = np.mean(eiv_numerical_coverage, axis=-1) 
+        std_eiv_numerical_coverage = np.std(eiv_numerical_coverage, axis=-1)
+        if against_theoretical:
+            x_values = np.mean(eiv_theoretical_coverage, axis=-1)
+        else:
+            x_values = np.array(q_range)
+        plt.plot(x_values, mean_eiv_numerical_coverage,
+                color=color, linestyle='solid')
+        plt.fill_between(x_values,
+                mean_eiv_numerical_coverage - std_eiv_numerical_coverage,
+                mean_eiv_numerical_coverage + std_eiv_numerical_coverage, 
+                color=color, alpha=0.5)
+    if noneiv_numerical_coverage is not None and noneiv_theoretical_coverage is not None:
+        mean_noneiv_numerical_coverage = np.mean(noneiv_numerical_coverage, axis=-1)
+        std_noneiv_numerical_coverage = np.std(noneiv_numerical_coverage, axis=-1)
+        if against_theoretical:
+            x_values = np.mean(noneiv_theoretical_coverage, axis=-1)
+        else:
+            x_values = np.array(q_range)
+        plt.plot(x_values, mean_noneiv_numerical_coverage,
+                color=color, linestyle='dashed')
+        plt.fill_between(x_values,
+                mean_noneiv_numerical_coverage - std_noneiv_numerical_coverage,
+                mean_noneiv_numerical_coverage + std_noneiv_numerical_coverage,
+                color=color, alpha=0.3)
+
+
+plt.figure(1)
+plt.clf()
+datasets = ['energy', 'kin8nm', 'yacht', 'wine']
+colors = ['red', 'blue', 'green', 'purple']
+use_ground_truth = False
+against_theoretical = True
+for data, color in zip(datasets, colors):
+    eiv_coverages = compute_coverages(data=data, eiv=True,
+            number_of_draws=[100,5],
+            use_ground_truth=use_ground_truth)
+    noneiv_coverages = compute_coverages(data=data, eiv=False,
+            number_of_draws=100,
+            use_ground_truth=use_ground_truth)
+    plot_data(eiv_coverages, noneiv_coverages, color=color,
+            against_theoretical=against_theoretical)
+x_diag = np.linspace(0.0, 1.0)
+plt.plot(x_diag, x_diag, color='k', linestyle='dotted' )
+plt.show()