diff --git a/Experiments/plot_coverage.py b/Experiments/plot_coverage.py
index f9d3f22e81cc77f0533a9b975d811372bd8ea5c9..f4a93a4c8776b4101bc6277e885d18e3fdf786b3 100644
--- a/Experiments/plot_coverage.py
+++ b/Experiments/plot_coverage.py
@@ -1,6 +1,8 @@
"""
-Compute metrics for datasets for which there is not necessarily a ground truth.
-Results will be stored in the results folder
+Compute the coverage for various coverage factors and compare them with
+with the corresponding q or theoretical coverage. Results will be stored
+in various plots in the results/figures folder.
+TODO: Do the latter
"""
import importlib
import os
@@ -174,8 +176,19 @@ def compute_coverages(data, eiv, number_of_draws,
####
-def plot_data(eiv_coverages, noneiv_coverages, color='r',
- against_theoretical = False):
+def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color,
+ against_theoretical = False, label = ''):
+ """
+ Plot numerical coverages against q (used coverage value), if
+ against_theoretical is False, or the theoretical coverage, if
+ `against_theoretical` is True.
+ :param eiv_coverages: The output of `compute_coverages` with `eiv=True`
+ :param noneiv_coverages: The output of `compute_coverages` with `eiv=False`
+ :param color: String, denoting the color.
+ :param against_theoretical: Boolean, see above.
+ :param label: String. Will be combined with "eiv_"/"noneiv_" and included
+ as label in the plot.
+ """
eiv_numerical_coverage, eiv_theoretical_coverage = eiv_coverages
noneiv_numerical_coverage, noneiv_theoretical_coverage = noneiv_coverages
if eiv_numerical_coverage is not None and eiv_theoretical_coverage is not None:
@@ -186,7 +199,7 @@ def plot_data(eiv_coverages, noneiv_coverages, color='r',
else:
x_values = np.array(q_range)
plt.plot(x_values, mean_eiv_numerical_coverage,
- color=color, linestyle='solid')
+ color=color, linestyle='solid', label=f'eiv_{label}')
plt.fill_between(x_values,
mean_eiv_numerical_coverage - std_eiv_numerical_coverage,
mean_eiv_numerical_coverage + std_eiv_numerical_coverage,
@@ -199,28 +212,147 @@ def plot_data(eiv_coverages, noneiv_coverages, color='r',
else:
x_values = np.array(q_range)
plt.plot(x_values, mean_noneiv_numerical_coverage,
- color=color, linestyle='dashed')
+ color=color, linestyle='dashed', label=f'noneiv_{label}')
plt.fill_between(x_values,
mean_noneiv_numerical_coverage - std_noneiv_numerical_coverage,
mean_noneiv_numerical_coverage + std_noneiv_numerical_coverage,
color=color, alpha=0.3)
+def coverage_residual_plot(eiv_coverages, noneiv_coverages, color,
+ absolute_values=True, against_theoretical=False, label=''):
+ """
+ Plot deviation of numerical coverages to q (used coverage value), if
+ against_theoretical is False, or to the theoretical coverage, if
+ `against_theoretical` is True. On the x axis q is plotted (in contrast to
+ coverage_diagonal_plot).
+ :param eiv_coverages: The output of `compute_coverages` with `eiv=True`
+ :param noneiv_coverages: The output of `compute_coverages` with `eiv=False`
+ :param color: String, denoting the color.
+ :param absolute_values:Boolean. If True (default) the absolute value of the
+ differences will be plotted.
+ :param against_theoretical: Boolean, see above.
+ :param label: String. Will be combined with "eiv_"/"noneiv_" and included
+ as label in the plot.
+ """
+ if not against_theoretical:
+ eiv_coverages = eiv_coverages[0]
+ noneiv_coverages = noneiv_coverages[0]
+ assert len(eiv_coverages.shape) == 2 and\
+ eiv_coverages.shape[0] == len(q_range)
+ assert len(eiv_coverages.shape) == 2 and\
+ eiv_coverages.shape[0] == len(q_range)
+ assert len(noneiv_coverages.shape) == 2
+ eiv_cov_res = eiv_coverages - q_range.reshape((-1,1))
+ noneiv_cov_res = noneiv_coverages - q_range.reshape((-1,1))
+ else:
+ eiv_cov_res = eiv_coverages[0] - eiv_coverages[1]
+ noneiv_cov_res = noneiv_coverages[0] - noneiv_coverages[1]
+
+ if absolute_values:
+ eiv_cov_res = np.abs(eiv_cov_res)
+ noneiv_cov_res = np.abs(noneiv_cov_res)
+ mean_eiv_cov_res = np.mean(eiv_cov_res, axis=-1)
+ std_eiv_cov_res = np.std(eiv_cov_res, axis=-1)
+ mean_noneiv_cov_res = np.mean(noneiv_cov_res, axis=-1)
+ std_noneiv_cov_res = np.std(noneiv_cov_res, axis=-1)
+ plt.plot(q_range, mean_eiv_cov_res,
+ color=color, linestyle='solid', label=f'eiv_{label}')
+ plt.fill_between(q_range,
+ mean_eiv_cov_res - std_eiv_cov_res,
+ mean_eiv_cov_res + std_eiv_cov_res,
+ color=color, alpha=0.5)
+ plt.plot(q_range, mean_noneiv_cov_res,
+ color=color, linestyle='dashed', label=f'noneiv_{label}')
+ plt.fill_between(q_range,
+ mean_noneiv_cov_res - std_noneiv_cov_res,
+ mean_noneiv_cov_res + std_noneiv_cov_res,
+ color=color, alpha=0.3)
+
+
+
+
+# create figures, together with title and axis labels
plt.figure(1)
plt.clf()
-datasets = ['energy', 'kin8nm', 'yacht', 'wine']
-colors = ['red', 'blue', 'green', 'purple']
-use_ground_truth = False
-against_theoretical = True
-for data, color in zip(datasets, colors):
- eiv_coverages = compute_coverages(data=data, eiv=True,
- number_of_draws=[100,5],
- use_ground_truth=use_ground_truth)
- noneiv_coverages = compute_coverages(data=data, eiv=False,
- number_of_draws=100,
- use_ground_truth=use_ground_truth)
- plot_data(eiv_coverages, noneiv_coverages, color=color,
- against_theoretical=against_theoretical)
+plt.title('Coverage for datasets with ground truth')
+plt.xlabel('q')
+plt.ylabel('coverage')
+plt.figure(2)
+plt.clf()
+plt.title('Coverage of noisy labels (vs. theory)')
+plt.xlabel('th. coverage')
+plt.ylabel('coverage')
+plt.figure(3)
+plt.clf()
+plt.title('Coverage of noisy labels (vs. q)')
+plt.xlabel('q')
+plt.ylabel('coverage')
+plt.figure(4)
+plt.clf()
+plt.title('Theoretical deviation Coverage for datasets with ground truth')
+plt.xlabel('q')
+plt.ylabel('deviation cov from q')
+plt.figure(5)
+plt.clf()
+plt.title('Deviation (theory) of noisy Coverage')
+plt.xlabel('q')
+plt.ylabel('deviation cov from theor. cov')
+
+# datasets to plot and their coloring
+datasets = ['linear', 'quadratic','yacht','wine','power']
+colors = ['red', 'blue','green','purple','orange','cyan']
+
+# loop through data
+for use_ground_truth in [False, True]:
+ for data, color in zip(datasets, colors):
+ # compute coverages
+ eiv_coverages = compute_coverages(data=data, eiv=True,
+ number_of_draws=[100,5],
+ use_ground_truth=use_ground_truth)
+ noneiv_coverages = compute_coverages(data=data, eiv=False,
+ number_of_draws=100,
+ use_ground_truth=use_ground_truth)
+ # If no result was produced (no ground truth although required), skip
+ if eiv_coverages[0] is None and eiv_coverages[1] is None:
+ assert noneiv_coverages[0] is None and noneiv_coverages[1] is None
+ break
+
+ # create plots
+ if use_ground_truth:
+ plt.figure(1)
+ coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color=color,
+ against_theoretical=False, label=data)
+ plt.figure(4)
+ coverage_residual_plot(eiv_coverages, noneiv_coverages, color=color,
+ against_theoretical=False, label=data)
+ else:
+ plt.figure(2)
+ coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color=color,
+ against_theoretical=True, label=data)
+ plt.figure(3)
+ coverage_residual_plot(eiv_coverages, noneiv_coverages, color=color,
+ against_theoretical=False, label=data)
+ plt.figure(5)
+ coverage_residual_plot(eiv_coverages, noneiv_coverages, color=color,
+ against_theoretical=True, label=data)
+
+# add diagonals, where meaningful
+plt.figure(1)
x_diag = np.linspace(0.0, 1.0)
plt.plot(x_diag, x_diag, color='k', linestyle='dotted' )
+plt.figure(2)
+x_diag = np.linspace(0.0, 1.0)
+plt.plot(x_diag, x_diag, color='k', linestyle='dotted' )
+plt.figure(3)
+x_diag = np.linspace(0.0, 1.0)
+plt.plot(x_diag, x_diag, color='k', linestyle='dotted' )
+
+
+# add legends
+for fig_nr in range(1,6):
+ plt.figure(fig_nr)
+ plt.legend()
+
+# show plots
plt.show()