diff --git a/Experiments/plot_coverage.py b/Experiments/plot_coverage.py
index 298603ce4b400606fa21e16c7997410183b7973f..ba21039aacbf8a02fc65303d4400c54a53938218 100644
--- a/Experiments/plot_coverage.py
+++ b/Experiments/plot_coverage.py
@@ -178,7 +178,7 @@ def compute_coverages(data, eiv, number_of_draws,
####
def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color,
- against_theoretical = False, label = ''):
+ against_theoretical = False, label = '', mean_error=True):
"""
Plot numerical coverages against q (used coverage value), if
against_theoretical is False, or the theoretical coverage, if
@@ -187,8 +187,10 @@ def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color,
:param noneiv_coverages: The output of `compute_coverages` with `eiv=False`
:param color: String, denoting the color.
:param against_theoretical: Boolean, see above.
- :param label: String. Will be combined with "eiv_"/"noneiv_" and included
- as label in the plot.
+ :param label: String. Will be included as label in the plot.
+ :param mean_error: Boolean. If True the standard deviation is divided
+ by the square root of the number of elements, to display the error
+ of the mean (and not the dispersion).
"""
eiv_numerical_coverage, eiv_theoretical_coverage = eiv_coverages
noneiv_numerical_coverage, noneiv_theoretical_coverage = noneiv_coverages
@@ -198,6 +200,8 @@ def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color,
# take mean/std over seed dimension
mean_eiv_numerical_coverage = np.mean(eiv_numerical_coverage, axis=-1)
std_eiv_numerical_coverage = np.std(eiv_numerical_coverage, axis=-1)
+ if mean_error:
+ std_eiv_numerical_coverage /= np.sqrt(eiv_numerical_coverage.shape[1])
if against_theoretical:
# show theoretical coverage on x-axis
x_values = np.mean(eiv_theoretical_coverage, axis=-1)
@@ -206,7 +210,7 @@ def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color,
x_values = np.array(q_range)
# plot mean
plt.plot(x_values, mean_eiv_numerical_coverage,
- color=color, linestyle='solid', label=f'eiv_{label}')
+ color=color, linestyle='solid', label=label)
# plot std
plt.fill_between(x_values,
mean_eiv_numerical_coverage - std_eiv_numerical_coverage,
@@ -216,6 +220,9 @@ def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color,
# take mean/std over seed dimension
mean_noneiv_numerical_coverage = np.mean(noneiv_numerical_coverage, axis=-1)
std_noneiv_numerical_coverage = np.std(noneiv_numerical_coverage, axis=-1)
+ if mean_error:
+ std_noneiv_numerical_coverage /= \
+ np.sqrt(noneiv_numerical_coverage.shape[1])
if against_theoretical:
# show theoretical coverage on x-axis
x_values = np.mean(noneiv_theoretical_coverage, axis=-1)
@@ -224,7 +231,7 @@ def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color,
x_values = np.array(q_range)
# plot mean
plt.plot(x_values, mean_noneiv_numerical_coverage,
- color=color, linestyle='dashed', label=f'noneiv_{label}')
+ color=color, linestyle='dashed')
# plot std
plt.fill_between(x_values,
mean_noneiv_numerical_coverage - std_noneiv_numerical_coverage,
@@ -232,7 +239,8 @@ def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color,
color=color, alpha=0.3)
def coverage_residual_plot(eiv_coverages, noneiv_coverages, color,
- absolute_values=True, against_theoretical=False, label=''):
+ absolute_values=True, against_theoretical=False, label='',
+ mean_error=True):
"""
Plot deviation of numerical coverages to q (used coverage value), if
against_theoretical is False, or to the theoretical coverage, if
@@ -244,8 +252,10 @@ def coverage_residual_plot(eiv_coverages, noneiv_coverages, color,
:param absolute_values:Boolean. If True (default) the absolute value of the
differences will be plotted.
:param against_theoretical: Boolean, see above.
- :param label: String. Will be combined with "eiv_"/"noneiv_" and included
- as label in the plot.
+ :param label: String. Will be included as label in the plot.
+ :param mean_error: Boolean. If True the standard deviation is divided
+ by the square root of the number of elements, to display the error
+ of the mean (and not the dispersion).
"""
if not against_theoretical:
eiv_coverages = eiv_coverages[0]
@@ -266,29 +276,59 @@ def coverage_residual_plot(eiv_coverages, noneiv_coverages, color,
noneiv_cov_res = np.abs(noneiv_cov_res)
mean_eiv_cov_res = np.mean(eiv_cov_res, axis=-1)
std_eiv_cov_res = np.std(eiv_cov_res, axis=-1)
+ if mean_error:
+ std_eiv_cov_res /= np.sqrt(eiv_cov_res.shape[1])
mean_noneiv_cov_res = np.mean(noneiv_cov_res, axis=-1)
std_noneiv_cov_res = np.std(noneiv_cov_res, axis=-1)
+ if mean_error:
+ std_noneiv_cov_res /= np.sqrt(noneiv_cov_res.shape[1])
plt.plot(q_range, mean_eiv_cov_res,
- color=color, linestyle='solid', label=f'eiv_{label}')
+ color=color, linestyle='solid', label=label)
plt.fill_between(q_range,
mean_eiv_cov_res - std_eiv_cov_res,
mean_eiv_cov_res + std_eiv_cov_res,
color=color, alpha=0.5)
plt.plot(q_range, mean_noneiv_cov_res,
- color=color, linestyle='dashed', label=f'noneiv_{label}')
+ color=color, linestyle='dashed')
plt.fill_between(q_range,
mean_noneiv_cov_res - std_noneiv_cov_res,
mean_noneiv_cov_res + std_noneiv_cov_res,
color=color, alpha=0.3)
+def coverage_difference_plot(eiv_coverages, noneiv_coverages, color, label='',
+ mean_error=True):
+ """
+ Plot difference of numerical coverages from EiV to the one of non-EiV.
+ On the x axis q is plotted.
+ :param eiv_coverages: The output of `compute_coverages` with `eiv=True`
+ :param noneiv_coverages: The output of `compute_coverages` with `eiv=False`
+ :param color: String, denoting the color.
+ :param label: String. Will be included as label in the plot.
+ :param mean_error: Boolean. If True the standard deviation is divided
+ by the square root of the number of elements, to display the error
+ of the mean (and not the dispersion).
+ """
+ cov_diff = eiv_coverages[0] - noneiv_coverages[0]
+
+ mean_cov_diff = np.mean(cov_diff, axis=-1)
+ std_cov_diff = np.std(cov_diff, axis=-1)
+ if mean_error:
+ std_cov_diff /= np.sqrt(cov_diff.shape[1])
+ plt.plot(q_range, mean_cov_diff,
+ color=color, linestyle='solid', label=label)
+ plt.fill_between(q_range, mean_cov_diff - std_cov_diff,
+ mean_cov_diff + std_cov_diff,
+ color=color, alpha=0.5)
+
+
# create figures, together with title and axis labels
plt.figure(1)
plt.clf()
-plt.title('Coverage for datasets with ground truth (vs. q)')
+plt.title('Coverage for datasets with ground truth')
plt.xlabel('q')
plt.ylabel('coverage')
plt.figure(2)
@@ -298,20 +338,29 @@ plt.xlabel('th. coverage')
plt.ylabel('coverage')
plt.figure(3)
plt.clf()
-plt.title('q Deviation of coverage of noisy labels')
+plt.title('Coverage of noisy labels (vs. q)')
plt.xlabel('q')
plt.ylabel('coverage')
plt.figure(4)
plt.clf()
+plt.title('q Deviation of coverage of noisy labels')
+plt.xlabel('q')
+plt.ylabel('coverage')
+plt.figure(5)
+plt.clf()
plt.title('q Deviation of coverage for datasets with ground truth')
plt.xlabel('q')
plt.ylabel('deviation cov from q')
-plt.figure(5)
+plt.figure(6)
plt.clf()
plt.title('Theory deviation of noisy coverage')
plt.xlabel('q')
plt.ylabel('deviation cov from theor. cov')
-
+plt.figure(7)
+plt.clf()
+plt.title('Difference coverages')
+plt.xlabel('q')
+plt.ylabel('EiV cov - nonEiV-cov')
# datasets to plot and their coloring
datasets = ['linear', 'quadratic','yacht','wine','power',
'protein','concrete','california','energy','kin8nm','msd','naval']
@@ -335,21 +384,27 @@ for use_ground_truth in [False, True]:
# create plots
if use_ground_truth:
plt.figure(1)
- coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color=color,
- against_theoretical=False, label=data)
- plt.figure(4)
- coverage_residual_plot(eiv_coverages, noneiv_coverages, color=color,
- against_theoretical=False, label=data)
+ coverage_diagonal_plot(eiv_coverages, noneiv_coverages,
+ color=color, against_theoretical=False, label=data)
+ plt.figure(5)
+ coverage_residual_plot(eiv_coverages, noneiv_coverages,
+ color=color, against_theoretical=False, label=data)
else:
plt.figure(2)
- coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color=color,
- against_theoretical=True, label=data)
+ coverage_diagonal_plot(eiv_coverages, noneiv_coverages,
+ color=color, against_theoretical=True, label=data)
plt.figure(3)
- coverage_residual_plot(eiv_coverages, noneiv_coverages, color=color,
- against_theoretical=False, label=data)
- plt.figure(5)
- coverage_residual_plot(eiv_coverages, noneiv_coverages, color=color,
- against_theoretical=True, label=data)
+ coverage_diagonal_plot(eiv_coverages, noneiv_coverages,
+ color=color, against_theoretical=False, label=data)
+ plt.figure(4)
+ coverage_residual_plot(eiv_coverages, noneiv_coverages,
+ color=color, against_theoretical=False, label=data)
+ plt.figure(6)
+ coverage_residual_plot(eiv_coverages, noneiv_coverages,
+ color=color, against_theoretical=True, label=data)
+ plt.figure(7)
+ coverage_difference_plot(eiv_coverages, noneiv_coverages,
+ color=color, label=data)
# add diagonals, where meaningful
plt.figure(1)
@@ -364,21 +419,26 @@ plt.plot(x_diag, x_diag, color='k', linestyle='dotted' )
# add legends
-for fig_nr in range(1,6):
+for fig_nr in range(1,8):
plt.figure(fig_nr)
plt.legend()
plt.figure(1)
plt.savefig('results/figures/summary_coverage_ground_truth.pdf')
plt.figure(2)
-plt.savefig('results/figures/summary_coverage_noisy.pdf')
+plt.savefig('results/figures/summary_coverage_noisy_vs_th.pdf')
plt.figure(3)
-plt.savefig('results/figures/summary_q_deviation_coverage_noisy.pdf')
+plt.savefig('results/figures/summary_coverage_noisy_vs_q.pdf')
plt.figure(4)
-plt.savefig('results/figures/summary_q_deviation_coverage_ground_truth.pdf')
+plt.savefig('results/figures/summary_q_deviation_coverage_noisy.pdf')
plt.figure(5)
-plt.savefig('results/figures/summary_theory_deviation_coverage_ground_truth.pdf')
+plt.savefig('results/figures/summary_q_deviation_coverage_ground_truth.pdf')
+plt.figure(6)
+plt.savefig('results/figures/summary_th_deviation_coverage_ground_truth.pdf')
+plt.figure(7)
+plt.savefig('results/figures/summary_difference_coverage.pdf')
+
-# show plot
-plt.show()
+#####
+plt.show()