diff --git a/Experiments/create_tabular.py b/Experiments/create_tabular.py
index a83dcb94ab6484b853dd4487e710a13543a27170..cd68380681bc53a858d5e55cf21e2f5a989d6dfe 100644
--- a/Experiments/create_tabular.py
+++ b/Experiments/create_tabular.py
@@ -2,7 +2,8 @@ import os
import glob
import json
-metrics_to_display = ['rmse','true_coverage_numerical', 'total_coverage']
+metrics_to_display = ['rmse', 'coverage_numerical',
+ 'true_coverage_numerical', 'total_coverage']
show_incomplete = True
list_of_result_files = glob.glob(os.path.join('results','*.json'))
diff --git a/Experiments/plot_prediction.py b/Experiments/plot_prediction.py
index a574c529e1e75b244075b32e743392e57298c636..f1f985ffbbcb4b772e75a3bbb10942e31fdd6b8c 100644
--- a/Experiments/plot_prediction.py
+++ b/Experiments/plot_prediction.py
@@ -283,9 +283,9 @@ for data, x_range, color, number_of_draws in zip(data_list,
fignum += 1
func = eiv_plotting_dictionary['func']
x_point = x_values[zoom_point]
- y_point = func(x_point)
+ y_point = func(torch.tensor(x_point)).numpy()
noisy_x_point = noisy_x_values[zoom_point]
- func_noisy_x_point = func(noisy_x_point)
+ func_noisy_x_point = func(torch.tensor(noisy_x_point)).numpy()
plt.figure(fignum)
plt.clf()
plt.plot(x_values.flatten(), y_values.flatten(),'-', color='k', linewidth=linewidth)
diff --git a/Experiments/plot_summary.py b/Experiments/plot_summary.py
index c0863c604ff618df5e45351576c493cece331f50..4c7780a3463f3605694024933c5e845a8811148a 100644
--- a/Experiments/plot_summary.py
+++ b/Experiments/plot_summary.py
@@ -10,10 +10,17 @@ import glob
import json
import numpy as np
+import matplotlib
import matplotlib.pyplot as plt
+font = {'family' : 'DejaVu Sans',
+ 'weight' : 'normal',
+ 'size' : 16}
+
+matplotlib.rc('font', **font)
k = 2
+
# load in all available result files
list_of_result_files = glob.glob(os.path.join('results','*.json'))
results = {}
@@ -46,6 +53,7 @@ metric = 'rmse'
data_list = results.keys()
colors = ['red', 'blue']
ymax = 0.8
+minimal_bar_size = ymax * 1.5e-3
# read out EiV and non-EiV results for all datasets
metric_results = [
(save_readout(results[data]['eiv'], metric),
@@ -55,7 +63,7 @@ metric_results = [
# create figure
plt.figure(1)
plt.clf()
-plt.title('RMSE')
+plt.gcf().canvas.manager.set_window_title('RMSE')
# plot bars
for i, ([(eiv_metric_mean, eiv_metric_std),
@@ -66,24 +74,28 @@ for i, ([(eiv_metric_mean, eiv_metric_std),
assert noneiv_metric_mean is not None
if eiv_metric_std is not None:
assert noneiv_metric_std is not None
+ eiv_bar_size = max(eiv_metric_std, minimal_bar_size)
+ noneiv_bar_size = max(noneiv_metric_std, minimal_bar_size)
plt.plot(i+1, eiv_metric_mean, '^', color=colors[0])
plt.bar(i+1,
- height = 2*eiv_metric_std,
- width = 0.1,
- bottom = eiv_metric_mean - eiv_metric_std,
+ height = 2*eiv_bar_size,
+ width = 0.3,
+ bottom = eiv_metric_mean - eiv_bar_size,
color=colors[0],
alpha=0.5)
plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1])
plt.bar(i+1,
- height = 2 * k *noneiv_metric_std,
- width = 0.1,
- bottom = noneiv_metric_mean - k* noneiv_metric_std,
+ height = 2 * k *noneiv_bar_size,
+ width = 0.3,
+ bottom = noneiv_metric_mean - k* noneiv_bar_size,
color=colors[1],
alpha=0.5)
plt.ylim(bottom=0, top=ymax)
ax = plt.gca()
ax.set_xticks(np.arange(1,len(data_list)+1))
ax.set_xticklabels(data_list, rotation='vertical')
+plt.ylabel('RMSE')
+plt.tight_layout()
plt.savefig('results/figures/RMSE_bar_plot.pdf')
## coverage plot
@@ -92,6 +104,7 @@ metric = 'true_coverage_numerical'
data_list = ['linear','quadratic','cubic','sine']
colors = ['red', 'blue']
ymax = 1.0
+minimal_bar_size = ymax * 1.5e-3
# read out EiV and non-EiV results for all datasets
metric_results = [
(save_readout(results[data]['eiv'], metric),
@@ -101,7 +114,7 @@ metric_results = [
# create figure
plt.figure(2)
plt.clf()
-plt.title('coverage (ground truth)')
+plt.gcf().canvas.manager.set_window_title('coverage (ground truth)')
# plot bars
for i, ([(eiv_metric_mean, eiv_metric_std),
@@ -112,18 +125,20 @@ for i, ([(eiv_metric_mean, eiv_metric_std),
assert noneiv_metric_mean is not None
if eiv_metric_std is not None:
assert noneiv_metric_std is not None
+ eiv_bar_size = max(eiv_metric_std, minimal_bar_size)
+ noneiv_bar_size = max(noneiv_metric_std, minimal_bar_size)
plt.plot(i+1, eiv_metric_mean, '^', color=colors[0])
plt.bar(i+1,
- height = 2*eiv_metric_std,
+ height = 2*eiv_bar_size,
width = 0.1,
- bottom = eiv_metric_mean - eiv_metric_std,
+ bottom = eiv_metric_mean - eiv_bar_size,
color=colors[0],
alpha=0.5)
plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1])
plt.bar(i+1,
- height = 2 * k *noneiv_metric_std,
+ height = 2 * k *noneiv_bar_size,
width = 0.1,
- bottom = noneiv_metric_mean - k* noneiv_metric_std,
+ bottom = noneiv_metric_mean - k* noneiv_bar_size,
color=colors[1],
alpha=0.5)
plt.axhline(0.95,0.0,1.0,color='k', linestyle='dashed')
@@ -131,6 +146,8 @@ plt.ylim(bottom=0, top=ymax)
ax = plt.gca()
ax.set_xticks(np.arange(1,len(data_list)+1))
ax.set_xticklabels(data_list, rotation='vertical')
+plt.ylabel('coverage ground truth')
+plt.tight_layout()
plt.savefig('results/figures/true_coverage_bar_plot.pdf')
## noisy coverage plot
@@ -139,6 +156,7 @@ metric = 'coverage_numerical'
data_list = results.keys()
colors = ['red', 'blue']
ymax = 1.0
+minimal_bar_size = ymax * 1.5e-3
# read out EiV and non-EiV results for all datasets
metric_results = [
(save_readout(results[data]['eiv'], metric),
@@ -148,7 +166,7 @@ metric_results = [
# create figure
plt.figure(3)
plt.clf()
-plt.title('coverage (noisy labels)')
+plt.gcf().canvas.manager.set_window_title('coverage (noisy labels)')
# plot bars
for i, ([(eiv_metric_mean, eiv_metric_std),
@@ -159,22 +177,78 @@ for i, ([(eiv_metric_mean, eiv_metric_std),
assert noneiv_metric_mean is not None
if eiv_metric_std is not None:
assert noneiv_metric_std is not None
+ eiv_bar_size = max(eiv_metric_std, minimal_bar_size)
+ noneiv_bar_size = max(noneiv_metric_std, minimal_bar_size)
plt.plot(i+1, eiv_metric_mean, '^', color=colors[0])
plt.bar(i+1,
- height = 2*eiv_metric_std,
+ height = 2*eiv_bar_size,
width = 0.1,
- bottom = eiv_metric_mean - eiv_metric_std,
+ bottom = eiv_metric_mean - eiv_bar_size,
color=colors[0],
alpha=0.5)
plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1])
plt.bar(i+1,
- height = 2 * k *noneiv_metric_std,
+ height = 2 * k *noneiv_bar_size,
width = 0.1,
- bottom = noneiv_metric_mean - k* noneiv_metric_std,
+ bottom = noneiv_metric_mean - k* noneiv_bar_size,
color=colors[1],
alpha=0.5)
plt.ylim(bottom=0, top=ymax)
ax = plt.gca()
ax.set_xticks(np.arange(1,len(data_list)+1))
ax.set_xticklabels(data_list, rotation='vertical')
+plt.ylabel('coverage (epis. unc. / ' + r'$u(x)$' + ')' )
+plt.tight_layout()
plt.savefig('results/figures/noisy_coverage_bar_plot.pdf')
+
+## coverage by total uncertainty
+
+metric = 'total_coverage'
+data_list = results.keys()
+colors = ['red', 'blue']
+ymax = 1.0
+minimal_bar_size = ymax * 1.5e-3
+# read out EiV and non-EiV results for all datasets
+metric_results = [
+ (save_readout(results[data]['eiv'], metric),
+ save_readout(results[data]['noneiv'], metric))
+ for data in data_list]
+
+# create figure
+plt.figure(4)
+plt.clf()
+plt.gcf().canvas.manager.set_window_title('total_coverage')
+
+# plot bars
+for i, ([(eiv_metric_mean, eiv_metric_std),
+ (noneiv_metric_mean, noneiv_metric_std)],\
+ data) in\
+ enumerate(zip(metric_results, data_list)):
+ if eiv_metric_mean is not None:
+ assert noneiv_metric_mean is not None
+ if eiv_metric_std is not None:
+ assert noneiv_metric_std is not None
+ eiv_bar_size = max(eiv_metric_std, minimal_bar_size)
+ noneiv_bar_size = max(noneiv_metric_std, minimal_bar_size)
+ plt.plot(i+1, eiv_metric_mean, '^', color=colors[0])
+ plt.bar(i+1,
+ height = 2*eiv_bar_size,
+ width = 0.3,
+ bottom = eiv_metric_mean - eiv_bar_size,
+ color=colors[0],
+ alpha=0.5)
+ plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1])
+ plt.bar(i+1,
+ height = 2 * k *noneiv_bar_size,
+ width = 0.3,
+ bottom = noneiv_metric_mean - k* noneiv_bar_size,
+ color=colors[1],
+ alpha=0.5)
+plt.ylim(bottom=0, top=ymax)
+ax = plt.gca()
+ax.set_xticks(np.arange(1,len(data_list)+1))
+ax.set_xticklabels(data_list, rotation='vertical')
+plt.axhline(0.95,0.0,1.0,color='k', linestyle='dashed')
+plt.ylabel('coverage (total unc.)')
+plt.tight_layout()
+plt.savefig('results/figures/total_coverage_bar_plot.pdf')