diff --git a/Experiments/configurations/eiv_naval.json b/Experiments/configurations/eiv_naval.json
index 8975c95eb1fbfa030d4bc018bed4da7192174a30..26fc92aeec3d9474b27bc5fe624ffdbd6fe94127 100644
--- a/Experiments/configurations/eiv_naval.json
+++ b/Experiments/configurations/eiv_naval.json
@@ -15,7 +15,7 @@
"init_std_y_list": [0.5],
"gamma": 0.5,
"hidden_layers": [1024, 1024, 1024, 1024],
- "fixed_std_x": 0.05,
+ "fixed_std_x": 0.025,
"seed_range": [0,10],
"gpu_number": 1
}
diff --git a/Experiments/create_tabular.py b/Experiments/create_tabular.py
index a83dcb94ab6484b853dd4487e710a13543a27170..cd68380681bc53a858d5e55cf21e2f5a989d6dfe 100644
--- a/Experiments/create_tabular.py
+++ b/Experiments/create_tabular.py
@@ -2,7 +2,8 @@ import os
import glob
import json
-metrics_to_display = ['rmse','true_coverage_numerical', 'total_coverage']
+metrics_to_display = ['rmse', 'coverage_numerical',
+ 'true_coverage_numerical', 'total_coverage']
show_incomplete = True
list_of_result_files = glob.glob(os.path.join('results','*.json'))
diff --git a/Experiments/plot_prediction.py b/Experiments/plot_prediction.py
index a603d78742a38971ffbac057919f034ad2c4b906..f1f985ffbbcb4b772e75a3bbb10942e31fdd6b8c 100644
--- a/Experiments/plot_prediction.py
+++ b/Experiments/plot_prediction.py
@@ -11,11 +11,20 @@ import json
import torch
import numpy as np
+import matplotlib
import matplotlib.pyplot as plt
from EIVArchitectures import Networks
from EIVTrainingRoutines import train_and_store
+
+font = {'family' : 'DejaVu Sans',
+ 'weight' : 'normal',
+ 'size' : 16}
+
+matplotlib.rc('font', **font)
+linewidth = 2.0
+
# coverage factor
k = 1.96
@@ -107,6 +116,7 @@ def compute_predictions_and_uncertainties(data, x_range, eiv, number_of_draws,
return_ground_truth=False,
return_normalized_func=True,
normalize=normalize)
+ plotting_dictionary['func'] = normalized_func
input_dim = test_data[0][0].numel()
output_dim = test_data[0][1].numel()
assert output_dim == 1
@@ -223,8 +233,19 @@ list_x_range = [torch.linspace(-1.0,1.0, 50),
torch.linspace(-0.2,0.8, 50)]
list_color = [('red','blue')] * len(data_list)
list_number_of_draws = [((100,5), 100)] * len(data_list)
-for i, (data, x_range, color, number_of_draws) in enumerate(zip(data_list,
- list_x_range, list_color, list_number_of_draws)):
+
+# create an extra zoom plot for zoom_example
+zoom_example = 'linear'
+# where to zoom
+zoom_point = 14
+# size of the zoom plot
+x_zoom_radius = 0.4
+y_zoom_radius = x_zoom_radius
+
+fignum = 0
+for data, x_range, color, number_of_draws in zip(data_list,
+ list_x_range, list_color, list_number_of_draws):
+ fignum += 1
eiv_plotting_dictionary = compute_predictions_and_uncertainties(
data=data,
x_range=x_range,
@@ -237,25 +258,62 @@ for i, (data, x_range, color, number_of_draws) in enumerate(zip(data_list,
number_of_draws=number_of_draws[1])
input_dim = eiv_plotting_dictionary['input_dim']
if input_dim == 1:
- plt.figure(i+1)
+ plt.figure(fignum)
plt.clf()
x_values, y_values = eiv_plotting_dictionary['range_points']
- plt.plot(x_values.flatten(), y_values.flatten(),'-', color='k')
+ noisy_x_values, _ = eiv_plotting_dictionary['noisy_range_points']
+ plt.plot(x_values.flatten(), y_values.flatten(),'-', color='k', linewidth=linewidth)
eiv_pred = eiv_plotting_dictionary['prediction']
eiv_unc = eiv_plotting_dictionary['uncertainty']
plt.plot(x_values, eiv_pred,'-',
- color=color[0])
+ color=color[0], linewidth=linewidth)
plt.fill_between(x_values.flatten(), eiv_pred-k * eiv_unc,
eiv_pred + k * eiv_unc,
color=color[0], alpha=0.5)
noneiv_pred = noneiv_plotting_dictionary['prediction']
noneiv_unc = noneiv_plotting_dictionary['uncertainty']
plt.plot(x_values.flatten(), noneiv_pred,'-',
- color=color[1])
+ color=color[1], linewidth=linewidth)
plt.fill_between(x_values.flatten(), noneiv_pred-k * noneiv_unc,
noneiv_pred + k * noneiv_unc,
color=color[1], alpha=0.5)
+ plt.tight_layout()
plt.savefig(f'results/figures/prediction_{data}.pdf')
+ if data == zoom_example:
+ fignum += 1
+ func = eiv_plotting_dictionary['func']
+ x_point = x_values[zoom_point]
+ y_point = func(torch.tensor(x_point)).numpy()
+ noisy_x_point = noisy_x_values[zoom_point]
+ func_noisy_x_point = func(torch.tensor(noisy_x_point)).numpy()
+ plt.figure(fignum)
+ plt.clf()
+ plt.plot(x_values.flatten(), y_values.flatten(),'-', color='k', linewidth=linewidth)
+ plt.plot(x_values, eiv_pred,'-',
+ color=color[0], linewidth=linewidth)
+ plt.fill_between(x_values.flatten(), eiv_pred-k * eiv_unc,
+ eiv_pred + k * eiv_unc,
+ color=color[0], alpha=0.5)
+ plt.plot(x_values.flatten(), noneiv_pred,'-',
+ color=color[1], linewidth=linewidth)
+ plt.fill_between(x_values.flatten(), noneiv_pred-k * noneiv_unc,
+ noneiv_pred + k * noneiv_unc,
+ color=color[1], alpha=0.5)
+ plt.axvline(x_point, color='black', linestyle='dotted')
+ plt.axhline(y_point, color='black', linestyle='dotted')
+ plt.axvline(noisy_x_point, color='gray', linestyle='dashed')
+ plt.axhline(func_noisy_x_point, color='gray', linestyle='dashed')
+ plt.text(x_point - 0.1 * x_zoom_radius,y_point-0.9 * y_zoom_radius, r'$\zeta$', color='k')
+ plt.text(noisy_x_point - 0.1 * x_zoom_radius,y_point-0.9 * y_zoom_radius, r'$x$', color='gray')
+ plt.text(x_point - 0.92 * x_zoom_radius,y_point-0.13 * y_zoom_radius, r'$g(\zeta)$', color='k')
+ plt.text(x_point - 0.92 * x_zoom_radius,func_noisy_x_point-0.13 * y_zoom_radius, r'$g(x)$', color='gray')
+ plt.gca().set_xlim(left=x_point - x_zoom_radius,
+ right=x_point + x_zoom_radius)
+ plt.gca().set_ylim(bottom=y_point - y_zoom_radius,
+ top=y_point + y_zoom_radius)
+ plt.gca().set_aspect('equal', adjustable='box')
+ plt.tight_layout()
+ plt.savefig(f'results/figures/prediction_{data}_zoom.pdf')
else:
# multidimensional handling not included yet
pass
diff --git a/Experiments/plot_summary.py b/Experiments/plot_summary.py
index c0863c604ff618df5e45351576c493cece331f50..4c7780a3463f3605694024933c5e845a8811148a 100644
--- a/Experiments/plot_summary.py
+++ b/Experiments/plot_summary.py
@@ -10,10 +10,17 @@ import glob
import json
import numpy as np
+import matplotlib
import matplotlib.pyplot as plt
+font = {'family' : 'DejaVu Sans',
+ 'weight' : 'normal',
+ 'size' : 16}
+
+matplotlib.rc('font', **font)
k = 2
+
# load in all available result files
list_of_result_files = glob.glob(os.path.join('results','*.json'))
results = {}
@@ -46,6 +53,7 @@ metric = 'rmse'
data_list = results.keys()
colors = ['red', 'blue']
ymax = 0.8
+minimal_bar_size = ymax * 1.5e-3
# read out EiV and non-EiV results for all datasets
metric_results = [
(save_readout(results[data]['eiv'], metric),
@@ -55,7 +63,7 @@ metric_results = [
# create figure
plt.figure(1)
plt.clf()
-plt.title('RMSE')
+plt.gcf().canvas.manager.set_window_title('RMSE')
# plot bars
for i, ([(eiv_metric_mean, eiv_metric_std),
@@ -66,24 +74,28 @@ for i, ([(eiv_metric_mean, eiv_metric_std),
assert noneiv_metric_mean is not None
if eiv_metric_std is not None:
assert noneiv_metric_std is not None
+ eiv_bar_size = max(eiv_metric_std, minimal_bar_size)
+ noneiv_bar_size = max(noneiv_metric_std, minimal_bar_size)
plt.plot(i+1, eiv_metric_mean, '^', color=colors[0])
plt.bar(i+1,
- height = 2*eiv_metric_std,
- width = 0.1,
- bottom = eiv_metric_mean - eiv_metric_std,
+ height = 2*eiv_bar_size,
+ width = 0.3,
+ bottom = eiv_metric_mean - eiv_bar_size,
color=colors[0],
alpha=0.5)
plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1])
plt.bar(i+1,
- height = 2 * k *noneiv_metric_std,
- width = 0.1,
- bottom = noneiv_metric_mean - k* noneiv_metric_std,
+ height = 2 * k *noneiv_bar_size,
+ width = 0.3,
+ bottom = noneiv_metric_mean - k* noneiv_bar_size,
color=colors[1],
alpha=0.5)
plt.ylim(bottom=0, top=ymax)
ax = plt.gca()
ax.set_xticks(np.arange(1,len(data_list)+1))
ax.set_xticklabels(data_list, rotation='vertical')
+plt.ylabel('RMSE')
+plt.tight_layout()
plt.savefig('results/figures/RMSE_bar_plot.pdf')
## coverage plot
@@ -92,6 +104,7 @@ metric = 'true_coverage_numerical'
data_list = ['linear','quadratic','cubic','sine']
colors = ['red', 'blue']
ymax = 1.0
+minimal_bar_size = ymax * 1.5e-3
# read out EiV and non-EiV results for all datasets
metric_results = [
(save_readout(results[data]['eiv'], metric),
@@ -101,7 +114,7 @@ metric_results = [
# create figure
plt.figure(2)
plt.clf()
-plt.title('coverage (ground truth)')
+plt.gcf().canvas.manager.set_window_title('coverage (ground truth)')
# plot bars
for i, ([(eiv_metric_mean, eiv_metric_std),
@@ -112,18 +125,20 @@ for i, ([(eiv_metric_mean, eiv_metric_std),
assert noneiv_metric_mean is not None
if eiv_metric_std is not None:
assert noneiv_metric_std is not None
+ eiv_bar_size = max(eiv_metric_std, minimal_bar_size)
+ noneiv_bar_size = max(noneiv_metric_std, minimal_bar_size)
plt.plot(i+1, eiv_metric_mean, '^', color=colors[0])
plt.bar(i+1,
- height = 2*eiv_metric_std,
+ height = 2*eiv_bar_size,
width = 0.1,
- bottom = eiv_metric_mean - eiv_metric_std,
+ bottom = eiv_metric_mean - eiv_bar_size,
color=colors[0],
alpha=0.5)
plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1])
plt.bar(i+1,
- height = 2 * k *noneiv_metric_std,
+ height = 2 * k *noneiv_bar_size,
width = 0.1,
- bottom = noneiv_metric_mean - k* noneiv_metric_std,
+ bottom = noneiv_metric_mean - k* noneiv_bar_size,
color=colors[1],
alpha=0.5)
plt.axhline(0.95,0.0,1.0,color='k', linestyle='dashed')
@@ -131,6 +146,8 @@ plt.ylim(bottom=0, top=ymax)
ax = plt.gca()
ax.set_xticks(np.arange(1,len(data_list)+1))
ax.set_xticklabels(data_list, rotation='vertical')
+plt.ylabel('coverage ground truth')
+plt.tight_layout()
plt.savefig('results/figures/true_coverage_bar_plot.pdf')
## noisy coverage plot
@@ -139,6 +156,7 @@ metric = 'coverage_numerical'
data_list = results.keys()
colors = ['red', 'blue']
ymax = 1.0
+minimal_bar_size = ymax * 1.5e-3
# read out EiV and non-EiV results for all datasets
metric_results = [
(save_readout(results[data]['eiv'], metric),
@@ -148,7 +166,7 @@ metric_results = [
# create figure
plt.figure(3)
plt.clf()
-plt.title('coverage (noisy labels)')
+plt.gcf().canvas.manager.set_window_title('coverage (noisy labels)')
# plot bars
for i, ([(eiv_metric_mean, eiv_metric_std),
@@ -159,22 +177,78 @@ for i, ([(eiv_metric_mean, eiv_metric_std),
assert noneiv_metric_mean is not None
if eiv_metric_std is not None:
assert noneiv_metric_std is not None
+ eiv_bar_size = max(eiv_metric_std, minimal_bar_size)
+ noneiv_bar_size = max(noneiv_metric_std, minimal_bar_size)
plt.plot(i+1, eiv_metric_mean, '^', color=colors[0])
plt.bar(i+1,
- height = 2*eiv_metric_std,
+ height = 2*eiv_bar_size,
width = 0.1,
- bottom = eiv_metric_mean - eiv_metric_std,
+ bottom = eiv_metric_mean - eiv_bar_size,
color=colors[0],
alpha=0.5)
plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1])
plt.bar(i+1,
- height = 2 * k *noneiv_metric_std,
+ height = 2 * k *noneiv_bar_size,
width = 0.1,
- bottom = noneiv_metric_mean - k* noneiv_metric_std,
+ bottom = noneiv_metric_mean - k* noneiv_bar_size,
color=colors[1],
alpha=0.5)
plt.ylim(bottom=0, top=ymax)
ax = plt.gca()
ax.set_xticks(np.arange(1,len(data_list)+1))
ax.set_xticklabels(data_list, rotation='vertical')
+plt.ylabel('coverage (epis. unc. / ' + r'$u(x)$' + ')' )
+plt.tight_layout()
plt.savefig('results/figures/noisy_coverage_bar_plot.pdf')
+
+## coverage by total uncertainty
+
+metric = 'total_coverage'
+data_list = results.keys()
+colors = ['red', 'blue']
+ymax = 1.0
+minimal_bar_size = ymax * 1.5e-3
+# read out EiV and non-EiV results for all datasets
+metric_results = [
+ (save_readout(results[data]['eiv'], metric),
+ save_readout(results[data]['noneiv'], metric))
+ for data in data_list]
+
+# create figure
+plt.figure(4)
+plt.clf()
+plt.gcf().canvas.manager.set_window_title('total_coverage')
+
+# plot bars
+for i, ([(eiv_metric_mean, eiv_metric_std),
+ (noneiv_metric_mean, noneiv_metric_std)],\
+ data) in\
+ enumerate(zip(metric_results, data_list)):
+ if eiv_metric_mean is not None:
+ assert noneiv_metric_mean is not None
+ if eiv_metric_std is not None:
+ assert noneiv_metric_std is not None
+ eiv_bar_size = max(eiv_metric_std, minimal_bar_size)
+ noneiv_bar_size = max(noneiv_metric_std, minimal_bar_size)
+ plt.plot(i+1, eiv_metric_mean, '^', color=colors[0])
+ plt.bar(i+1,
+ height = 2*eiv_bar_size,
+ width = 0.3,
+ bottom = eiv_metric_mean - eiv_bar_size,
+ color=colors[0],
+ alpha=0.5)
+ plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1])
+ plt.bar(i+1,
+ height = 2 * k *noneiv_bar_size,
+ width = 0.3,
+ bottom = noneiv_metric_mean - k* noneiv_bar_size,
+ color=colors[1],
+ alpha=0.5)
+plt.ylim(bottom=0, top=ymax)
+ax = plt.gca()
+ax.set_xticks(np.arange(1,len(data_list)+1))
+ax.set_xticklabels(data_list, rotation='vertical')
+plt.axhline(0.95,0.0,1.0,color='k', linestyle='dashed')
+plt.ylabel('coverage (total unc.)')
+plt.tight_layout()
+plt.savefig('results/figures/total_coverage_bar_plot.pdf')