diff --git a/Experiments/configurations/eiv_naval.json b/Experiments/configurations/eiv_naval.json index 8975c95eb1fbfa030d4bc018bed4da7192174a30..26fc92aeec3d9474b27bc5fe624ffdbd6fe94127 100644 --- a/Experiments/configurations/eiv_naval.json +++ b/Experiments/configurations/eiv_naval.json @@ -15,7 +15,7 @@ "init_std_y_list": [0.5], "gamma": 0.5, "hidden_layers": [1024, 1024, 1024, 1024], - "fixed_std_x": 0.05, + "fixed_std_x": 0.025, "seed_range": [0,10], "gpu_number": 1 } diff --git a/Experiments/create_tabular.py b/Experiments/create_tabular.py index a83dcb94ab6484b853dd4487e710a13543a27170..cd68380681bc53a858d5e55cf21e2f5a989d6dfe 100644 --- a/Experiments/create_tabular.py +++ b/Experiments/create_tabular.py @@ -2,7 +2,8 @@ import os import glob import json -metrics_to_display = ['rmse','true_coverage_numerical', 'total_coverage'] +metrics_to_display = ['rmse', 'coverage_numerical', + 'true_coverage_numerical', 'total_coverage'] show_incomplete = True list_of_result_files = glob.glob(os.path.join('results','*.json')) diff --git a/Experiments/plot_prediction.py b/Experiments/plot_prediction.py index a603d78742a38971ffbac057919f034ad2c4b906..f1f985ffbbcb4b772e75a3bbb10942e31fdd6b8c 100644 --- a/Experiments/plot_prediction.py +++ b/Experiments/plot_prediction.py @@ -11,11 +11,20 @@ import json import torch import numpy as np +import matplotlib import matplotlib.pyplot as plt from EIVArchitectures import Networks from EIVTrainingRoutines import train_and_store + +font = {'family' : 'DejaVu Sans', + 'weight' : 'normal', + 'size' : 16} + +matplotlib.rc('font', **font) +linewidth = 2.0 + # coverage factor k = 1.96 @@ -107,6 +116,7 @@ def compute_predictions_and_uncertainties(data, x_range, eiv, number_of_draws, return_ground_truth=False, return_normalized_func=True, normalize=normalize) + plotting_dictionary['func'] = normalized_func input_dim = test_data[0][0].numel() output_dim = test_data[0][1].numel() assert output_dim == 1 @@ -223,8 +233,19 @@ list_x_range = [torch.linspace(-1.0,1.0, 50), torch.linspace(-0.2,0.8, 50)] list_color = [('red','blue')] * len(data_list) list_number_of_draws = [((100,5), 100)] * len(data_list) -for i, (data, x_range, color, number_of_draws) in enumerate(zip(data_list, - list_x_range, list_color, list_number_of_draws)): + +# create an extra zoom plot for zoom_example +zoom_example = 'linear' +# where to zoom +zoom_point = 14 +# size of the zoom plot +x_zoom_radius = 0.4 +y_zoom_radius = x_zoom_radius + +fignum = 0 +for data, x_range, color, number_of_draws in zip(data_list, + list_x_range, list_color, list_number_of_draws): + fignum += 1 eiv_plotting_dictionary = compute_predictions_and_uncertainties( data=data, x_range=x_range, @@ -237,25 +258,62 @@ for i, (data, x_range, color, number_of_draws) in enumerate(zip(data_list, number_of_draws=number_of_draws[1]) input_dim = eiv_plotting_dictionary['input_dim'] if input_dim == 1: - plt.figure(i+1) + plt.figure(fignum) plt.clf() x_values, y_values = eiv_plotting_dictionary['range_points'] - plt.plot(x_values.flatten(), y_values.flatten(),'-', color='k') + noisy_x_values, _ = eiv_plotting_dictionary['noisy_range_points'] + plt.plot(x_values.flatten(), y_values.flatten(),'-', color='k', linewidth=linewidth) eiv_pred = eiv_plotting_dictionary['prediction'] eiv_unc = eiv_plotting_dictionary['uncertainty'] plt.plot(x_values, eiv_pred,'-', - color=color[0]) + color=color[0], linewidth=linewidth) plt.fill_between(x_values.flatten(), eiv_pred-k * eiv_unc, eiv_pred + k * eiv_unc, color=color[0], alpha=0.5) noneiv_pred = noneiv_plotting_dictionary['prediction'] noneiv_unc = noneiv_plotting_dictionary['uncertainty'] plt.plot(x_values.flatten(), noneiv_pred,'-', - color=color[1]) + color=color[1], linewidth=linewidth) plt.fill_between(x_values.flatten(), noneiv_pred-k * noneiv_unc, noneiv_pred + k * noneiv_unc, color=color[1], alpha=0.5) + plt.tight_layout() plt.savefig(f'results/figures/prediction_{data}.pdf') + if data == zoom_example: + fignum += 1 + func = eiv_plotting_dictionary['func'] + x_point = x_values[zoom_point] + y_point = func(torch.tensor(x_point)).numpy() + noisy_x_point = noisy_x_values[zoom_point] + func_noisy_x_point = func(torch.tensor(noisy_x_point)).numpy() + plt.figure(fignum) + plt.clf() + plt.plot(x_values.flatten(), y_values.flatten(),'-', color='k', linewidth=linewidth) + plt.plot(x_values, eiv_pred,'-', + color=color[0], linewidth=linewidth) + plt.fill_between(x_values.flatten(), eiv_pred-k * eiv_unc, + eiv_pred + k * eiv_unc, + color=color[0], alpha=0.5) + plt.plot(x_values.flatten(), noneiv_pred,'-', + color=color[1], linewidth=linewidth) + plt.fill_between(x_values.flatten(), noneiv_pred-k * noneiv_unc, + noneiv_pred + k * noneiv_unc, + color=color[1], alpha=0.5) + plt.axvline(x_point, color='black', linestyle='dotted') + plt.axhline(y_point, color='black', linestyle='dotted') + plt.axvline(noisy_x_point, color='gray', linestyle='dashed') + plt.axhline(func_noisy_x_point, color='gray', linestyle='dashed') + plt.text(x_point - 0.1 * x_zoom_radius,y_point-0.9 * y_zoom_radius, r'$\zeta$', color='k') + plt.text(noisy_x_point - 0.1 * x_zoom_radius,y_point-0.9 * y_zoom_radius, r'$x$', color='gray') + plt.text(x_point - 0.92 * x_zoom_radius,y_point-0.13 * y_zoom_radius, r'$g(\zeta)$', color='k') + plt.text(x_point - 0.92 * x_zoom_radius,func_noisy_x_point-0.13 * y_zoom_radius, r'$g(x)$', color='gray') + plt.gca().set_xlim(left=x_point - x_zoom_radius, + right=x_point + x_zoom_radius) + plt.gca().set_ylim(bottom=y_point - y_zoom_radius, + top=y_point + y_zoom_radius) + plt.gca().set_aspect('equal', adjustable='box') + plt.tight_layout() + plt.savefig(f'results/figures/prediction_{data}_zoom.pdf') else: # multidimensional handling not included yet pass diff --git a/Experiments/plot_summary.py b/Experiments/plot_summary.py index c0863c604ff618df5e45351576c493cece331f50..4c7780a3463f3605694024933c5e845a8811148a 100644 --- a/Experiments/plot_summary.py +++ b/Experiments/plot_summary.py @@ -10,10 +10,17 @@ import glob import json import numpy as np +import matplotlib import matplotlib.pyplot as plt +font = {'family' : 'DejaVu Sans', + 'weight' : 'normal', + 'size' : 16} + +matplotlib.rc('font', **font) k = 2 + # load in all available result files list_of_result_files = glob.glob(os.path.join('results','*.json')) results = {} @@ -46,6 +53,7 @@ metric = 'rmse' data_list = results.keys() colors = ['red', 'blue'] ymax = 0.8 +minimal_bar_size = ymax * 1.5e-3 # read out EiV and non-EiV results for all datasets metric_results = [ (save_readout(results[data]['eiv'], metric), @@ -55,7 +63,7 @@ metric_results = [ # create figure plt.figure(1) plt.clf() -plt.title('RMSE') +plt.gcf().canvas.manager.set_window_title('RMSE') # plot bars for i, ([(eiv_metric_mean, eiv_metric_std), @@ -66,24 +74,28 @@ for i, ([(eiv_metric_mean, eiv_metric_std), assert noneiv_metric_mean is not None if eiv_metric_std is not None: assert noneiv_metric_std is not None + eiv_bar_size = max(eiv_metric_std, minimal_bar_size) + noneiv_bar_size = max(noneiv_metric_std, minimal_bar_size) plt.plot(i+1, eiv_metric_mean, '^', color=colors[0]) plt.bar(i+1, - height = 2*eiv_metric_std, - width = 0.1, - bottom = eiv_metric_mean - eiv_metric_std, + height = 2*eiv_bar_size, + width = 0.3, + bottom = eiv_metric_mean - eiv_bar_size, color=colors[0], alpha=0.5) plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1]) plt.bar(i+1, - height = 2 * k *noneiv_metric_std, - width = 0.1, - bottom = noneiv_metric_mean - k* noneiv_metric_std, + height = 2 * k *noneiv_bar_size, + width = 0.3, + bottom = noneiv_metric_mean - k* noneiv_bar_size, color=colors[1], alpha=0.5) plt.ylim(bottom=0, top=ymax) ax = plt.gca() ax.set_xticks(np.arange(1,len(data_list)+1)) ax.set_xticklabels(data_list, rotation='vertical') +plt.ylabel('RMSE') +plt.tight_layout() plt.savefig('results/figures/RMSE_bar_plot.pdf') ## coverage plot @@ -92,6 +104,7 @@ metric = 'true_coverage_numerical' data_list = ['linear','quadratic','cubic','sine'] colors = ['red', 'blue'] ymax = 1.0 +minimal_bar_size = ymax * 1.5e-3 # read out EiV and non-EiV results for all datasets metric_results = [ (save_readout(results[data]['eiv'], metric), @@ -101,7 +114,7 @@ metric_results = [ # create figure plt.figure(2) plt.clf() -plt.title('coverage (ground truth)') +plt.gcf().canvas.manager.set_window_title('coverage (ground truth)') # plot bars for i, ([(eiv_metric_mean, eiv_metric_std), @@ -112,18 +125,20 @@ for i, ([(eiv_metric_mean, eiv_metric_std), assert noneiv_metric_mean is not None if eiv_metric_std is not None: assert noneiv_metric_std is not None + eiv_bar_size = max(eiv_metric_std, minimal_bar_size) + noneiv_bar_size = max(noneiv_metric_std, minimal_bar_size) plt.plot(i+1, eiv_metric_mean, '^', color=colors[0]) plt.bar(i+1, - height = 2*eiv_metric_std, + height = 2*eiv_bar_size, width = 0.1, - bottom = eiv_metric_mean - eiv_metric_std, + bottom = eiv_metric_mean - eiv_bar_size, color=colors[0], alpha=0.5) plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1]) plt.bar(i+1, - height = 2 * k *noneiv_metric_std, + height = 2 * k *noneiv_bar_size, width = 0.1, - bottom = noneiv_metric_mean - k* noneiv_metric_std, + bottom = noneiv_metric_mean - k* noneiv_bar_size, color=colors[1], alpha=0.5) plt.axhline(0.95,0.0,1.0,color='k', linestyle='dashed') @@ -131,6 +146,8 @@ plt.ylim(bottom=0, top=ymax) ax = plt.gca() ax.set_xticks(np.arange(1,len(data_list)+1)) ax.set_xticklabels(data_list, rotation='vertical') +plt.ylabel('coverage ground truth') +plt.tight_layout() plt.savefig('results/figures/true_coverage_bar_plot.pdf') ## noisy coverage plot @@ -139,6 +156,7 @@ metric = 'coverage_numerical' data_list = results.keys() colors = ['red', 'blue'] ymax = 1.0 +minimal_bar_size = ymax * 1.5e-3 # read out EiV and non-EiV results for all datasets metric_results = [ (save_readout(results[data]['eiv'], metric), @@ -148,7 +166,7 @@ metric_results = [ # create figure plt.figure(3) plt.clf() -plt.title('coverage (noisy labels)') +plt.gcf().canvas.manager.set_window_title('coverage (noisy labels)') # plot bars for i, ([(eiv_metric_mean, eiv_metric_std), @@ -159,22 +177,78 @@ for i, ([(eiv_metric_mean, eiv_metric_std), assert noneiv_metric_mean is not None if eiv_metric_std is not None: assert noneiv_metric_std is not None + eiv_bar_size = max(eiv_metric_std, minimal_bar_size) + noneiv_bar_size = max(noneiv_metric_std, minimal_bar_size) plt.plot(i+1, eiv_metric_mean, '^', color=colors[0]) plt.bar(i+1, - height = 2*eiv_metric_std, + height = 2*eiv_bar_size, width = 0.1, - bottom = eiv_metric_mean - eiv_metric_std, + bottom = eiv_metric_mean - eiv_bar_size, color=colors[0], alpha=0.5) plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1]) plt.bar(i+1, - height = 2 * k *noneiv_metric_std, + height = 2 * k *noneiv_bar_size, width = 0.1, - bottom = noneiv_metric_mean - k* noneiv_metric_std, + bottom = noneiv_metric_mean - k* noneiv_bar_size, color=colors[1], alpha=0.5) plt.ylim(bottom=0, top=ymax) ax = plt.gca() ax.set_xticks(np.arange(1,len(data_list)+1)) ax.set_xticklabels(data_list, rotation='vertical') +plt.ylabel('coverage (epis. unc. / ' + r'$u(x)$' + ')' ) +plt.tight_layout() plt.savefig('results/figures/noisy_coverage_bar_plot.pdf') + +## coverage by total uncertainty + +metric = 'total_coverage' +data_list = results.keys() +colors = ['red', 'blue'] +ymax = 1.0 +minimal_bar_size = ymax * 1.5e-3 +# read out EiV and non-EiV results for all datasets +metric_results = [ + (save_readout(results[data]['eiv'], metric), + save_readout(results[data]['noneiv'], metric)) + for data in data_list] + +# create figure +plt.figure(4) +plt.clf() +plt.gcf().canvas.manager.set_window_title('total_coverage') + +# plot bars +for i, ([(eiv_metric_mean, eiv_metric_std), + (noneiv_metric_mean, noneiv_metric_std)],\ + data) in\ + enumerate(zip(metric_results, data_list)): + if eiv_metric_mean is not None: + assert noneiv_metric_mean is not None + if eiv_metric_std is not None: + assert noneiv_metric_std is not None + eiv_bar_size = max(eiv_metric_std, minimal_bar_size) + noneiv_bar_size = max(noneiv_metric_std, minimal_bar_size) + plt.plot(i+1, eiv_metric_mean, '^', color=colors[0]) + plt.bar(i+1, + height = 2*eiv_bar_size, + width = 0.3, + bottom = eiv_metric_mean - eiv_bar_size, + color=colors[0], + alpha=0.5) + plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1]) + plt.bar(i+1, + height = 2 * k *noneiv_bar_size, + width = 0.3, + bottom = noneiv_metric_mean - k* noneiv_bar_size, + color=colors[1], + alpha=0.5) +plt.ylim(bottom=0, top=ymax) +ax = plt.gca() +ax.set_xticks(np.arange(1,len(data_list)+1)) +ax.set_xticklabels(data_list, rotation='vertical') +plt.axhline(0.95,0.0,1.0,color='k', linestyle='dashed') +plt.ylabel('coverage (total unc.)') +plt.tight_layout() +plt.savefig('results/figures/total_coverage_bar_plot.pdf')