From d88c989f6470e1cf0f7e16eb0396b20155664075 Mon Sep 17 00:00:00 2001 From: Joerg Martin <joerg.martin@ptb.de> Date: Mon, 7 Feb 2022 10:27:19 +0000 Subject: [PATCH] plot_summary added --- Experiments/plot_summary.py | 169 +++++++++++++++++++++++++++++++++++- 1 file changed, 168 insertions(+), 1 deletion(-) diff --git a/Experiments/plot_summary.py b/Experiments/plot_summary.py index 953ec8d..c3d8c0d 100644 --- a/Experiments/plot_summary.py +++ b/Experiments/plot_summary.py @@ -9,5 +9,172 @@ import os import glob import json +import numpy as np +import matplotlib.pyplot as plt -## include evaluate_metrics content here and adapt +k = 2 + +# load in all available result files +list_of_result_files = glob.glob(os.path.join('results','*.json')) +results = {} +for filename in list_of_result_files: + data = filename.replace(os.path.join('results','metrics_'),'').replace('.json','') + with open(filename,'r') as f: + results[data] = json.load(f) + +def save_readout(dictionary, key): + """ + Returns the value of the `dictionary` for `key`, unless + the later doesn't exist, in which case (None,None) is returned. + """ + try: + readout = dictionary[key] + if type(readout) is list: + assert len(readout) == 2 + return readout + else: + readout = float(readout) + return (readout, None) + except KeyError: + return (None,None) + + + +## RMSE plot + +metric = 'rmse' +data_list = results.keys() +colors = ['red', 'blue'] +ymax = 0.8 +# read out EiV and non-EiV results for all datasets +metric_results = [ + (save_readout(results[data]['eiv'], metric), + save_readout(results[data]['noneiv'], metric)) + for data in data_list] + +# create figure +plt.figure(1) +plt.clf() +plt.title('RMSE') + +# plot bars +for i, ([(eiv_metric_mean, eiv_metric_std), + (noneiv_metric_mean, noneiv_metric_std)],\ + data) in\ + enumerate(zip(metric_results, data_list)): + if eiv_metric_mean is not None: + assert noneiv_metric_mean is not None + if eiv_metric_std is not None: + assert noneiv_metric_std is not None + plt.plot(i+1, eiv_metric_mean, '^', color=colors[0]) + plt.bar(i+1, + height = 2*eiv_metric_std, + width = 0.1, + bottom = eiv_metric_mean - eiv_metric_std, + color=colors[0], + alpha=0.5) + plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1]) + plt.bar(i+1, + height = 2 * k *noneiv_metric_std, + width = 0.1, + bottom = noneiv_metric_mean - k* noneiv_metric_std, + color=colors[1], + alpha=0.5) +plt.ylim(bottom=0, top=y_max) +ax = plt.gca() +ax.set_xticks(np.arange(1,len(data_list)+1)) +ax.set_xticklabels(data_list, rotation='vertical') +plt.savefig('results/figures/RMSE_bar_plot.pdf') + +## coverage plot + +metric = 'true_coverage_numerical' +data_list = ['linear','quadratic','cubic','sine'] +colors = ['red', 'blue'] +ymax = 1.0 +# read out EiV and non-EiV results for all datasets +metric_results = [ + (save_readout(results[data]['eiv'], metric), + save_readout(results[data]['noneiv'], metric)) + for data in data_list] + +# create figure +plt.figure(2) +plt.clf() +plt.title('coverage (ground truth)') + +# plot bars +for i, ([(eiv_metric_mean, eiv_metric_std), + (noneiv_metric_mean, noneiv_metric_std)],\ + data) in\ + enumerate(zip(metric_results, data_list)): + if eiv_metric_mean is not None: + assert noneiv_metric_mean is not None + if eiv_metric_std is not None: + assert noneiv_metric_std is not None + plt.plot(i+1, eiv_metric_mean, '^', color=colors[0]) + plt.bar(i+1, + height = 2*eiv_metric_std, + width = 0.1, + bottom = eiv_metric_mean - eiv_metric_std, + color=colors[0], + alpha=0.5) + plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1]) + plt.bar(i+1, + height = 2 * k *noneiv_metric_std, + width = 0.1, + bottom = noneiv_metric_mean - k* noneiv_metric_std, + color=colors[1], + alpha=0.5) +plt.axhline(0.95,0.0,1.0,color='k', linestyle='dashed') +plt.ylim(bottom=0, top=y_max) +ax = plt.gca() +ax.set_xticks(np.arange(1,len(data_list)+1)) +ax.set_xticklabels(data_list, rotation='vertical') +plt.savefig('results/figures/true_coverage_bar_plot.pdf') + +## noisy coverage plot + +metric = 'coverage_numerical' +data_list = results.keys() +colors = ['red', 'blue'] +ymax = 1.0 +# read out EiV and non-EiV results for all datasets +metric_results = [ + (save_readout(results[data]['eiv'], metric), + save_readout(results[data]['noneiv'], metric)) + for data in data_list] + +# create figure +plt.figure(3) +plt.clf() +plt.title('coverage (noisy labels)') + +# plot bars +for i, ([(eiv_metric_mean, eiv_metric_std), + (noneiv_metric_mean, noneiv_metric_std)],\ + data) in\ + enumerate(zip(metric_results, data_list)): + if eiv_metric_mean is not None: + assert noneiv_metric_mean is not None + if eiv_metric_std is not None: + assert noneiv_metric_std is not None + plt.plot(i+1, eiv_metric_mean, '^', color=colors[0]) + plt.bar(i+1, + height = 2*eiv_metric_std, + width = 0.1, + bottom = eiv_metric_mean - eiv_metric_std, + color=colors[0], + alpha=0.5) + plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1]) + plt.bar(i+1, + height = 2 * k *noneiv_metric_std, + width = 0.1, + bottom = noneiv_metric_mean - k* noneiv_metric_std, + color=colors[1], + alpha=0.5) +plt.ylim(bottom=0, top=y_max) +ax = plt.gca() +ax.set_xticks(np.arange(1,len(data_list)+1)) +ax.set_xticklabels(data_list, rotation='vertical') +plt.savefig('results/figures/noisy_coverage_bar_plot.pdf') -- GitLab