From d88c989f6470e1cf0f7e16eb0396b20155664075 Mon Sep 17 00:00:00 2001
From: Joerg Martin <joerg.martin@ptb.de>
Date: Mon, 7 Feb 2022 10:27:19 +0000
Subject: [PATCH] plot_summary added
---
Experiments/plot_summary.py | 169 +++++++++++++++++++++++++++++++++++-
1 file changed, 168 insertions(+), 1 deletion(-)
diff --git a/Experiments/plot_summary.py b/Experiments/plot_summary.py
index 953ec8d..c3d8c0d 100644
--- a/Experiments/plot_summary.py
+++ b/Experiments/plot_summary.py
@@ -9,5 +9,172 @@ import os
import glob
import json
+import numpy as np
+import matplotlib.pyplot as plt
-## include evaluate_metrics content here and adapt
+k = 2
+
+# load in all available result files
+list_of_result_files = glob.glob(os.path.join('results','*.json'))
+results = {}
+for filename in list_of_result_files:
+ data = filename.replace(os.path.join('results','metrics_'),'').replace('.json','')
+ with open(filename,'r') as f:
+ results[data] = json.load(f)
+
+def save_readout(dictionary, key):
+ """
+ Returns the value of the `dictionary` for `key`, unless
+ the later doesn't exist, in which case (None,None) is returned.
+ """
+ try:
+ readout = dictionary[key]
+ if type(readout) is list:
+ assert len(readout) == 2
+ return readout
+ else:
+ readout = float(readout)
+ return (readout, None)
+ except KeyError:
+ return (None,None)
+
+
+
+## RMSE plot
+
+metric = 'rmse'
+data_list = results.keys()
+colors = ['red', 'blue']
+ymax = 0.8
+# read out EiV and non-EiV results for all datasets
+metric_results = [
+ (save_readout(results[data]['eiv'], metric),
+ save_readout(results[data]['noneiv'], metric))
+ for data in data_list]
+
+# create figure
+plt.figure(1)
+plt.clf()
+plt.title('RMSE')
+
+# plot bars
+for i, ([(eiv_metric_mean, eiv_metric_std),
+ (noneiv_metric_mean, noneiv_metric_std)],\
+ data) in\
+ enumerate(zip(metric_results, data_list)):
+ if eiv_metric_mean is not None:
+ assert noneiv_metric_mean is not None
+ if eiv_metric_std is not None:
+ assert noneiv_metric_std is not None
+ plt.plot(i+1, eiv_metric_mean, '^', color=colors[0])
+ plt.bar(i+1,
+ height = 2*eiv_metric_std,
+ width = 0.1,
+ bottom = eiv_metric_mean - eiv_metric_std,
+ color=colors[0],
+ alpha=0.5)
+ plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1])
+ plt.bar(i+1,
+ height = 2 * k *noneiv_metric_std,
+ width = 0.1,
+ bottom = noneiv_metric_mean - k* noneiv_metric_std,
+ color=colors[1],
+ alpha=0.5)
+plt.ylim(bottom=0, top=y_max)
+ax = plt.gca()
+ax.set_xticks(np.arange(1,len(data_list)+1))
+ax.set_xticklabels(data_list, rotation='vertical')
+plt.savefig('results/figures/RMSE_bar_plot.pdf')
+
+## coverage plot
+
+metric = 'true_coverage_numerical'
+data_list = ['linear','quadratic','cubic','sine']
+colors = ['red', 'blue']
+ymax = 1.0
+# read out EiV and non-EiV results for all datasets
+metric_results = [
+ (save_readout(results[data]['eiv'], metric),
+ save_readout(results[data]['noneiv'], metric))
+ for data in data_list]
+
+# create figure
+plt.figure(2)
+plt.clf()
+plt.title('coverage (ground truth)')
+
+# plot bars
+for i, ([(eiv_metric_mean, eiv_metric_std),
+ (noneiv_metric_mean, noneiv_metric_std)],\
+ data) in\
+ enumerate(zip(metric_results, data_list)):
+ if eiv_metric_mean is not None:
+ assert noneiv_metric_mean is not None
+ if eiv_metric_std is not None:
+ assert noneiv_metric_std is not None
+ plt.plot(i+1, eiv_metric_mean, '^', color=colors[0])
+ plt.bar(i+1,
+ height = 2*eiv_metric_std,
+ width = 0.1,
+ bottom = eiv_metric_mean - eiv_metric_std,
+ color=colors[0],
+ alpha=0.5)
+ plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1])
+ plt.bar(i+1,
+ height = 2 * k *noneiv_metric_std,
+ width = 0.1,
+ bottom = noneiv_metric_mean - k* noneiv_metric_std,
+ color=colors[1],
+ alpha=0.5)
+plt.axhline(0.95,0.0,1.0,color='k', linestyle='dashed')
+plt.ylim(bottom=0, top=y_max)
+ax = plt.gca()
+ax.set_xticks(np.arange(1,len(data_list)+1))
+ax.set_xticklabels(data_list, rotation='vertical')
+plt.savefig('results/figures/true_coverage_bar_plot.pdf')
+
+## noisy coverage plot
+
+metric = 'coverage_numerical'
+data_list = results.keys()
+colors = ['red', 'blue']
+ymax = 1.0
+# read out EiV and non-EiV results for all datasets
+metric_results = [
+ (save_readout(results[data]['eiv'], metric),
+ save_readout(results[data]['noneiv'], metric))
+ for data in data_list]
+
+# create figure
+plt.figure(3)
+plt.clf()
+plt.title('coverage (noisy labels)')
+
+# plot bars
+for i, ([(eiv_metric_mean, eiv_metric_std),
+ (noneiv_metric_mean, noneiv_metric_std)],\
+ data) in\
+ enumerate(zip(metric_results, data_list)):
+ if eiv_metric_mean is not None:
+ assert noneiv_metric_mean is not None
+ if eiv_metric_std is not None:
+ assert noneiv_metric_std is not None
+ plt.plot(i+1, eiv_metric_mean, '^', color=colors[0])
+ plt.bar(i+1,
+ height = 2*eiv_metric_std,
+ width = 0.1,
+ bottom = eiv_metric_mean - eiv_metric_std,
+ color=colors[0],
+ alpha=0.5)
+ plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1])
+ plt.bar(i+1,
+ height = 2 * k *noneiv_metric_std,
+ width = 0.1,
+ bottom = noneiv_metric_mean - k* noneiv_metric_std,
+ color=colors[1],
+ alpha=0.5)
+plt.ylim(bottom=0, top=y_max)
+ax = plt.gca()
+ax.set_xticks(np.arange(1,len(data_list)+1))
+ax.set_xticklabels(data_list, rotation='vertical')
+plt.savefig('results/figures/noisy_coverage_bar_plot.pdf')
--
GitLab