plot_summary added

Experiments/plot_summary.py

@@ -9,5 +9,172 @@ import os
 import glob
 import json
 
+import numpy as np
+import matplotlib.pyplot as plt
 
-## include evaluate_metrics content here and adapt
+k = 2
+
+# load in all available result files
+list_of_result_files = glob.glob(os.path.join('results','*.json'))
+results = {}
+for filename in list_of_result_files:
+    data = filename.replace(os.path.join('results','metrics_'),'').replace('.json','')
+    with open(filename,'r') as f:
+       results[data] = json.load(f)
+
+def save_readout(dictionary, key):
+    """
+    Returns the value of the `dictionary` for `key`, unless
+    the later doesn't exist, in which case (None,None) is returned.
+    """
+    try:
+        readout = dictionary[key]
+        if type(readout) is list:
+            assert len(readout) == 2
+            return readout
+        else:
+            readout = float(readout)
+            return (readout, None)
+    except KeyError:
+        return (None,None)
+
+
+
+## RMSE plot
+
+metric = 'rmse'
+data_list = results.keys()
+colors = ['red', 'blue']
+ymax = 0.8
+# read out EiV and non-EiV results for all datasets
+metric_results = [
+        (save_readout(results[data]['eiv'], metric),
+        save_readout(results[data]['noneiv'], metric))
+            for data in data_list]
+
+# create figure
+plt.figure(1)
+plt.clf()
+plt.title('RMSE')
+
+# plot bars
+for i, ([(eiv_metric_mean, eiv_metric_std),
+        (noneiv_metric_mean, noneiv_metric_std)],\
+                data) in\
+                enumerate(zip(metric_results, data_list)):
+    if eiv_metric_mean is not None:
+        assert noneiv_metric_mean is not None
+        if eiv_metric_std is not None:
+            assert noneiv_metric_std is not None
+            plt.plot(i+1, eiv_metric_mean, '^', color=colors[0])
+            plt.bar(i+1,
+                    height = 2*eiv_metric_std,
+                    width = 0.1,
+                    bottom = eiv_metric_mean - eiv_metric_std,
+                    color=colors[0],
+                    alpha=0.5)
+            plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1])
+            plt.bar(i+1,
+                    height = 2 * k *noneiv_metric_std,
+                    width = 0.1,
+                    bottom = noneiv_metric_mean - k*  noneiv_metric_std,
+                    color=colors[1],
+                    alpha=0.5)
+plt.ylim(bottom=0, top=y_max)
+ax = plt.gca()
+ax.set_xticks(np.arange(1,len(data_list)+1))
+ax.set_xticklabels(data_list, rotation='vertical')
+plt.savefig('results/figures/RMSE_bar_plot.pdf')
+
+## coverage plot
+
+metric = 'true_coverage_numerical'
+data_list = ['linear','quadratic','cubic','sine']
+colors = ['red', 'blue']
+ymax = 1.0
+# read out EiV and non-EiV results for all datasets
+metric_results = [
+        (save_readout(results[data]['eiv'], metric),
+        save_readout(results[data]['noneiv'], metric))
+            for data in data_list]
+
+# create figure
+plt.figure(2)
+plt.clf()
+plt.title('coverage (ground truth)')
+
+# plot bars
+for i, ([(eiv_metric_mean, eiv_metric_std),
+        (noneiv_metric_mean, noneiv_metric_std)],\
+                data) in\
+                enumerate(zip(metric_results, data_list)):
+    if eiv_metric_mean is not None:
+        assert noneiv_metric_mean is not None
+        if eiv_metric_std is not None:
+            assert noneiv_metric_std is not None
+            plt.plot(i+1, eiv_metric_mean, '^', color=colors[0])
+            plt.bar(i+1,
+                    height = 2*eiv_metric_std,
+                    width = 0.1,
+                    bottom = eiv_metric_mean - eiv_metric_std,
+                    color=colors[0],
+                    alpha=0.5)
+            plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1])
+            plt.bar(i+1,
+                    height = 2 * k *noneiv_metric_std,
+                    width = 0.1,
+                    bottom = noneiv_metric_mean - k*  noneiv_metric_std,
+                    color=colors[1],
+                    alpha=0.5)
+plt.axhline(0.95,0.0,1.0,color='k', linestyle='dashed')
+plt.ylim(bottom=0, top=y_max)
+ax = plt.gca()
+ax.set_xticks(np.arange(1,len(data_list)+1))
+ax.set_xticklabels(data_list, rotation='vertical')
+plt.savefig('results/figures/true_coverage_bar_plot.pdf')
+
+## noisy coverage plot
+
+metric = 'coverage_numerical'
+data_list = results.keys()
+colors = ['red', 'blue']
+ymax = 1.0
+# read out EiV and non-EiV results for all datasets
+metric_results = [
+        (save_readout(results[data]['eiv'], metric),
+        save_readout(results[data]['noneiv'], metric))
+            for data in data_list]
+
+# create figure
+plt.figure(3)
+plt.clf()
+plt.title('coverage (noisy labels)')
+
+# plot bars
+for i, ([(eiv_metric_mean, eiv_metric_std),
+        (noneiv_metric_mean, noneiv_metric_std)],\
+                data) in\
+                enumerate(zip(metric_results, data_list)):
+    if eiv_metric_mean is not None:
+        assert noneiv_metric_mean is not None
+        if eiv_metric_std is not None:
+            assert noneiv_metric_std is not None
+            plt.plot(i+1, eiv_metric_mean, '^', color=colors[0])
+            plt.bar(i+1,
+                    height = 2*eiv_metric_std,
+                    width = 0.1,
+                    bottom = eiv_metric_mean - eiv_metric_std,
+                    color=colors[0],
+                    alpha=0.5)
+            plt.plot(i+1, noneiv_metric_mean, '^', color=colors[1])
+            plt.bar(i+1,
+                    height = 2 * k *noneiv_metric_std,
+                    width = 0.1,
+                    bottom = noneiv_metric_mean - k*  noneiv_metric_std,
+                    color=colors[1],
+                    alpha=0.5)
+plt.ylim(bottom=0, top=y_max)
+ax = plt.gca()
+ax.set_xticks(np.arange(1,len(data_list)+1))
+ax.set_xticklabels(data_list, rotation='vertical')
+plt.savefig('results/figures/noisy_coverage_bar_plot.pdf')