Evaluating metrics for multiple datasets

The script evaluate_tabular.py was renamed to evaluate_metrics. This
script now not only prints the results, but also stores them in JSON
files in a Experiments/results folder (should be created). These files
can be read via a new script create_tabular.py
The JSON files have now all been changed to a less frequent update of
std_y.

Experiments/configurations/eiv_california.json

@@ -9,8 +9,8 @@
 	"report_point": 5,
 	"p": 0.1,
 	"lr_update": 20,
-	"std_y_update_points": [10,5],
-	"eiv_prediction_number_of_draws": 100,
+	"std_y_update_points": [1,40],
+	"eiv_prediction_number_of_draws": [100,5],
 	"eiv_prediction_number_of_batches": 10,
 	"init_std_y_list": [0.5],
 	"gamma": 0.5,

Experiments/configurations/eiv_concrete.json

@@ -9,9 +9,9 @@
 	"report_point": 5,
 	"p": 0.2,
 	"lr_update": 20,
-	"std_y_update_points": 10,
-	"eiv_prediction_number_of_draws": 100,
-	"eiv_prediction_number_of_batches": 10,
+	"std_y_update_points": [10, 40],
+	"eiv_prediction_number_of_draws": [100,5],
+	"eiv_prediction_number_of_batches": 20,
 	"init_std_y_list": [0.5],
 	"gamma": 0.5,
 	"hidden_layers": [1024, 1024, 1024, 1024],

Experiments/configurations/eiv_energy.json

@@ -9,8 +9,8 @@
 	"report_point": 5,
 	"p": 0.2,
 	"lr_update": 100,
-	"std_y_update_points": 100,
-	"eiv_prediction_number_of_draws": 100,
+	"std_y_update_points": [100, 250],
+	"eiv_prediction_number_of_draws": [100,5],
 	"eiv_prediction_number_of_batches": 10,
 	"init_std_y_list": [0.5],
 	"gamma": 0.5,

Experiments/configurations/eiv_kin8nm.json

@@ -9,8 +9,8 @@
 	"report_point": 5,
 	"p": 0.2,
 	"lr_update": 20,
-	"std_y_update_points": 19,
-	"eiv_prediction_number_of_draws": 100,
+	"std_y_update_points": [1,14],
+	"eiv_prediction_number_of_draws": [100,5],
 	"eiv_prediction_number_of_batches": 10,
 	"init_std_y_list": [0.5],
 	"gamma": 0.5,

Experiments/configurations/eiv_msd.json

@@ -9,8 +9,8 @@
 	"report_point": 5,
 	"p": 0.2,
 	"lr_update": 4,
-	"std_y_update_points": 4,
-	"eiv_prediction_number_of_draws": 100,
+	"std_y_update_points": [1,4],
+	"eiv_prediction_number_of_draws": [100,5],
 	"eiv_prediction_number_of_batches": 10,
 	"init_std_y_list": [0.5],
 	"gamma": 0.5,

Experiments/configurations/eiv_naval.json

@@ -9,8 +9,8 @@
         "report_point": 5,
         "p": 0.2,
         "lr_update": 20,
-        "std_y_update_points": 20,
-	"eiv_prediction_number_of_draws": 100,
+        "std_y_update_points": [1,14],
+	"eiv_prediction_number_of_draws": [100,5],
 	"eiv_prediction_number_of_batches": 10,
         "init_std_y_list": [0.5],
         "gamma": 0.5,

Experiments/configurations/eiv_power.json

@@ -9,8 +9,8 @@
 	"report_point": 5,
 	"p": 0.2,
 	"lr_update": 10,
-	"std_y_update_points": 15,
-	"eiv_prediction_number_of_draws": 100,
+	"std_y_update_points": [1,15],
+	"eiv_prediction_number_of_draws": [100,5],
 	"eiv_prediction_number_of_batches": 10,
 	"init_std_y_list": [0.5],
 	"gamma": 0.5,

Experiments/configurations/eiv_protein.json

@@ -9,8 +9,8 @@
 	"report_point": 5,
 	"p": 0.2,
 	"lr_update": 10,
-	"std_y_update_points": 10,
-	"eiv_prediction_number_of_draws": 100,
+	"std_y_update_points": [1,14],
+	"eiv_prediction_number_of_draws": [100,5],
 	"eiv_prediction_number_of_batches": 10,
 	"init_std_y_list": [0.5],
 	"gamma": 0.5,

Experiments/configurations/eiv_wine.json

@@ -9,8 +9,8 @@
 	"report_point": 5,
 	"p": 0.2,
 	"lr_update": 30,
-	"std_y_update_points": 50,
-	"eiv_prediction_number_of_draws": 100,
+	"std_y_update_points": [1,40],
+	"eiv_prediction_number_of_draws": [100,5],
 	"eiv_prediction_number_of_batches": 10,
 	"init_std_y_list": [0.5],
 	"gamma": 0.5,

Experiments/configurations/eiv_yacht.json

@@ -10,7 +10,7 @@
 	"p": 0.2,
 	"lr_update": 200,
 	"std_y_update_points": [1,500],
-	"eiv_prediction_number_of_draws": 100,
+	"eiv_prediction_number_of_draws": [100,5],
 	"eiv_prediction_number_of_batches": 10,
 	"init_std_y_list": [0.5],
 	"gamma": 0.5,

Experiments/configurations/noneiv_california.json

@@ -9,7 +9,7 @@
 	"report_point": 5,
 	"p": 0.1,
 	"lr_update": 20,
-	"std_y_update_points": [10,5] ,
+	"std_y_update_points": [1,40] ,
 	"noneiv_prediction_number_of_draws": 100,
 	"noneiv_prediction_number_of_batches": 10,
 	"init_std_y_list": [0.5],

Experiments/configurations/noneiv_concrete.json

@@ -9,9 +9,9 @@
 	"report_point": 5,
 	"p": 0.2,
 	"lr_update": 20,
-	"std_y_update_points": 10,
+	"std_y_update_points": [10,40],
 	"noneiv_prediction_number_of_draws": 100,
-	"noneiv_prediction_number_of_batches": 10,
+	"noneiv_prediction_number_of_batches": 20,
 	"init_std_y_list": [0.5],
 	"gamma": 0.5,
 	"hidden_layers": [1024, 1024, 1024, 1024],

Experiments/configurations/noneiv_energy.json

@@ -9,7 +9,7 @@
 	"report_point": 5,
 	"p": 0.2,
 	"lr_update": 100,
-	"std_y_update_points": 100,
+	"std_y_update_points": [100, 250],
 	"noneiv_prediction_number_of_draws": 100,
 	"noneiv_prediction_number_of_batches": 10,
 	"init_std_y_list": [0.5],

Experiments/configurations/noneiv_kin8nm.json

@@ -9,7 +9,7 @@
 	"report_point": 5,
 	"p": 0.2,
 	"lr_update": 20,
-	"std_y_update_points": 19,
+	"std_y_update_points": [1,14],
 	"noneiv_prediction_number_of_draws": 100,
 	"noneiv_prediction_number_of_batches": 10,
 	"init_std_y_list": [0.5],

Experiments/configurations/noneiv_msd.json

@@ -9,7 +9,7 @@
 	"report_point": 5,
 	"p": 0.2,
 	"lr_update": 4,
-	"std_y_update_points": 4,
+	"std_y_update_points": [1,4],
 	"noneiv_prediction_number_of_draws": 100,
 	"noneiv_prediction_number_of_batches": 10,
 	"init_std_y_list": [0.5],

Experiments/configurations/noneiv_naval.json

@@ -9,7 +9,7 @@
 	"report_point": 5,
 	"p": 0.2,
 	"lr_update": 20,
-	"std_y_update_points": 20,
+	"std_y_update_points": [1,14],
 	"noneiv_prediction_number_of_draws": 100,
 	"noneiv_prediction_number_of_batches": 10,
 	"init_std_y_list": [0.5],

Experiments/configurations/noneiv_power.json

@@ -9,7 +9,7 @@
 	"report_point": 5,
 	"p": 0.2,
 	"lr_update": 10,
-	"std_y_update_points": 15,
+	"std_y_update_points": [1,15],
 	"noneiv_prediction_number_of_draws": 100,
 	"noneiv_prediction_number_of_batches": 10,
 	"init_std_y_list": [0.5],

Experiments/configurations/noneiv_protein.json

@@ -9,7 +9,7 @@
 	"report_point": 5,
 	"p": 0.2,
 	"lr_update": 10,
-	"std_y_update_points": 10,
+	"std_y_update_points": [1,14],
 	"noneiv_prediction_number_of_draws": 100,
 	"noneiv_prediction_number_of_batches": 10,
 	"init_std_y_list": [0.5],

Experiments/configurations/noneiv_wine.json

@@ -9,7 +9,7 @@
 	"report_point": 5,
 	"p": 0.2,
 	"lr_update": 30,
-	"std_y_update_points": 50,
+	"std_y_update_points": [1,40],
 	"noneiv_prediction_number_of_draws": 100,
 	"noneiv_prediction_number_of_batches": 10,
 	"init_std_y_list": [0.5],

Experiments/create_tabular.py

+import os
+import glob
+import json
+
+metrics_to_display = ['rmse','logdens','bias','coverage_normalized']
+
+
+list_of_result_files = glob.glob(os.path.join('results','*.json'))
+results = {}
+for filename in list_of_result_files:
+    data = filename.replace(os.path.join('results','metrics_'),'').replace('.json','')
+    with open(filename,'r') as f:
+       results[data] = json.load(f)
+
+## header
+header_string = 'DATA'
+for metric in metrics_to_display:
+    header_string += f'          {metric}'
+print(header_string)
+## results
+for data in results.keys():
+    noneiv_results = [results[data]['noneiv'][metric]
+            for metric in metrics_to_display]
+    noneiv_results_string = f'{data} - nonEiV:'
+    for [metric_mean, metric_std] in noneiv_results:
+        noneiv_results_string += f'  {metric_mean:.3f} ({metric_std:.3f})'
+    print(noneiv_results_string)
+    eiv_results = [results[data]['eiv'][metric]
+            for metric in metrics_to_display]
+    eiv_results_string = f'{data} - EiV:'
+    for [metric_mean, metric_std] in eiv_results:
+        eiv_results_string += f'  {metric_mean:.3f} ({metric_std:.3f})'
+    print(eiv_results_string)
+
+