Skip to content
Snippets Groups Projects
Commit a3b24d3b authored by Jörg Martin's avatar Jörg Martin
Browse files

Evaluating metrics for multiple datasets

The script evaluate_tabular.py was renamed to evaluate_metrics. This
script now not only prints the results, but also stores them in JSON
files in a Experiments/results folder (should be created). These files
can be read via a new script create_tabular.py
The JSON files have now all been changed to a less frequent update of
std_y.
parents ba899f54 60fb0526
No related branches found
No related tags found
No related merge requests found
Showing
with 65 additions and 30 deletions
......@@ -9,8 +9,8 @@
"report_point": 5,
"p": 0.1,
"lr_update": 20,
"std_y_update_points": [10,5],
"eiv_prediction_number_of_draws": 100,
"std_y_update_points": [1,40],
"eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
......
......@@ -9,9 +9,9 @@
"report_point": 5,
"p": 0.2,
"lr_update": 20,
"std_y_update_points": 10,
"eiv_prediction_number_of_draws": 100,
"eiv_prediction_number_of_batches": 10,
"std_y_update_points": [10, 40],
"eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 20,
"init_std_y_list": [0.5],
"gamma": 0.5,
"hidden_layers": [1024, 1024, 1024, 1024],
......
......@@ -9,8 +9,8 @@
"report_point": 5,
"p": 0.2,
"lr_update": 100,
"std_y_update_points": 100,
"eiv_prediction_number_of_draws": 100,
"std_y_update_points": [100, 250],
"eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
......
......@@ -9,8 +9,8 @@
"report_point": 5,
"p": 0.2,
"lr_update": 20,
"std_y_update_points": 19,
"eiv_prediction_number_of_draws": 100,
"std_y_update_points": [1,14],
"eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
......
......@@ -9,8 +9,8 @@
"report_point": 5,
"p": 0.2,
"lr_update": 4,
"std_y_update_points": 4,
"eiv_prediction_number_of_draws": 100,
"std_y_update_points": [1,4],
"eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
......
......@@ -9,8 +9,8 @@
"report_point": 5,
"p": 0.2,
"lr_update": 20,
"std_y_update_points": 20,
"eiv_prediction_number_of_draws": 100,
"std_y_update_points": [1,14],
"eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
......
......@@ -9,8 +9,8 @@
"report_point": 5,
"p": 0.2,
"lr_update": 10,
"std_y_update_points": 15,
"eiv_prediction_number_of_draws": 100,
"std_y_update_points": [1,15],
"eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
......
......@@ -9,8 +9,8 @@
"report_point": 5,
"p": 0.2,
"lr_update": 10,
"std_y_update_points": 10,
"eiv_prediction_number_of_draws": 100,
"std_y_update_points": [1,14],
"eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
......
......@@ -9,8 +9,8 @@
"report_point": 5,
"p": 0.2,
"lr_update": 30,
"std_y_update_points": 50,
"eiv_prediction_number_of_draws": 100,
"std_y_update_points": [1,40],
"eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
......
......@@ -10,7 +10,7 @@
"p": 0.2,
"lr_update": 200,
"std_y_update_points": [1,500],
"eiv_prediction_number_of_draws": 100,
"eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
......
......@@ -9,7 +9,7 @@
"report_point": 5,
"p": 0.1,
"lr_update": 20,
"std_y_update_points": [10,5] ,
"std_y_update_points": [1,40] ,
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
......
......@@ -9,9 +9,9 @@
"report_point": 5,
"p": 0.2,
"lr_update": 20,
"std_y_update_points": 10,
"std_y_update_points": [10,40],
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"noneiv_prediction_number_of_batches": 20,
"init_std_y_list": [0.5],
"gamma": 0.5,
"hidden_layers": [1024, 1024, 1024, 1024],
......
......@@ -9,7 +9,7 @@
"report_point": 5,
"p": 0.2,
"lr_update": 100,
"std_y_update_points": 100,
"std_y_update_points": [100, 250],
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
......
......@@ -9,7 +9,7 @@
"report_point": 5,
"p": 0.2,
"lr_update": 20,
"std_y_update_points": 19,
"std_y_update_points": [1,14],
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
......
......@@ -9,7 +9,7 @@
"report_point": 5,
"p": 0.2,
"lr_update": 4,
"std_y_update_points": 4,
"std_y_update_points": [1,4],
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
......
......@@ -9,7 +9,7 @@
"report_point": 5,
"p": 0.2,
"lr_update": 20,
"std_y_update_points": 20,
"std_y_update_points": [1,14],
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
......
......@@ -9,7 +9,7 @@
"report_point": 5,
"p": 0.2,
"lr_update": 10,
"std_y_update_points": 15,
"std_y_update_points": [1,15],
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
......
......@@ -9,7 +9,7 @@
"report_point": 5,
"p": 0.2,
"lr_update": 10,
"std_y_update_points": 10,
"std_y_update_points": [1,14],
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
......
......@@ -9,7 +9,7 @@
"report_point": 5,
"p": 0.2,
"lr_update": 30,
"std_y_update_points": 50,
"std_y_update_points": [1,40],
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
......
import os
import glob
import json
metrics_to_display = ['rmse','logdens','bias','coverage_normalized']
list_of_result_files = glob.glob(os.path.join('results','*.json'))
results = {}
for filename in list_of_result_files:
data = filename.replace(os.path.join('results','metrics_'),'').replace('.json','')
with open(filename,'r') as f:
results[data] = json.load(f)
## header
header_string = 'DATA'
for metric in metrics_to_display:
header_string += f' {metric}'
print(header_string)
## results
for data in results.keys():
noneiv_results = [results[data]['noneiv'][metric]
for metric in metrics_to_display]
noneiv_results_string = f'{data} - nonEiV:'
for [metric_mean, metric_std] in noneiv_results:
noneiv_results_string += f' {metric_mean:.3f} ({metric_std:.3f})'
print(noneiv_results_string)
eiv_results = [results[data]['eiv'][metric]
for metric in metrics_to_display]
eiv_results_string = f'{data} - EiV:'
for [metric_mean, metric_std] in eiv_results:
eiv_results_string += f' {metric_mean:.3f} ({metric_std:.3f})'
print(eiv_results_string)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment