evaluate_tabular reads in JSON configuration

Experiments/evaluate_tabular.py

 import importlib
 import os
+import argparse
+import json
 
 import numpy as np
 import torch
@@ -11,13 +13,27 @@ from EIVArchitectures import Networks
 from EIVTrainingRoutines import train_and_store
 from EIVGeneral.coverage_metrics import epistemic_coverage, normalized_std
 
-long_dataname = 'energy_efficiency'
-short_dataname = 'energy'
+# read in data via --data option
+parser = argparse.ArgumentParser()
+parser.add_argument("--data", help="Loads data", default='california')
+parser.add_argument("--no-autoindent", help="",
+        action="store_true") # to avoid conflics in IPython
+args = parser.parse_args()
+data = args.data
+
+# load hyperparameters from JSON file
+with open(os.path.join('configurations',f'eiv_{data}.json'),'r') as conf_file:
+    eiv_conf_dict = json.load(conf_file)
+with open(os.path.join('configurations',f'noneiv_{data}.json'),'r') as conf_file:
+    noneiv_conf_dict = json.load(conf_file)
+
+long_dataname = eiv_conf_dict["long_dataname"]
+short_dataname = eiv_conf_dict["short_dataname"]
+
+print(f"Evaluating {long_dataname}")
 
 scale_outputs = False 
 load_data = importlib.import_module(f'EIVData.{long_dataname}').load_data
-train_noneiv = importlib.import_module(f'train_noneiv_{short_dataname}')
-train_eiv = importlib.import_module(f'train_eiv_{short_dataname}')
 
 train_data, test_data = load_data()
 input_dim = train_data[0][0].numel()
@@ -50,10 +66,10 @@ def collect_metrics(x,y, seed=0,
 
     # non-EiV
     noneiv_metrics = {}
-    init_std_y = train_noneiv.init_std_y_list[0]
-    unscaled_reg = train_noneiv.unscaled_reg
-    p = train_noneiv.p
-    hidden_layers = train_noneiv.hidden_layers
+    init_std_y = noneiv_conf_dict["init_std_y_list"][0]
+    unscaled_reg = noneiv_conf_dict["unscaled_reg"]
+    p = noneiv_conf_dict["p"]
+    hidden_layers = noneiv_conf_dict["hidden_layers"]
     saved_file = os.path.join('saved_networks',
                 f'noneiv_{short_dataname}'\
                         f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
@@ -107,11 +123,11 @@ def collect_metrics(x,y, seed=0,
 
     # EiV
     eiv_metrics = {}
-    init_std_y = train_eiv.init_std_y_list[0]
-    unscaled_reg = train_eiv.unscaled_reg
-    p = train_eiv.p
-    hidden_layers = train_eiv.hidden_layers
-    fixed_std_x = train_eiv.fixed_std_x
+    init_std_y = eiv_conf_dict["init_std_y_list"][0]
+    unscaled_reg = eiv_conf_dict["unscaled_reg"]
+    p = eiv_conf_dict["p"]
+    hidden_layers = eiv_conf_dict["hidden_layers"]
+    fixed_std_x = eiv_conf_dict["fixed_std_x"]
     saved_file = os.path.join('saved_networks',
             f'eiv_{short_dataname}'\
                     f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
@@ -180,8 +196,9 @@ for key in collection_keys:
     noneiv_metrics_collection[key] = []
     eiv_metrics_collection[key] = []
 num_test_epochs = 10
-assert train_noneiv.seed_list == train_eiv.seed_list
-seed_list = train_noneiv.seed_list
+assert noneiv_conf_dict["seed_range"] == eiv_conf_dict["seed_range"]
+seed_list = range(noneiv_conf_dict["seed_range"][0],
+        noneiv_conf_dict["seed_range"][1])
 max_batch_number = 2
 for seed in tqdm(seed_list):
     train_data, test_data = load_data(seed=seed)

Experiments/train_noneiv.py

@@ -49,6 +49,8 @@ gamma = conf_dict["gamma"]
 hidden_layers = conf_dict["hidden_layers"]
 seed_range = conf_dict['seed_range']
 
+print(f"Training on {long_dataname} data")
+
 try:
     gpu_number = conf_dict["gpu_number"]
     device = torch.device(f'cuda:{gpu_number}' if torch.cuda.is_available() else 'cpu')