diff --git a/Experiments/evaluate_tabular.py b/Experiments/evaluate_tabular.py index 2df70458341e2f7bb7a16e70e149586f3130c08a..7493e13b37fd417bb00fec04873d740a79781950 100644 --- a/Experiments/evaluate_tabular.py +++ b/Experiments/evaluate_tabular.py @@ -1,5 +1,7 @@ import importlib import os +import argparse +import json import numpy as np import torch @@ -11,13 +13,27 @@ from EIVArchitectures import Networks from EIVTrainingRoutines import train_and_store from EIVGeneral.coverage_metrics import epistemic_coverage, normalized_std -long_dataname = 'energy_efficiency' -short_dataname = 'energy' +# read in data via --data option +parser = argparse.ArgumentParser() +parser.add_argument("--data", help="Loads data", default='california') +parser.add_argument("--no-autoindent", help="", + action="store_true") # to avoid conflics in IPython +args = parser.parse_args() +data = args.data + +# load hyperparameters from JSON file +with open(os.path.join('configurations',f'eiv_{data}.json'),'r') as conf_file: + eiv_conf_dict = json.load(conf_file) +with open(os.path.join('configurations',f'noneiv_{data}.json'),'r') as conf_file: + noneiv_conf_dict = json.load(conf_file) + +long_dataname = eiv_conf_dict["long_dataname"] +short_dataname = eiv_conf_dict["short_dataname"] + +print(f"Evaluating {long_dataname}") scale_outputs = False load_data = importlib.import_module(f'EIVData.{long_dataname}').load_data -train_noneiv = importlib.import_module(f'train_noneiv_{short_dataname}') -train_eiv = importlib.import_module(f'train_eiv_{short_dataname}') train_data, test_data = load_data() input_dim = train_data[0][0].numel() @@ -50,10 +66,10 @@ def collect_metrics(x,y, seed=0, # non-EiV noneiv_metrics = {} - init_std_y = train_noneiv.init_std_y_list[0] - unscaled_reg = train_noneiv.unscaled_reg - p = train_noneiv.p - hidden_layers = train_noneiv.hidden_layers + init_std_y = noneiv_conf_dict["init_std_y_list"][0] + unscaled_reg = noneiv_conf_dict["unscaled_reg"] + p = noneiv_conf_dict["p"] + hidden_layers = noneiv_conf_dict["hidden_layers"] saved_file = os.path.join('saved_networks', f'noneiv_{short_dataname}'\ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\ @@ -107,11 +123,11 @@ def collect_metrics(x,y, seed=0, # EiV eiv_metrics = {} - init_std_y = train_eiv.init_std_y_list[0] - unscaled_reg = train_eiv.unscaled_reg - p = train_eiv.p - hidden_layers = train_eiv.hidden_layers - fixed_std_x = train_eiv.fixed_std_x + init_std_y = eiv_conf_dict["init_std_y_list"][0] + unscaled_reg = eiv_conf_dict["unscaled_reg"] + p = eiv_conf_dict["p"] + hidden_layers = eiv_conf_dict["hidden_layers"] + fixed_std_x = eiv_conf_dict["fixed_std_x"] saved_file = os.path.join('saved_networks', f'eiv_{short_dataname}'\ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\ @@ -180,8 +196,9 @@ for key in collection_keys: noneiv_metrics_collection[key] = [] eiv_metrics_collection[key] = [] num_test_epochs = 10 -assert train_noneiv.seed_list == train_eiv.seed_list -seed_list = train_noneiv.seed_list +assert noneiv_conf_dict["seed_range"] == eiv_conf_dict["seed_range"] +seed_list = range(noneiv_conf_dict["seed_range"][0], + noneiv_conf_dict["seed_range"][1]) max_batch_number = 2 for seed in tqdm(seed_list): train_data, test_data = load_data(seed=seed) diff --git a/Experiments/train_noneiv.py b/Experiments/train_noneiv.py index 24d051fc8f72971c5b3501b1221e732894b0c1c7..e63dbf233062d7ee47465cd5cd50f5708271cd68 100644 --- a/Experiments/train_noneiv.py +++ b/Experiments/train_noneiv.py @@ -49,6 +49,8 @@ gamma = conf_dict["gamma"] hidden_layers = conf_dict["hidden_layers"] seed_range = conf_dict['seed_range'] +print(f"Training on {long_dataname} data") + try: gpu_number = conf_dict["gpu_number"] device = torch.device(f'cuda:{gpu_number}' if torch.cuda.is_available() else 'cpu')