diff --git a/Experiments/configurations/eiv_california.json b/Experiments/configurations/eiv_california.json
index 6cfa57f9f95c63dd347629d0acf4df37b96b180f..46d2c4771972f09a34944f48b188ddefcd3a1f00 100644
--- a/Experiments/configurations/eiv_california.json
+++ b/Experiments/configurations/eiv_california.json
@@ -9,8 +9,8 @@
"report_point": 5,
"p": 0.1,
"lr_update": 20,
- "std_y_update_points": [10,5],
- "eiv_prediction_number_of_draws": 100,
+ "std_y_update_points": [1,40],
+ "eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
diff --git a/Experiments/configurations/eiv_concrete.json b/Experiments/configurations/eiv_concrete.json
index 0e234346f6dda090daacb9aebf33f8c3f666081c..0997d650b05907e078c2aa2dd68bcb15c5bd94df 100644
--- a/Experiments/configurations/eiv_concrete.json
+++ b/Experiments/configurations/eiv_concrete.json
@@ -9,9 +9,9 @@
"report_point": 5,
"p": 0.2,
"lr_update": 20,
- "std_y_update_points": 10,
- "eiv_prediction_number_of_draws": 100,
- "eiv_prediction_number_of_batches": 10,
+ "std_y_update_points": [10, 40],
+ "eiv_prediction_number_of_draws": [100,5],
+ "eiv_prediction_number_of_batches": 20,
"init_std_y_list": [0.5],
"gamma": 0.5,
"hidden_layers": [1024, 1024, 1024, 1024],
diff --git a/Experiments/configurations/eiv_energy.json b/Experiments/configurations/eiv_energy.json
index f13ab1b52417aa7ed20c4db9d75a0c653ff0b321..ee4219897179aa91d2e22d2ee860a8a0e425d5d7 100644
--- a/Experiments/configurations/eiv_energy.json
+++ b/Experiments/configurations/eiv_energy.json
@@ -9,8 +9,8 @@
"report_point": 5,
"p": 0.2,
"lr_update": 100,
- "std_y_update_points": 100,
- "eiv_prediction_number_of_draws": 100,
+ "std_y_update_points": [100, 250],
+ "eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
diff --git a/Experiments/configurations/eiv_kin8nm.json b/Experiments/configurations/eiv_kin8nm.json
index df2e4bafbee99aca95dcd36f8f67097d3a6ce3b5..65ba9a2936e5af1efa5dc4d104c85c0defc8d665 100644
--- a/Experiments/configurations/eiv_kin8nm.json
+++ b/Experiments/configurations/eiv_kin8nm.json
@@ -9,8 +9,8 @@
"report_point": 5,
"p": 0.2,
"lr_update": 20,
- "std_y_update_points": 19,
- "eiv_prediction_number_of_draws": 100,
+ "std_y_update_points": [1,14],
+ "eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
diff --git a/Experiments/configurations/eiv_msd.json b/Experiments/configurations/eiv_msd.json
index a738f2e8bf034e173f60b17e5cd6d2f7d6fdb856..d29808616f409eb56467b162271e0362b196de76 100644
--- a/Experiments/configurations/eiv_msd.json
+++ b/Experiments/configurations/eiv_msd.json
@@ -9,8 +9,8 @@
"report_point": 5,
"p": 0.2,
"lr_update": 4,
- "std_y_update_points": 4,
- "eiv_prediction_number_of_draws": 100,
+ "std_y_update_points": [1,4],
+ "eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
diff --git a/Experiments/configurations/eiv_naval.json b/Experiments/configurations/eiv_naval.json
index 092f4f822abf0e9c677affa8ae275e46309435b8..8975c95eb1fbfa030d4bc018bed4da7192174a30 100644
--- a/Experiments/configurations/eiv_naval.json
+++ b/Experiments/configurations/eiv_naval.json
@@ -9,8 +9,8 @@
"report_point": 5,
"p": 0.2,
"lr_update": 20,
- "std_y_update_points": 20,
- "eiv_prediction_number_of_draws": 100,
+ "std_y_update_points": [1,14],
+ "eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
diff --git a/Experiments/configurations/eiv_power.json b/Experiments/configurations/eiv_power.json
index ab5649f61de1ea2214249b8a07e25cd5b2adea9e..917f1f002b4b79382d002183cd577e5ac653d3cb 100644
--- a/Experiments/configurations/eiv_power.json
+++ b/Experiments/configurations/eiv_power.json
@@ -9,8 +9,8 @@
"report_point": 5,
"p": 0.2,
"lr_update": 10,
- "std_y_update_points": 15,
- "eiv_prediction_number_of_draws": 100,
+ "std_y_update_points": [1,15],
+ "eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
diff --git a/Experiments/configurations/eiv_protein.json b/Experiments/configurations/eiv_protein.json
index 9623799e3d8b85b2f568987afb4d406fbb7fe45c..330713a1118dcfc2e2ef3b8159574fbbdb696932 100644
--- a/Experiments/configurations/eiv_protein.json
+++ b/Experiments/configurations/eiv_protein.json
@@ -9,8 +9,8 @@
"report_point": 5,
"p": 0.2,
"lr_update": 10,
- "std_y_update_points": 10,
- "eiv_prediction_number_of_draws": 100,
+ "std_y_update_points": [1,14],
+ "eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
diff --git a/Experiments/configurations/eiv_wine.json b/Experiments/configurations/eiv_wine.json
index f0beaf490846c2281a477f627f05c98b18cfa7f3..fef0c488758678ca809102ce53f30e226a3b77c1 100644
--- a/Experiments/configurations/eiv_wine.json
+++ b/Experiments/configurations/eiv_wine.json
@@ -9,8 +9,8 @@
"report_point": 5,
"p": 0.2,
"lr_update": 30,
- "std_y_update_points": 50,
- "eiv_prediction_number_of_draws": 100,
+ "std_y_update_points": [1,40],
+ "eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
diff --git a/Experiments/configurations/eiv_yacht.json b/Experiments/configurations/eiv_yacht.json
index 7617289d9e49bb9ceddd9962d6d865fee3413ae7..25b64573186e3dd568bbd2b117d15092cfe9cc31 100644
--- a/Experiments/configurations/eiv_yacht.json
+++ b/Experiments/configurations/eiv_yacht.json
@@ -10,7 +10,7 @@
"p": 0.2,
"lr_update": 200,
"std_y_update_points": [1,500],
- "eiv_prediction_number_of_draws": 100,
+ "eiv_prediction_number_of_draws": [100,5],
"eiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
"gamma": 0.5,
diff --git a/Experiments/configurations/noneiv_california.json b/Experiments/configurations/noneiv_california.json
index e0729764699ba987d4a8e8b4e344601303c16f3b..d15d2300aed80e83db7a40ad753cabbbe40cae59 100644
--- a/Experiments/configurations/noneiv_california.json
+++ b/Experiments/configurations/noneiv_california.json
@@ -9,7 +9,7 @@
"report_point": 5,
"p": 0.1,
"lr_update": 20,
- "std_y_update_points": [10,5] ,
+ "std_y_update_points": [1,40] ,
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
diff --git a/Experiments/configurations/noneiv_concrete.json b/Experiments/configurations/noneiv_concrete.json
index 15e28c53a67064987c267f932733a09d00698c43..911817ece3dc8ac1d21f192cd00c776c9681f336 100644
--- a/Experiments/configurations/noneiv_concrete.json
+++ b/Experiments/configurations/noneiv_concrete.json
@@ -9,9 +9,9 @@
"report_point": 5,
"p": 0.2,
"lr_update": 20,
- "std_y_update_points": 10,
+ "std_y_update_points": [10,40],
"noneiv_prediction_number_of_draws": 100,
- "noneiv_prediction_number_of_batches": 10,
+ "noneiv_prediction_number_of_batches": 20,
"init_std_y_list": [0.5],
"gamma": 0.5,
"hidden_layers": [1024, 1024, 1024, 1024],
diff --git a/Experiments/configurations/noneiv_energy.json b/Experiments/configurations/noneiv_energy.json
index 18c068ea7a9cb96f1075a522ae3f3ad6552bacac..003877a936d6878d7134407a579c7f1cb21fbb16 100644
--- a/Experiments/configurations/noneiv_energy.json
+++ b/Experiments/configurations/noneiv_energy.json
@@ -9,7 +9,7 @@
"report_point": 5,
"p": 0.2,
"lr_update": 100,
- "std_y_update_points": 100,
+ "std_y_update_points": [100, 250],
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
diff --git a/Experiments/configurations/noneiv_kin8nm.json b/Experiments/configurations/noneiv_kin8nm.json
index 8ecf817149b1a37dd9a4ab5c44fd2700795fbcde..694d7f0d065021462c841e636957caf0d46622ad 100644
--- a/Experiments/configurations/noneiv_kin8nm.json
+++ b/Experiments/configurations/noneiv_kin8nm.json
@@ -9,7 +9,7 @@
"report_point": 5,
"p": 0.2,
"lr_update": 20,
- "std_y_update_points": 19,
+ "std_y_update_points": [1,14],
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
diff --git a/Experiments/configurations/noneiv_msd.json b/Experiments/configurations/noneiv_msd.json
index a0e2f73beb63e165c7da5266b774f227e90310c1..126a571eb955cf2dd320d0b40bd4c749446a3253 100644
--- a/Experiments/configurations/noneiv_msd.json
+++ b/Experiments/configurations/noneiv_msd.json
@@ -9,7 +9,7 @@
"report_point": 5,
"p": 0.2,
"lr_update": 4,
- "std_y_update_points": 4,
+ "std_y_update_points": [1,4],
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
diff --git a/Experiments/configurations/noneiv_naval.json b/Experiments/configurations/noneiv_naval.json
index 2da83b73101d1a5a957f9773053b9610b0cb8aff..2562b1cf5014df581bb3b86dfc271aad3daf2472 100644
--- a/Experiments/configurations/noneiv_naval.json
+++ b/Experiments/configurations/noneiv_naval.json
@@ -9,7 +9,7 @@
"report_point": 5,
"p": 0.2,
"lr_update": 20,
- "std_y_update_points": 20,
+ "std_y_update_points": [1,14],
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
diff --git a/Experiments/configurations/noneiv_power.json b/Experiments/configurations/noneiv_power.json
index 48aac9dcf4cc208095142760d7bf286c2589b22b..e14e82d00f0048d0fa42a47baddf0ff520dbc27e 100644
--- a/Experiments/configurations/noneiv_power.json
+++ b/Experiments/configurations/noneiv_power.json
@@ -9,7 +9,7 @@
"report_point": 5,
"p": 0.2,
"lr_update": 10,
- "std_y_update_points": 15,
+ "std_y_update_points": [1,15],
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
diff --git a/Experiments/configurations/noneiv_protein.json b/Experiments/configurations/noneiv_protein.json
index 454deaef823c1a25c03588c4f430b5d0f67e5e93..36f5fafd53fc56e16387b86a038507e848564c97 100644
--- a/Experiments/configurations/noneiv_protein.json
+++ b/Experiments/configurations/noneiv_protein.json
@@ -9,7 +9,7 @@
"report_point": 5,
"p": 0.2,
"lr_update": 10,
- "std_y_update_points": 10,
+ "std_y_update_points": [1,14],
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
diff --git a/Experiments/configurations/noneiv_wine.json b/Experiments/configurations/noneiv_wine.json
index ddd2199fae58e2dd62409d554f51983b5bd1ef02..1c014763748a07c5ecb846a2391eff968fb09a78 100644
--- a/Experiments/configurations/noneiv_wine.json
+++ b/Experiments/configurations/noneiv_wine.json
@@ -9,7 +9,7 @@
"report_point": 5,
"p": 0.2,
"lr_update": 30,
- "std_y_update_points": 50,
+ "std_y_update_points": [1,40],
"noneiv_prediction_number_of_draws": 100,
"noneiv_prediction_number_of_batches": 10,
"init_std_y_list": [0.5],
diff --git a/Experiments/create_tabular.py b/Experiments/create_tabular.py
new file mode 100644
index 0000000000000000000000000000000000000000..3d128431359c29961892df2f49e3a2f11720f450
--- /dev/null
+++ b/Experiments/create_tabular.py
@@ -0,0 +1,35 @@
+import os
+import glob
+import json
+
+metrics_to_display = ['rmse','logdens','bias','coverage_normalized']
+
+
+list_of_result_files = glob.glob(os.path.join('results','*.json'))
+results = {}
+for filename in list_of_result_files:
+ data = filename.replace(os.path.join('results','metrics_'),'').replace('.json','')
+ with open(filename,'r') as f:
+ results[data] = json.load(f)
+
+## header
+header_string = 'DATA'
+for metric in metrics_to_display:
+ header_string += f' {metric}'
+print(header_string)
+## results
+for data in results.keys():
+ noneiv_results = [results[data]['noneiv'][metric]
+ for metric in metrics_to_display]
+ noneiv_results_string = f'{data} - nonEiV:'
+ for [metric_mean, metric_std] in noneiv_results:
+ noneiv_results_string += f' {metric_mean:.3f} ({metric_std:.3f})'
+ print(noneiv_results_string)
+ eiv_results = [results[data]['eiv'][metric]
+ for metric in metrics_to_display]
+ eiv_results_string = f'{data} - EiV:'
+ for [metric_mean, metric_std] in eiv_results:
+ eiv_results_string += f' {metric_mean:.3f} ({metric_std:.3f})'
+ print(eiv_results_string)
+
+
diff --git a/Experiments/evaluate_tabular.py b/Experiments/evaluate_metrics.py
similarity index 86%
rename from Experiments/evaluate_tabular.py
rename to Experiments/evaluate_metrics.py
index ba0654f071cdb96cfa72145fe048da4b088bc16b..5dbc16fcc5277e95db69d61b5decaee1a17fc0df 100644
--- a/Experiments/evaluate_tabular.py
+++ b/Experiments/evaluate_metrics.py
@@ -39,9 +39,25 @@ train_data, test_data = load_data()
input_dim = train_data[0][0].numel()
output_dim = train_data[0][1].numel()
+try:
+ gpu_number = eiv_conf_dict["gpu_number"]
+ device = torch.device(f'cuda:{gpu_number}')
+ try:
+ torch.tensor([0.0]).to(device)
+ except RuntimeError:
+ if torch.cuda.is_available():
+ print('Switched to GPU 0')
+ device = torch.device('cuda:0')
+ else:
+ print('No cuda available, using CPU')
+ device = torch.device('cpu')
+except KeyError:
+ device = torch.device('cpu')
+
+
def collect_metrics(x,y, seed=0,
noneiv_number_of_draws=100, eiv_number_of_draws=[100,5],
- decouple_dimensions=False, device=torch.device('cuda:1'),
+ decouple_dimensions=False, device=device,
scale_outputs=scale_outputs):
"""
Compute various metrics for EiV and non-EiV. Will be returned as
@@ -215,11 +231,23 @@ for seed in tqdm(seed_list):
noneiv_metrics_collection[key].append(noneiv_metrics[key])
eiv_metrics_collection[key].append(eiv_metrics[key])
-print('Non-EiV\n-----')
+results_dict = {}
+print('Non-EiV:\n-----')
+results_dict['noneiv'] = {}
for key in collection_keys:
- print(f'{key} {np.mean(noneiv_metrics_collection[key]):.5f}'\
- f'({np.std(noneiv_metrics_collection[key])/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
-print('EiV\n-----')
+ metric_mean = float(np.mean(noneiv_metrics_collection[key]))
+ metric_std = float(np.std(noneiv_metrics_collection[key])/np.sqrt(num_test_epochs*len(seed_list)))
+ results_dict['noneiv'][key] = (metric_mean, metric_std)
+ print(f'{key}: {metric_mean:.5f} ({metric_std:.5f})')
+print('\n')
+print('EiV:\n-----')
+results_dict['eiv'] = {}
for key in collection_keys:
- print(f'{key} {np.mean(eiv_metrics_collection[key]):.5f}'\
- f'({np.std(eiv_metrics_collection[key])/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
+ metric_mean = float(np.mean(eiv_metrics_collection[key]))
+ metric_std = float(np.std(eiv_metrics_collection[key])/np.sqrt(num_test_epochs*len(seed_list)))
+ print(f'{key}: {metric_mean:.5f} ({metric_std:.5f})')
+ results_dict['eiv'][key] = (metric_mean, metric_std)
+
+# write results to a JSON file in the results folder
+with open(os.path.join('results',f'metrics_{short_dataname}.json'), 'w') as f:
+ json.dump(results_dict, f)
diff --git a/Experiments/train_eiv.py b/Experiments/train_eiv.py
index eb8d9d88292dd63130b38bc78db4eb1ed1db9014..68229469c167fc934a06bbf61e7c5673a3005b12 100644
--- a/Experiments/train_eiv.py
+++ b/Experiments/train_eiv.py
@@ -55,8 +55,16 @@ print(f"Training on {long_dataname} data")
try:
gpu_number = conf_dict["gpu_number"]
- device = torch.device(f'cuda:{gpu_number}' if torch.cuda.is_available()
- else 'cpu')
+ device = torch.device(f'cuda:{gpu_number}')
+ try:
+ torch.tensor([0.0]).to(device)
+ except RuntimeError:
+ if torch.cuda.is_available():
+ print('Switched to GPU 0')
+ device = torch.device('cuda:0')
+ else:
+ print('No cuda available, using CPU')
+ device = torch.device('cpu')
except KeyError:
device = torch.device('cpu')
diff --git a/Experiments/train_noneiv.py b/Experiments/train_noneiv.py
index 316dda1e8f5fbf42f46ae9135dadd5c803623df8..ee5687b268d762aeadac294ebdd6e43c37a25149 100644
--- a/Experiments/train_noneiv.py
+++ b/Experiments/train_noneiv.py
@@ -54,8 +54,16 @@ print(f"Training on {long_dataname} data")
try:
gpu_number = conf_dict["gpu_number"]
- device = torch.device(f'cuda:{gpu_number}' if torch.cuda.is_available()
- else 'cpu')
+ device = torch.device(f'cuda:{gpu_number}')
+ try:
+ torch.tensor([0.0]).to(device)
+ except RuntimeError:
+ if torch.cuda.is_available():
+ print('Switched to GPU 0')
+ device = torch.device('cuda:0')
+ else:
+ print('No cuda available, using CPU')
+ device = torch.device('cpu')
except KeyError:
device = torch.device('cpu')