Evaluating metrics for multiple datasets

The script evaluate_tabular.py was renamed to evaluate_metrics. This
script now not only prints the results, but also stores them in JSON
files in a Experiments/results folder (should be created). These files
can be read via a new script create_tabular.py
The JSON files have now all been changed to a less frequent update of
std_y.

Experiments/evaluate_tabular.py → Experiments/evaluate_metrics.py

@@ -39,9 +39,25 @@ train_data, test_data = load_data()
 input_dim = train_data[0][0].numel()
 output_dim = train_data[0][1].numel()
 
+try:
+    gpu_number = eiv_conf_dict["gpu_number"]
+    device = torch.device(f'cuda:{gpu_number}')
+    try:
+        torch.tensor([0.0]).to(device)
+    except RuntimeError:
+        if torch.cuda.is_available():
+            print('Switched to GPU 0')
+            device = torch.device('cuda:0')
+        else:
+            print('No cuda available, using CPU')
+            device = torch.device('cpu')
+except KeyError:
+    device = torch.device('cpu')
+
+
 def collect_metrics(x,y, seed=0,
     noneiv_number_of_draws=100, eiv_number_of_draws=[100,5],
-    decouple_dimensions=False, device=torch.device('cuda:1'),
+    decouple_dimensions=False, device=device,
     scale_outputs=scale_outputs):
     """
     Compute various metrics for EiV and non-EiV. Will be returned as
@@ -215,11 +231,23 @@ for seed in tqdm(seed_list):
                 noneiv_metrics_collection[key].append(noneiv_metrics[key])
                 eiv_metrics_collection[key].append(eiv_metrics[key])
 
-print('Non-EiV\n-----')
+results_dict = {}
+print('Non-EiV:\n-----')
+results_dict['noneiv'] = {}
 for key in collection_keys:
-    print(f'{key} {np.mean(noneiv_metrics_collection[key]):.5f}'\
-            f'({np.std(noneiv_metrics_collection[key])/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
-print('EiV\n-----')
+    metric_mean = float(np.mean(noneiv_metrics_collection[key]))
+    metric_std  = float(np.std(noneiv_metrics_collection[key])/np.sqrt(num_test_epochs*len(seed_list)))
+    results_dict['noneiv'][key] = (metric_mean, metric_std)
+    print(f'{key}: {metric_mean:.5f} ({metric_std:.5f})')
+print('\n')
+print('EiV:\n-----')
+results_dict['eiv'] = {}
 for key in collection_keys:
-    print(f'{key} {np.mean(eiv_metrics_collection[key]):.5f}'\
-            f'({np.std(eiv_metrics_collection[key])/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
+    metric_mean = float(np.mean(eiv_metrics_collection[key]))
+    metric_std  = float(np.std(eiv_metrics_collection[key])/np.sqrt(num_test_epochs*len(seed_list)))
+    print(f'{key}: {metric_mean:.5f} ({metric_std:.5f})')
+    results_dict['eiv'][key] = (metric_mean, metric_std)
+
+# write results to a JSON file in the results folder
+with open(os.path.join('results',f'metrics_{short_dataname}.json'), 'w') as f:
+    json.dump(results_dict, f)

Experiments/train_eiv.py

@@ -55,8 +55,16 @@ print(f"Training on {long_dataname} data")
 
 try:
     gpu_number = conf_dict["gpu_number"]
-    device = torch.device(f'cuda:{gpu_number}' if torch.cuda.is_available()
-            else 'cpu')
+    device = torch.device(f'cuda:{gpu_number}')
+    try:
+        torch.tensor([0.0]).to(device)
+    except RuntimeError:
+        if torch.cuda.is_available():
+            print('Switched to GPU 0')
+            device = torch.device('cuda:0')
+        else:
+            print('No cuda available, using CPU')
+            device = torch.device('cpu')
 except KeyError:
     device = torch.device('cpu')
 

Experiments/train_noneiv.py

@@ -54,8 +54,16 @@ print(f"Training on {long_dataname} data")
 
 try:
     gpu_number = conf_dict["gpu_number"]
-    device = torch.device(f'cuda:{gpu_number}' if torch.cuda.is_available()
-            else 'cpu')
+    device = torch.device(f'cuda:{gpu_number}')
+    try:
+        torch.tensor([0.0]).to(device)
+    except RuntimeError:
+        if torch.cuda.is_available():
+            print('Switched to GPU 0')
+            device = torch.device('cuda:0')
+        else:
+            print('No cuda available, using CPU')
+            device = torch.device('cpu')
 except KeyError:
     device = torch.device('cpu')