Metrics for coverage and bias included

Several metrics were included in evaluate_tabular.py. To this end
the file coverage_metrices.py was added and the processing of a larger
number of metrics in evaluate_tabular.py was simplified.

EIVPackage/EIVArchitectures/Networks.py

@@ -244,17 +244,23 @@ class FNNEIV(nn.Module):
         :param average_batch_dimension: Boolean. If True (default) the values
         will be averaged over the batch dimension. If False, the batch
         dimension will be left untouched and all values will be returned.
+        :scale_labels: If not None (the default), scale labels in evaluation to
+        make result comparable with the literature. 
+        :decouple_dimensions: If True, treat dimensions seperate and finally
+        average, to make results comparable with the literature. Defaults to
+        False.
         """
         out, sigmas = self.predict(x, number_of_draws=number_of_draws,
                 number_of_parameter_chunks=number_of_parameter_chunks,
                 remove_graph=remove_graph,
                 take_average_of_prediction=False)
-        # Add "repetition" dimension to y and out
+        # Add "repetition" dimension to y and sigmas
         y = y[:,None,...]
         sigmas = sigmas[:,None,...]
+        # add an output axis if necessary
         if len(y.shape) <= 2:
-            # add an output axis if necessary
             y = y[...,None]
+        if len(sigmas.shape) <= 2:
             sigmas = sigmas[...,None]
         # squeeze last dimensions into one
         y = y.view((*y.shape[:2], -1))
@@ -285,6 +291,39 @@ class FNNEIV(nn.Module):
         else:
             return predictive_log_density_values
 
+    def predict_mean_and_unc(self, x, number_of_draws=[100,5], number_of_parameter_chunks = None,
+            remove_graph=True):
+        """
+        Take the mean and standard deviation over `number_of_draws` forward
+        passes and return them together with the predicted sigmas.
+        **Note**: This method does neither touch the input noise nor Dropout.
+        The corresponding setting is left to the user!
+        :param x: A torch.tensor, the input
+        :param number_of_draws: An integer or a list. If an integer
+        `number_of_draws`, will be converted internally to
+        `[number_of_draws,1]`.Numbers of draws to obtain from x via parameter
+        sampling (first element) and noise input sampling (second element).
+        :param number_of_parameter_chunks: An integer or None (default). If
+        None, the second element of `number_of_draws` will be taken (and will
+        thus be identical to 1 if `number_of_draws` is an integer, see above).
+        Samples in the parameter space will be divided into
+        `number_of_parameter_chunks` chunks when collected. Can be used to
+        reduced the memory usage. 
+        :param remove_graph: If True (default) the output will 
+        be detached to save memory
+        :return: mean, std, sigmas
+        """
+        out, sigmas = self.predict(x=x,
+                number_of_draws=number_of_draws,
+                number_of_parameter_chunks=number_of_parameter_chunks,
+                remove_graph=remove_graph,
+                take_average_of_prediction=False)
+        mean = torch.mean(out, dim=1)
+        std = torch.std(out, dim=1)
+        return mean, std, sigmas
+
+        
+
 
 class FNNBer(nn.Module):
     """
@@ -403,6 +442,11 @@ class FNNBer(nn.Module):
         :param average_batch_dimension: Boolean. If True (default) the values
         will be averaged over the batch dimension. If False, the batch
         dimension will be left untouched and all values will be returned.
+        :scale_labels: If not None (the default), scale labels in evaluation to
+        make result comparable with the literature. 
+        :decouple_dimensions: If True, treat dimensions seperate and finally
+        average, to make results comparable with the literature. Defaults to
+        False.
         """
         out, sigmas = self.predict(x, number_of_draws=number_of_draws,
                 take_average_of_prediction=False, remove_graph=remove_graph)
@@ -442,6 +486,31 @@ class FNNBer(nn.Module):
         else:
             return predictive_log_density_values
 
+    def predict_mean_and_unc(self, x, number_of_draws=100, 
+            remove_graph=True):
+        """
+        Take the mean and standard deviation over `number_of_draws` forward
+        passes and return them together with the predicted sigmas.
+        **Note**: This method does not touch the Dropout.
+        The corresponding setting is left to the user!
+        :param x: A torch.tensor, the input
+        :param number_of_draws: An integer or a list. If an integer
+        `number_of_draws`, will be converted internally to
+        `[number_of_draws,1]`.Numbers of draws to obtain from x via parameter
+        sampling (first element) and noise input sampling (second element).
+        :param remove_graph: If True (default) the output will 
+        be detached to save memory
+        :return: mean, std, sigmas
+        """
+        out, sigmas = self.predict(x=x,
+                number_of_draws=number_of_draws,
+                remove_graph=remove_graph,
+                take_average_of_prediction=False)
+        mean = torch.mean(out, dim=1)
+        std = torch.std(out, dim=1)
+        return mean, std, sigmas
+
+
 
 class SmallFNNBer(FNNBer):
     """

EIVPackage/EIVGeneral/coverage_metrices.py

+"""
+Computes the coverage of observations by predictions and uncertainty and
+similar quantities. This module contains three functions
+- epistemic_coverage: Computes the coverage of observations by predictions and
+  their epistemic uncertainty and the averaged theoretical ground truth. If
+  `normalized` is set to True, residuals are normalized and a fixed interval
+  length is considered.
+- normalized_std: Computes the standard deviation of the normalized residuals.
+"""
+import numpy as np
+import scipy.stats
+import torch
+
+def logical_and_along_dimension(x, dim=-1, keepdim=False):
+    """
+    For a boolean tensor `x` perform a logical AND along the axis `dim`.
+    If `keepdim` is true the axis `dim` will be kept (defaults to False).
+    """
+    return torch.prod(x.to(torch.bool), dim=dim, keepdim=keepdim).to(torch.bool)
+
+
+def multivariate_interval_length(dim, q=0.95):
+    """
+    Returns the half side length of multivariate cube, symmetrically centered
+    around 0 such that its measure under a standard normal distribution equals
+    `q`.
+    :param dim: A non-negative integer, the dimension.
+    :param q: Float, should be between 0 and 1. Defaults to 0.95.
+    """
+    # use independence of components to reduce to a univariate quantile
+    univariate_quantile = 0.5 * (1+0.95**(1/dim))
+    return scipy.stats.norm.ppf(univariate_quantile)
+
+
+def epistemic_coverage(prediction_triple,  y, q=0.95, normalize_errors=False):
+    """
+    Returns the average coverage of `y` by the interval 
+    "prefactor * (predictions + q-Interval)",
+    where "q-Interval" is the interval of measure `q` under the standard normal, 
+    "predictions" is the first component of `prediction_triple` and prefactor is either
+    the epistemic uncertainty, given by the second component of `prediction_triple`, if
+    `normalize_errors` is False, or 1 if it is true. The coverage is returned
+    as given by `y` and as a theoretical_coverage computed from the epistemic
+    uncertainty (second component of `prediction_triple`) and the aleatoric uncertainty
+    (third component of `prediction_triple`)
+    :param prediction_triple: A triple of tensors containing (in this order): the
+    predictions of the neural net (the average under the posterior), the
+    epistemic uncertainty (the standard deviation under the posterior) and
+    the aleatoric uncertainty. All tensors are expected to have two dimensions:
+    a batch and a feature dimension.
+    :param y: A `torch.tensor` of the same shape then the first two components
+    of `prediction_triple`. If the feature dimension is missing, it is added.
+    :param q: A float between 0 and 1. Defaults to 0.95.
+    :param normalize_errors: If True, the deviations between predictions and
+    `y` are normalized by the total uncertainty, computed from the aleatoric
+    and epistemic uncertainty and the coverage w.r.t. q-interval is computed.
+    :returns: numerical_coverage, theoretical_coverage
+    """
+    mean, epis_unc, aleat_unc = prediction_triple
+    assert epis_unc.shape == aleat_unc.shape
+    assert mean.shape == epis_unc.shape
+    # Add feature dimension to y if missing
+    if len(y.shape) <= 1:
+        y = y.view((-1,1))
+    assert y.shape == mean.shape
+    # fix interval based on epis_unc
+    interval_length = multivariate_interval_length(dim=y.shape[1], q=q) \
+            * epis_unc
+    total_unc = torch.sqrt(epis_unc**2 + aleat_unc **2)
+    # numerical computation
+    errors = mean - y
+    if normalize_errors:
+        assert errors.shape == total_unc.shape
+        errors /= total_unc
+    check_if_in_interval = logical_and_along_dimension(
+            torch.abs(errors) <= interval_length, dim=1)
+    numerical_coverage = torch.mean(
+            check_if_in_interval.to(torch.float32)
+            ).cpu().detach().item()
+    # theoretical computation
+    if not normalize_errors:
+        cdf_args = (interval_length/total_unc).detach().cpu().numpy()
+        cdf_values = scipy.stats.norm.cdf(cdf_args)
+        prob_values = 2*cdf_values -1
+        assert len(cdf_values.shape) == 2
+        theoretical_coverage = np.mean(np.prod(prob_values, axis=1)).item()
+    else:
+        theoretical_coverage = q
+    return numerical_coverage, theoretical_coverage
+
+def normalized_std(prediction_triple, y):
+    """
+    Returns the standard deviation of normalized residuals. In theory this
+    number should be equal to 1.0.
+    :param prediction_triple: A triple of tensors containing (in this order): the
+    predictions of the neural net (the average under the posterior), the
+    epistemic uncertainty (the standard deviation under the posterior) and
+    the aleatoric uncertainty.
+    :param y: A `torch.tensor` of the same shape then the first two components
+    of `prediction_triple`. If the feature dimension is missing, it is added.
+    :returns: numerical_coverage, theoretical_coverage
+    """
+    mean, epis_unc, aleat_unc = prediction_triple
+    assert epis_unc.shape == aleat_unc.shape
+    assert mean.shape == epis_unc.shape
+    # Add feature dimension to y if missing
+    if len(y.shape) <= 1:
+        y = y.view((-1,1))
+    assert y.shape == mean.shape
+    total_unc = torch.sqrt(epis_unc**2 + aleat_unc **2)
+    # numerical computation
+    errors = mean - y
+    assert errors.shape == total_unc.shape
+    errors /= total_unc
+    return torch.mean(torch.std(errors, dim=0)).item()

Experiments/evaluate_tabular.py

 import importlib
 import os
-import matplotlib
 
 import numpy as np
 import torch
@@ -10,25 +9,27 @@ from tqdm import tqdm
 
 from EIVArchitectures import Networks
 from EIVTrainingRoutines import train_and_store
-
+from EIVGeneral.coverage_metrices import epistemic_coverage, normalized_std
 
 long_dataname = 'energy_efficiency'
 short_dataname = 'energy'
 
+scale_outputs = False 
 load_data = importlib.import_module(f'EIVData.{long_dataname}').load_data
 train_noneiv = importlib.import_module(f'train_noneiv_{short_dataname}')
 train_eiv = importlib.import_module(f'train_eiv_{short_dataname}')
 
 train_data, test_data = load_data()
-test_dataloader = DataLoader(test_data, batch_size=int(np.max((len(test_data),
-    64))), shuffle=True)
 input_dim = train_data[0][0].numel()
 output_dim = train_data[0][1].numel()
 
 def collect_metrics(x,y, seed=0,
     noneiv_number_of_draws=100, eiv_number_of_draws=[100,5],
-    decouple_dimensions=False, device=torch.device('cuda:1')):
+    decouple_dimensions=False, device=torch.device('cuda:1'),
+    scale_outputs=scale_outputs):
     """
+    Compute various metrics for EiV and non-EiV. Will be returned as
+    dictionaries.
     :param x: A torch.tensor, taken as input
     :param y: A torch.tensor, taken as output
     :param seed: Integer. The seed used for loading, defaults to 0.
@@ -40,10 +41,15 @@ def collect_metrics(x,y, seed=0,
     of Gal et al. is followed where, in the evaluation of the
     log-posterior-predictive, each dimension is treated independently and then
     averaged. If False (default), a multivariate distribution is used.
-    :returns: noneiv_rmse, noneiv_logdens, noneiv_bias,
-    eiv_rmse, eiv_logdens, eiv_bias
+    :param scale_output: Boolean, scale the outputs for the RMSE, the bias and
+    the log-dens to make them comparable with the literature.
+    :returns: Dictionaries noneiv_metrics, eiv_metrics
     """
     x,y = x.to(device), y.to(device)
+
+
+    # non-EiV
+    noneiv_metrics = {}
     init_std_y = train_noneiv.init_std_y_list[0]
     unscaled_reg = train_noneiv.unscaled_reg
     p = train_noneiv.p
@@ -61,33 +67,45 @@ def collect_metrics(x,y, seed=0,
     # RMSE
     training_state = net.training
     net.train()
-    out = net.predict(x, number_of_draws=noneiv_number_of_draws, 
-            take_average_of_prediction=True)[0]
+    ######
+    prediction_triple =\
+            net.predict_mean_and_unc(x, number_of_draws=noneiv_number_of_draws)
     if len(y.shape) <= 1:
         y = y.view((-1,1))
-    assert y.shape == out.shape
-    res = y-out
-    scale = train_data.dataset.std_labels.to(device)
-    scaled_res = res * scale.view((1,-1))
+    assert y.shape == prediction_triple[0].shape
+    res = y-prediction_triple[0]
+    if scale_outputs:
+        scale = train_data.dataset.std_labels.to(device)
+        scaled_res = res * scale.view((1,-1))
+    else:
+        scaled_res = res
     scaled_res = scaled_res.detach().cpu().numpy().flatten()
-    noneiv_rmse = np.sqrt(np.mean(scaled_res**2))
-    noneiv_bias = np.mean(scaled_res)
+    noneiv_metrics['rmse'] = np.sqrt(np.mean(scaled_res**2))
+    noneiv_metrics['bias'] = np.mean(scaled_res)
+    noneiv_metrics['coverage_numerical'], noneiv_metrics['coverage_theory'] =\
+            epistemic_coverage(prediction_triple, y, normalize_errors=False)
+    noneiv_metrics['coverage_normalized'],_ =\
+            epistemic_coverage(prediction_triple, y, normalize_errors=True)
+    noneiv_metrics['res_std'] = normalized_std(prediction_triple, y)
+    
 
 
     # NLL
-    training_state = net.training
-    net.train()
-    noneiv_logdens = net.predictive_logdensity(x, y, number_of_draws=100,
+    if scale_outputs:
+        scale_labels = train_data.dataset.std_labels.view((-1,)).to(device)
+    else:
+        scale_labels = None
+    noneiv_metrics['logdens' ]= net.predictive_logdensity(x, y,
+            number_of_draws=100,
             decouple_dimensions=decouple_dimensions,
-            scale_labels=\
-                   train_data.dataset.std_labels.view((-1,)).to(device)\
-                   ).mean().detach().cpu().numpy()
+            scale_labels=scale_labels).mean().detach().cpu().numpy()
     if training_state:
         net.train()
     else:
         net.eval()
 
     # EiV
+    eiv_metrics = {}
     init_std_y = train_eiv.init_std_y_list[0]
     unscaled_reg = train_eiv.unscaled_reg
     p = train_eiv.p
@@ -103,22 +121,43 @@ def collect_metrics(x,y, seed=0,
             fixed_std_x=fixed_std_x).to(device)
     train_and_store.open_stored_training(saved_file=saved_file,
             net=net)
+
     # RMSE
     training_state = net.training
     noise_state = net.noise_is_on
     net.train()
     net.noise_on()
-    out = net.predict(x, number_of_draws=eiv_number_of_draws,
-            take_average_of_prediction=True)[0]
-    if len(y.shape) <=1:
+    prediction_triple =\
+            net.predict_mean_and_unc(x, number_of_draws=eiv_number_of_draws)
+    if len(y.shape) <= 1:
         y = y.view((-1,1))
-    assert y.shape == out.shape
-    res = y-out
+    assert y.shape == prediction_triple[0].shape
+    res = y-prediction_triple[0]
     scale = train_data.dataset.std_labels.to(device)
-    scaled_res = res * scale.view((1,-1))
+    if scale_outputs:
+        scale = train_data.dataset.std_labels.to(device)
+        scaled_res = res * scale.view((1,-1))
+    else:
+        scaled_res = res
     scaled_res = scaled_res.detach().cpu().numpy().flatten()
-    eiv_rmse = np.sqrt(np.mean(scaled_res**2))
-    eiv_bias = np.mean(scaled_res)
+    eiv_metrics['rmse' ]= np.sqrt(np.mean(scaled_res**2))
+    eiv_metrics['bias' ]= np.mean(scaled_res)
+    eiv_metrics['coverage_numerical'], eiv_metrics['coverage_theory'] =\
+            epistemic_coverage(prediction_triple, y, normalize_errors=False)
+    eiv_metrics['coverage_normalized'],_ =\
+            epistemic_coverage(prediction_triple, y, normalize_errors=True)
+    eiv_metrics['res_std' ]= normalized_std(prediction_triple, y)
+
+
+    # NLL
+    if scale_outputs:
+        scale_labels = train_data.dataset.std_labels.view((-1,)).to(device)
+    else:
+        scale_labels = None
+    eiv_metrics['logdens' ]= net.predictive_logdensity(x, y,
+            number_of_draws=eiv_number_of_draws,
+            decouple_dimensions=decouple_dimensions,
+            scale_labels=scale_labels).mean().detach().cpu().numpy()
     if training_state:
         net.train()
     else:
@@ -127,30 +166,16 @@ def collect_metrics(x,y, seed=0,
         net.noise_on()
     else:
         net.noise_off()
+    return noneiv_metrics, eiv_metrics
 
 
-    # NLL
-    training_state = net.training
-    net.train()
-    eiv_logdens = net.predictive_logdensity(x, y,
-            number_of_draws=eiv_number_of_draws,
-            decouple_dimensions=decouple_dimensions,
-            scale_labels=\
-            train_data.dataset.std_labels.view((-1,)).to(device)\
-            ).mean().detach().cpu().numpy()
-    if training_state:
-        net.train()
-    else:
-        net.eval()
-    return noneiv_rmse, noneiv_logdens, noneiv_bias, \
-            eiv_rmse, eiv_logdens, eiv_bias
-
-noneiv_rmse_collection = []
-noneiv_logdens_collection = []
-noneiv_bias_collection = []
-eiv_rmse_collection = []
-eiv_logdens_collection = []
-eiv_bias_collection = []
+collection_keys = ['rmse','logdens','bias','coverage_numerical',
+        'coverage_theory','coverage_normalized','res_std']
+noneiv_metrics_collection = {}
+eiv_metrics_collection = {}
+for key in collection_keys:
+    noneiv_metrics_collection[key] = []
+    eiv_metrics_collection[key] = []
 num_test_epochs = 10
 assert train_noneiv.seed_list == train_eiv.seed_list
 seed_list = train_noneiv.seed_list
@@ -158,34 +183,23 @@ max_batch_number = 2
 for seed in tqdm(seed_list):
     train_data, test_data = load_data(seed=seed)
     test_dataloader = DataLoader(test_data,
-            batch_size=int(np.max((len(test_data),
+            batch_size=int(np.min((len(test_data),
         800))), shuffle=True)
     for i in tqdm(range(num_test_epochs)):
         for j, (x,y) in enumerate(test_dataloader):
             if j > max_batch_number:
                 break
-            noneiv_rmse, noneiv_logdens, noneiv_bias, \
-                    eiv_rmse, eiv_logdens, eiv_bias =\
-                    collect_metrics(x,y, seed=seed)
-            noneiv_rmse_collection.append(noneiv_rmse)
-            noneiv_logdens_collection.append(noneiv_logdens)
-            noneiv_bias_collection.append(noneiv_bias)
-            eiv_rmse_collection.append(eiv_rmse)
-            eiv_logdens_collection.append(eiv_logdens)
-            eiv_bias_collection.append(eiv_bias)
-
-
-print('Non-EiV')
-print(f'RMSE {np.mean(noneiv_rmse_collection):.5f}'\
-        f'({np.std(noneiv_rmse_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
-print(f'LogDens {np.mean(noneiv_logdens_collection):.5f}'\
-        f'({np.std(noneiv_logdens_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
-print(f'Bias {np.mean(noneiv_bias_collection):.5f}'\
-        f'({np.std(noneiv_bias_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
-print('EiV')
-print(f'RMSE {np.mean(eiv_rmse_collection):.5f}'\
-        f'({np.std(eiv_rmse_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
-print(f'LogDens {np.mean(eiv_logdens_collection):.5f}'\
-        f'({np.std(eiv_logdens_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
-print(f'Bias {np.mean(eiv_bias_collection):.5f}'\
-        f'({np.std(eiv_bias_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
+
+            noneiv_metrics, eiv_metrics = collect_metrics(x,y, seed=seed)
+            for key in collection_keys:
+                noneiv_metrics_collection[key].append(noneiv_metrics[key])
+                eiv_metrics_collection[key].append(eiv_metrics[key])
+
+print('Non-EiV\n-----')
+for key in collection_keys:
+    print(f'{key} {np.mean(noneiv_metrics_collection[key]):.5f}'\
+            f'({np.std(noneiv_metrics_collection[key])/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
+print('EiV\n-----')
+for key in collection_keys:
+    print(f'{key} {np.mean(eiv_metrics_collection[key]):.5f}'\
+            f'({np.std(eiv_metrics_collection[key])/np.sqrt(num_test_epochs*len(seed_list)):.5f})')

Experiments/train_eiv_california.py

@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')

Experiments/train_eiv_concrete.py

@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')

Experiments/train_eiv_energy.py

@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')

Experiments/train_eiv_kin8nm.py

@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')

Experiments/train_eiv_msd.py

@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')

Experiments/train_eiv_naval.py

@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')

Experiments/train_eiv_power.py

@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')

Experiments/train_eiv_protein.py

@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')

Experiments/train_eiv_wine.py

@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')

Experiments/train_eiv_yacht.py

@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')

Experiments/train_noneiv_california.py

@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')

Experiments/train_noneiv_concrete.py

@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')

Experiments/train_noneiv_energy.py

@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')

Experiments/train_noneiv_kin8nm.py

@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')

Experiments/train_noneiv_msd.py

@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')

Experiments/train_noneiv_naval.py

@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')

Experiments/train_noneiv_power.py

@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
         rmse = self.rmse(net).item()
         rmse_chain.append(rmse)
         writer.add_scalar('RMSE', rmse, self.total_count)
+        writer.add_scalar('std_y', self.last_std_y, self.total_count)
+        writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
         writer.add_scalar('train loss', self.last_train_loss, self.total_count)
         writer.add_scalar('test loss', self.last_test_loss, self.total_count)
         print(f'RMSE {rmse:.3f}')