diff --git a/EIVPackage/EIVArchitectures/Networks.py b/EIVPackage/EIVArchitectures/Networks.py index 09d75b877ba2617af15447e9bacd22b8581de35d..ccae84842e6a29979c0a72156606ddcfaf8ca3e0 100644 --- a/EIVPackage/EIVArchitectures/Networks.py +++ b/EIVPackage/EIVArchitectures/Networks.py @@ -244,17 +244,23 @@ class FNNEIV(nn.Module): :param average_batch_dimension: Boolean. If True (default) the values will be averaged over the batch dimension. If False, the batch dimension will be left untouched and all values will be returned. + :scale_labels: If not None (the default), scale labels in evaluation to + make result comparable with the literature. + :decouple_dimensions: If True, treat dimensions seperate and finally + average, to make results comparable with the literature. Defaults to + False. """ out, sigmas = self.predict(x, number_of_draws=number_of_draws, number_of_parameter_chunks=number_of_parameter_chunks, remove_graph=remove_graph, take_average_of_prediction=False) - # Add "repetition" dimension to y and out + # Add "repetition" dimension to y and sigmas y = y[:,None,...] sigmas = sigmas[:,None,...] + # add an output axis if necessary if len(y.shape) <= 2: - # add an output axis if necessary y = y[...,None] + if len(sigmas.shape) <= 2: sigmas = sigmas[...,None] # squeeze last dimensions into one y = y.view((*y.shape[:2], -1)) @@ -285,6 +291,39 @@ class FNNEIV(nn.Module): else: return predictive_log_density_values + def predict_mean_and_unc(self, x, number_of_draws=[100,5], number_of_parameter_chunks = None, + remove_graph=True): + """ + Take the mean and standard deviation over `number_of_draws` forward + passes and return them together with the predicted sigmas. + **Note**: This method does neither touch the input noise nor Dropout. + The corresponding setting is left to the user! + :param x: A torch.tensor, the input + :param number_of_draws: An integer or a list. If an integer + `number_of_draws`, will be converted internally to + `[number_of_draws,1]`.Numbers of draws to obtain from x via parameter + sampling (first element) and noise input sampling (second element). + :param number_of_parameter_chunks: An integer or None (default). If + None, the second element of `number_of_draws` will be taken (and will + thus be identical to 1 if `number_of_draws` is an integer, see above). + Samples in the parameter space will be divided into + `number_of_parameter_chunks` chunks when collected. Can be used to + reduced the memory usage. + :param remove_graph: If True (default) the output will + be detached to save memory + :return: mean, std, sigmas + """ + out, sigmas = self.predict(x=x, + number_of_draws=number_of_draws, + number_of_parameter_chunks=number_of_parameter_chunks, + remove_graph=remove_graph, + take_average_of_prediction=False) + mean = torch.mean(out, dim=1) + std = torch.std(out, dim=1) + return mean, std, sigmas + + + class FNNBer(nn.Module): """ @@ -403,6 +442,11 @@ class FNNBer(nn.Module): :param average_batch_dimension: Boolean. If True (default) the values will be averaged over the batch dimension. If False, the batch dimension will be left untouched and all values will be returned. + :scale_labels: If not None (the default), scale labels in evaluation to + make result comparable with the literature. + :decouple_dimensions: If True, treat dimensions seperate and finally + average, to make results comparable with the literature. Defaults to + False. """ out, sigmas = self.predict(x, number_of_draws=number_of_draws, take_average_of_prediction=False, remove_graph=remove_graph) @@ -442,6 +486,31 @@ class FNNBer(nn.Module): else: return predictive_log_density_values + def predict_mean_and_unc(self, x, number_of_draws=100, + remove_graph=True): + """ + Take the mean and standard deviation over `number_of_draws` forward + passes and return them together with the predicted sigmas. + **Note**: This method does not touch the Dropout. + The corresponding setting is left to the user! + :param x: A torch.tensor, the input + :param number_of_draws: An integer or a list. If an integer + `number_of_draws`, will be converted internally to + `[number_of_draws,1]`.Numbers of draws to obtain from x via parameter + sampling (first element) and noise input sampling (second element). + :param remove_graph: If True (default) the output will + be detached to save memory + :return: mean, std, sigmas + """ + out, sigmas = self.predict(x=x, + number_of_draws=number_of_draws, + remove_graph=remove_graph, + take_average_of_prediction=False) + mean = torch.mean(out, dim=1) + std = torch.std(out, dim=1) + return mean, std, sigmas + + class SmallFNNBer(FNNBer): """ diff --git a/EIVPackage/EIVGeneral/coverage_metrices.py b/EIVPackage/EIVGeneral/coverage_metrices.py new file mode 100644 index 0000000000000000000000000000000000000000..1c6ea70cd2e015d319aee481255a2c22075c4453 --- /dev/null +++ b/EIVPackage/EIVGeneral/coverage_metrices.py @@ -0,0 +1,115 @@ +""" +Computes the coverage of observations by predictions and uncertainty and +similar quantities. This module contains three functions +- epistemic_coverage: Computes the coverage of observations by predictions and + their epistemic uncertainty and the averaged theoretical ground truth. If + `normalized` is set to True, residuals are normalized and a fixed interval + length is considered. +- normalized_std: Computes the standard deviation of the normalized residuals. +""" +import numpy as np +import scipy.stats +import torch + +def logical_and_along_dimension(x, dim=-1, keepdim=False): + """ + For a boolean tensor `x` perform a logical AND along the axis `dim`. + If `keepdim` is true the axis `dim` will be kept (defaults to False). + """ + return torch.prod(x.to(torch.bool), dim=dim, keepdim=keepdim).to(torch.bool) + + +def multivariate_interval_length(dim, q=0.95): + """ + Returns the half side length of multivariate cube, symmetrically centered + around 0 such that its measure under a standard normal distribution equals + `q`. + :param dim: A non-negative integer, the dimension. + :param q: Float, should be between 0 and 1. Defaults to 0.95. + """ + # use independence of components to reduce to a univariate quantile + univariate_quantile = 0.5 * (1+0.95**(1/dim)) + return scipy.stats.norm.ppf(univariate_quantile) + + +def epistemic_coverage(prediction_triple, y, q=0.95, normalize_errors=False): + """ + Returns the average coverage of `y` by the interval + "prefactor * (predictions + q-Interval)", + where "q-Interval" is the interval of measure `q` under the standard normal, + "predictions" is the first component of `prediction_triple` and prefactor is either + the epistemic uncertainty, given by the second component of `prediction_triple`, if + `normalize_errors` is False, or 1 if it is true. The coverage is returned + as given by `y` and as a theoretical_coverage computed from the epistemic + uncertainty (second component of `prediction_triple`) and the aleatoric uncertainty + (third component of `prediction_triple`) + :param prediction_triple: A triple of tensors containing (in this order): the + predictions of the neural net (the average under the posterior), the + epistemic uncertainty (the standard deviation under the posterior) and + the aleatoric uncertainty. All tensors are expected to have two dimensions: + a batch and a feature dimension. + :param y: A `torch.tensor` of the same shape then the first two components + of `prediction_triple`. If the feature dimension is missing, it is added. + :param q: A float between 0 and 1. Defaults to 0.95. + :param normalize_errors: If True, the deviations between predictions and + `y` are normalized by the total uncertainty, computed from the aleatoric + and epistemic uncertainty and the coverage w.r.t. q-interval is computed. + :returns: numerical_coverage, theoretical_coverage + """ + mean, epis_unc, aleat_unc = prediction_triple + assert epis_unc.shape == aleat_unc.shape + assert mean.shape == epis_unc.shape + # Add feature dimension to y if missing + if len(y.shape) <= 1: + y = y.view((-1,1)) + assert y.shape == mean.shape + # fix interval based on epis_unc + interval_length = multivariate_interval_length(dim=y.shape[1], q=q) \ + * epis_unc + total_unc = torch.sqrt(epis_unc**2 + aleat_unc **2) + # numerical computation + errors = mean - y + if normalize_errors: + assert errors.shape == total_unc.shape + errors /= total_unc + check_if_in_interval = logical_and_along_dimension( + torch.abs(errors) <= interval_length, dim=1) + numerical_coverage = torch.mean( + check_if_in_interval.to(torch.float32) + ).cpu().detach().item() + # theoretical computation + if not normalize_errors: + cdf_args = (interval_length/total_unc).detach().cpu().numpy() + cdf_values = scipy.stats.norm.cdf(cdf_args) + prob_values = 2*cdf_values -1 + assert len(cdf_values.shape) == 2 + theoretical_coverage = np.mean(np.prod(prob_values, axis=1)).item() + else: + theoretical_coverage = q + return numerical_coverage, theoretical_coverage + +def normalized_std(prediction_triple, y): + """ + Returns the standard deviation of normalized residuals. In theory this + number should be equal to 1.0. + :param prediction_triple: A triple of tensors containing (in this order): the + predictions of the neural net (the average under the posterior), the + epistemic uncertainty (the standard deviation under the posterior) and + the aleatoric uncertainty. + :param y: A `torch.tensor` of the same shape then the first two components + of `prediction_triple`. If the feature dimension is missing, it is added. + :returns: numerical_coverage, theoretical_coverage + """ + mean, epis_unc, aleat_unc = prediction_triple + assert epis_unc.shape == aleat_unc.shape + assert mean.shape == epis_unc.shape + # Add feature dimension to y if missing + if len(y.shape) <= 1: + y = y.view((-1,1)) + assert y.shape == mean.shape + total_unc = torch.sqrt(epis_unc**2 + aleat_unc **2) + # numerical computation + errors = mean - y + assert errors.shape == total_unc.shape + errors /= total_unc + return torch.mean(torch.std(errors, dim=0)).item() diff --git a/Experiments/evaluate_tabular.py b/Experiments/evaluate_tabular.py index d74c553789953779c12fee3a70f02de4b9d08328..6a738bdaae782541924d8c474efb90c7eac6b8b0 100644 --- a/Experiments/evaluate_tabular.py +++ b/Experiments/evaluate_tabular.py @@ -1,6 +1,5 @@ import importlib import os -import matplotlib import numpy as np import torch @@ -10,25 +9,27 @@ from tqdm import tqdm from EIVArchitectures import Networks from EIVTrainingRoutines import train_and_store - +from EIVGeneral.coverage_metrices import epistemic_coverage, normalized_std long_dataname = 'energy_efficiency' short_dataname = 'energy' +scale_outputs = False load_data = importlib.import_module(f'EIVData.{long_dataname}').load_data train_noneiv = importlib.import_module(f'train_noneiv_{short_dataname}') train_eiv = importlib.import_module(f'train_eiv_{short_dataname}') train_data, test_data = load_data() -test_dataloader = DataLoader(test_data, batch_size=int(np.max((len(test_data), - 64))), shuffle=True) input_dim = train_data[0][0].numel() output_dim = train_data[0][1].numel() def collect_metrics(x,y, seed=0, noneiv_number_of_draws=100, eiv_number_of_draws=[100,5], - decouple_dimensions=False, device=torch.device('cuda:1')): + decouple_dimensions=False, device=torch.device('cuda:1'), + scale_outputs=scale_outputs): """ + Compute various metrics for EiV and non-EiV. Will be returned as + dictionaries. :param x: A torch.tensor, taken as input :param y: A torch.tensor, taken as output :param seed: Integer. The seed used for loading, defaults to 0. @@ -40,10 +41,15 @@ def collect_metrics(x,y, seed=0, of Gal et al. is followed where, in the evaluation of the log-posterior-predictive, each dimension is treated independently and then averaged. If False (default), a multivariate distribution is used. - :returns: noneiv_rmse, noneiv_logdens, noneiv_bias, - eiv_rmse, eiv_logdens, eiv_bias + :param scale_output: Boolean, scale the outputs for the RMSE, the bias and + the log-dens to make them comparable with the literature. + :returns: Dictionaries noneiv_metrics, eiv_metrics """ x,y = x.to(device), y.to(device) + + + # non-EiV + noneiv_metrics = {} init_std_y = train_noneiv.init_std_y_list[0] unscaled_reg = train_noneiv.unscaled_reg p = train_noneiv.p @@ -61,33 +67,45 @@ def collect_metrics(x,y, seed=0, # RMSE training_state = net.training net.train() - out = net.predict(x, number_of_draws=noneiv_number_of_draws, - take_average_of_prediction=True)[0] + ###### + prediction_triple =\ + net.predict_mean_and_unc(x, number_of_draws=noneiv_number_of_draws) if len(y.shape) <= 1: y = y.view((-1,1)) - assert y.shape == out.shape - res = y-out - scale = train_data.dataset.std_labels.to(device) - scaled_res = res * scale.view((1,-1)) + assert y.shape == prediction_triple[0].shape + res = y-prediction_triple[0] + if scale_outputs: + scale = train_data.dataset.std_labels.to(device) + scaled_res = res * scale.view((1,-1)) + else: + scaled_res = res scaled_res = scaled_res.detach().cpu().numpy().flatten() - noneiv_rmse = np.sqrt(np.mean(scaled_res**2)) - noneiv_bias = np.mean(scaled_res) + noneiv_metrics['rmse'] = np.sqrt(np.mean(scaled_res**2)) + noneiv_metrics['bias'] = np.mean(scaled_res) + noneiv_metrics['coverage_numerical'], noneiv_metrics['coverage_theory'] =\ + epistemic_coverage(prediction_triple, y, normalize_errors=False) + noneiv_metrics['coverage_normalized'],_ =\ + epistemic_coverage(prediction_triple, y, normalize_errors=True) + noneiv_metrics['res_std'] = normalized_std(prediction_triple, y) + # NLL - training_state = net.training - net.train() - noneiv_logdens = net.predictive_logdensity(x, y, number_of_draws=100, + if scale_outputs: + scale_labels = train_data.dataset.std_labels.view((-1,)).to(device) + else: + scale_labels = None + noneiv_metrics['logdens' ]= net.predictive_logdensity(x, y, + number_of_draws=100, decouple_dimensions=decouple_dimensions, - scale_labels=\ - train_data.dataset.std_labels.view((-1,)).to(device)\ - ).mean().detach().cpu().numpy() + scale_labels=scale_labels).mean().detach().cpu().numpy() if training_state: net.train() else: net.eval() # EiV + eiv_metrics = {} init_std_y = train_eiv.init_std_y_list[0] unscaled_reg = train_eiv.unscaled_reg p = train_eiv.p @@ -103,22 +121,43 @@ def collect_metrics(x,y, seed=0, fixed_std_x=fixed_std_x).to(device) train_and_store.open_stored_training(saved_file=saved_file, net=net) + # RMSE training_state = net.training noise_state = net.noise_is_on net.train() net.noise_on() - out = net.predict(x, number_of_draws=eiv_number_of_draws, - take_average_of_prediction=True)[0] - if len(y.shape) <=1: + prediction_triple =\ + net.predict_mean_and_unc(x, number_of_draws=eiv_number_of_draws) + if len(y.shape) <= 1: y = y.view((-1,1)) - assert y.shape == out.shape - res = y-out + assert y.shape == prediction_triple[0].shape + res = y-prediction_triple[0] scale = train_data.dataset.std_labels.to(device) - scaled_res = res * scale.view((1,-1)) + if scale_outputs: + scale = train_data.dataset.std_labels.to(device) + scaled_res = res * scale.view((1,-1)) + else: + scaled_res = res scaled_res = scaled_res.detach().cpu().numpy().flatten() - eiv_rmse = np.sqrt(np.mean(scaled_res**2)) - eiv_bias = np.mean(scaled_res) + eiv_metrics['rmse' ]= np.sqrt(np.mean(scaled_res**2)) + eiv_metrics['bias' ]= np.mean(scaled_res) + eiv_metrics['coverage_numerical'], eiv_metrics['coverage_theory'] =\ + epistemic_coverage(prediction_triple, y, normalize_errors=False) + eiv_metrics['coverage_normalized'],_ =\ + epistemic_coverage(prediction_triple, y, normalize_errors=True) + eiv_metrics['res_std' ]= normalized_std(prediction_triple, y) + + + # NLL + if scale_outputs: + scale_labels = train_data.dataset.std_labels.view((-1,)).to(device) + else: + scale_labels = None + eiv_metrics['logdens' ]= net.predictive_logdensity(x, y, + number_of_draws=eiv_number_of_draws, + decouple_dimensions=decouple_dimensions, + scale_labels=scale_labels).mean().detach().cpu().numpy() if training_state: net.train() else: @@ -127,30 +166,16 @@ def collect_metrics(x,y, seed=0, net.noise_on() else: net.noise_off() + return noneiv_metrics, eiv_metrics - # NLL - training_state = net.training - net.train() - eiv_logdens = net.predictive_logdensity(x, y, - number_of_draws=eiv_number_of_draws, - decouple_dimensions=decouple_dimensions, - scale_labels=\ - train_data.dataset.std_labels.view((-1,)).to(device)\ - ).mean().detach().cpu().numpy() - if training_state: - net.train() - else: - net.eval() - return noneiv_rmse, noneiv_logdens, noneiv_bias, \ - eiv_rmse, eiv_logdens, eiv_bias - -noneiv_rmse_collection = [] -noneiv_logdens_collection = [] -noneiv_bias_collection = [] -eiv_rmse_collection = [] -eiv_logdens_collection = [] -eiv_bias_collection = [] +collection_keys = ['rmse','logdens','bias','coverage_numerical', + 'coverage_theory','coverage_normalized','res_std'] +noneiv_metrics_collection = {} +eiv_metrics_collection = {} +for key in collection_keys: + noneiv_metrics_collection[key] = [] + eiv_metrics_collection[key] = [] num_test_epochs = 10 assert train_noneiv.seed_list == train_eiv.seed_list seed_list = train_noneiv.seed_list @@ -158,34 +183,23 @@ max_batch_number = 2 for seed in tqdm(seed_list): train_data, test_data = load_data(seed=seed) test_dataloader = DataLoader(test_data, - batch_size=int(np.max((len(test_data), + batch_size=int(np.min((len(test_data), 800))), shuffle=True) for i in tqdm(range(num_test_epochs)): for j, (x,y) in enumerate(test_dataloader): if j > max_batch_number: break - noneiv_rmse, noneiv_logdens, noneiv_bias, \ - eiv_rmse, eiv_logdens, eiv_bias =\ - collect_metrics(x,y, seed=seed) - noneiv_rmse_collection.append(noneiv_rmse) - noneiv_logdens_collection.append(noneiv_logdens) - noneiv_bias_collection.append(noneiv_bias) - eiv_rmse_collection.append(eiv_rmse) - eiv_logdens_collection.append(eiv_logdens) - eiv_bias_collection.append(eiv_bias) - - -print('Non-EiV') -print(f'RMSE {np.mean(noneiv_rmse_collection):.5f}'\ - f'({np.std(noneiv_rmse_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})') -print(f'LogDens {np.mean(noneiv_logdens_collection):.5f}'\ - f'({np.std(noneiv_logdens_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})') -print(f'Bias {np.mean(noneiv_bias_collection):.5f}'\ - f'({np.std(noneiv_bias_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})') -print('EiV') -print(f'RMSE {np.mean(eiv_rmse_collection):.5f}'\ - f'({np.std(eiv_rmse_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})') -print(f'LogDens {np.mean(eiv_logdens_collection):.5f}'\ - f'({np.std(eiv_logdens_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})') -print(f'Bias {np.mean(eiv_bias_collection):.5f}'\ - f'({np.std(eiv_bias_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})') + + noneiv_metrics, eiv_metrics = collect_metrics(x,y, seed=seed) + for key in collection_keys: + noneiv_metrics_collection[key].append(noneiv_metrics[key]) + eiv_metrics_collection[key].append(eiv_metrics[key]) + +print('Non-EiV\n-----') +for key in collection_keys: + print(f'{key} {np.mean(noneiv_metrics_collection[key]):.5f}'\ + f'({np.std(noneiv_metrics_collection[key])/np.sqrt(num_test_epochs*len(seed_list)):.5f})') +print('EiV\n-----') +for key in collection_keys: + print(f'{key} {np.mean(eiv_metrics_collection[key]):.5f}'\ + f'({np.std(eiv_metrics_collection[key])/np.sqrt(num_test_epochs*len(seed_list)):.5f})') diff --git a/Experiments/train_eiv_california.py b/Experiments/train_eiv_california.py index 39c95fbe0cad3b7f839a8c230e9c54e5bfcde8f3..f90a204c2134254078ea55b76e5009ceb19d3402 100644 --- a/Experiments/train_eiv_california.py +++ b/Experiments/train_eiv_california.py @@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_eiv_concrete.py b/Experiments/train_eiv_concrete.py index 0cc42d5e05181c471b8281c2ff98eefc1a08aa41..e637d359d9a7eed1dd87f40d3dfd8709040298cf 100644 --- a/Experiments/train_eiv_concrete.py +++ b/Experiments/train_eiv_concrete.py @@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_eiv_energy.py b/Experiments/train_eiv_energy.py index 40be4b283ed12849418e6c760618f79a70122bda..304501b089043b8ea701f088e9859b655171dbd2 100644 --- a/Experiments/train_eiv_energy.py +++ b/Experiments/train_eiv_energy.py @@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_eiv_kin8nm.py b/Experiments/train_eiv_kin8nm.py index 58d191c062f61eed6a4725bb7f364ed0303aeaaa..96b6c79c9316978e656ba901ff2f3fd6b0814026 100644 --- a/Experiments/train_eiv_kin8nm.py +++ b/Experiments/train_eiv_kin8nm.py @@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_eiv_msd.py b/Experiments/train_eiv_msd.py index 16e617f0a64fb8f9567ac2846a22cefff109b901..4b533b691d5c052d2e28a43eac3c89caca1e70fa 100644 --- a/Experiments/train_eiv_msd.py +++ b/Experiments/train_eiv_msd.py @@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_eiv_naval.py b/Experiments/train_eiv_naval.py index 8db266c735d022fe4a2170a7de4d0a5ba3610d74..f4a9aa75a4b8e00dce105764544b2b2a9d9ad192 100644 --- a/Experiments/train_eiv_naval.py +++ b/Experiments/train_eiv_naval.py @@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_eiv_power.py b/Experiments/train_eiv_power.py index 2ddf75af75db7607a21cef6e212c9601d391b421..d9be09e4b7129599a41a784ed66e79f795fe67b8 100644 --- a/Experiments/train_eiv_power.py +++ b/Experiments/train_eiv_power.py @@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_eiv_protein.py b/Experiments/train_eiv_protein.py index 3801d5e07572a7f37b23b457a3adc8b83f3f8418..625fb6a3620af1fd5aef227895ae39ebc7095c91 100644 --- a/Experiments/train_eiv_protein.py +++ b/Experiments/train_eiv_protein.py @@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_eiv_wine.py b/Experiments/train_eiv_wine.py index ca561adc2e4d3f9a37060d90b370d21a06ed01d0..d6c9a41f9737005330c6aca56bf060a94a71ad56 100644 --- a/Experiments/train_eiv_wine.py +++ b/Experiments/train_eiv_wine.py @@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_eiv_yacht.py b/Experiments/train_eiv_yacht.py index 2c9ee88853d77b99fefc3f2507ad827597895d61..10e2607ea151ff071fd1ca8ebee8e8ccc6885b6a 100644 --- a/Experiments/train_eiv_yacht.py +++ b/Experiments/train_eiv_yacht.py @@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_noneiv_california.py b/Experiments/train_noneiv_california.py index 773a30ddf5d50d3a7d2fff2c58e6a9579ba24992..f359b4303aff0536ae52320c3e138478ac5da2ea 100644 --- a/Experiments/train_noneiv_california.py +++ b/Experiments/train_noneiv_california.py @@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_noneiv_concrete.py b/Experiments/train_noneiv_concrete.py index d4a48a8dc6fc7c0273ea4f894d63d38acc931d7a..5e7c1e33ea750b5ddc661471969682ecab2008cd 100644 --- a/Experiments/train_noneiv_concrete.py +++ b/Experiments/train_noneiv_concrete.py @@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_noneiv_energy.py b/Experiments/train_noneiv_energy.py index 106bc14e5802054f0bcfdbfdc3d7f4ebbd13d3ab..340b81ad389e443a2cc67a6e02388f463b735b0f 100644 --- a/Experiments/train_noneiv_energy.py +++ b/Experiments/train_noneiv_energy.py @@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_noneiv_kin8nm.py b/Experiments/train_noneiv_kin8nm.py index 90af665059e0eca165cfb97fd991ba877fcfbad0..7070c381f25135038a4e6b7ca5058d7efbccfb7f 100644 --- a/Experiments/train_noneiv_kin8nm.py +++ b/Experiments/train_noneiv_kin8nm.py @@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_noneiv_msd.py b/Experiments/train_noneiv_msd.py index f80c03e44ba37c0028806725e2beed1001ab0699..b72644d9e964d15e8e6fbf97a6b4f7e43226bce1 100644 --- a/Experiments/train_noneiv_msd.py +++ b/Experiments/train_noneiv_msd.py @@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_noneiv_naval.py b/Experiments/train_noneiv_naval.py index f96c66e1c20425042726f3c22c27285ed8b085d1..fb8c57f6774981d5099f8c18f2b6e0dbba58af5e 100644 --- a/Experiments/train_noneiv_naval.py +++ b/Experiments/train_noneiv_naval.py @@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_noneiv_power.py b/Experiments/train_noneiv_power.py index c63b5713cb6fd1a8dd1b9d0ea0d5e6a4bcec510f..3c87a2d74e27f09692d393366b0bacd8396c2881 100644 --- a/Experiments/train_noneiv_power.py +++ b/Experiments/train_noneiv_power.py @@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_noneiv_protein.py b/Experiments/train_noneiv_protein.py index ca73d6af733bffd469da898d8bd43c958068586b..9e2d3f4095520182cc3b8f859c746c691f14722b 100644 --- a/Experiments/train_noneiv_protein.py +++ b/Experiments/train_noneiv_protein.py @@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_noneiv_wine.py b/Experiments/train_noneiv_wine.py index dc239f74fda492662853d92859e355585f476db0..a445c1e87b3e72069614b4d4bb3f5f31a8477566 100644 --- a/Experiments/train_noneiv_wine.py +++ b/Experiments/train_noneiv_wine.py @@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}') diff --git a/Experiments/train_noneiv_yacht.py b/Experiments/train_noneiv_yacht.py index d163e544f50bdc09283d138ba7149b5290f2a515..489a81c146a1e63a679c1235c590add5be8785ca 100644 --- a/Experiments/train_noneiv_yacht.py +++ b/Experiments/train_noneiv_yacht.py @@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch): rmse = self.rmse(net).item() rmse_chain.append(rmse) writer.add_scalar('RMSE', rmse, self.total_count) + writer.add_scalar('std_y', self.last_std_y, self.total_count) + writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count) writer.add_scalar('train loss', self.last_train_loss, self.total_count) writer.add_scalar('test loss', self.last_test_loss, self.total_count) print(f'RMSE {rmse:.3f}')