From b2eb16c69e79b1fc8a466ccea46e2ce8f84cf2bc Mon Sep 17 00:00:00 2001
From: Joerg Martin <joerg.martin@ptb.de>
Date: Wed, 8 Dec 2021 14:08:50 +0100
Subject: [PATCH] coverage and bias added to evaluate_tabular
The evaluate_tabular script was simplified to ease the analysis of more
quantities. Several coverage quantities and the bias were added to these
quantities. However, they perform rather poor.
---
EIVPackage/EIVArchitectures/Networks.py | 25 ++++
EIVPackage/EIVGeneral/coverage_metrices.py | 42 ++++++-
Experiments/evaluate_tabular.py | 133 ++++++++++-----------
Experiments/train_eiv_california.py | 2 +
Experiments/train_eiv_concrete.py | 2 +
Experiments/train_eiv_energy.py | 2 +
Experiments/train_eiv_kin8nm.py | 2 +
Experiments/train_eiv_msd.py | 2 +
Experiments/train_eiv_naval.py | 2 +
Experiments/train_eiv_power.py | 2 +
Experiments/train_eiv_protein.py | 2 +
Experiments/train_eiv_wine.py | 2 +
Experiments/train_eiv_yacht.py | 2 +
Experiments/train_noneiv_california.py | 2 +
Experiments/train_noneiv_concrete.py | 2 +
Experiments/train_noneiv_energy.py | 2 +
Experiments/train_noneiv_kin8nm.py | 2 +
Experiments/train_noneiv_msd.py | 2 +
Experiments/train_noneiv_naval.py | 2 +
Experiments/train_noneiv_power.py | 2 +
Experiments/train_noneiv_protein.py | 2 +
Experiments/train_noneiv_wine.py | 2 +
Experiments/train_noneiv_yacht.py | 2 +
23 files changed, 168 insertions(+), 72 deletions(-)
diff --git a/EIVPackage/EIVArchitectures/Networks.py b/EIVPackage/EIVArchitectures/Networks.py
index a8f1200..56c7633 100644
--- a/EIVPackage/EIVArchitectures/Networks.py
+++ b/EIVPackage/EIVArchitectures/Networks.py
@@ -476,6 +476,31 @@ class FNNBer(nn.Module):
else:
return predictive_log_density_values
+ def predict_mean_and_unc(self, x, number_of_draws=100,
+ remove_graph=True):
+ """
+ Take the mean and standard deviation over `number_of_draws` forward
+ passes and return them together with the predicted sigmas.
+ **Note**: This method does not touch the Dropout.
+ The corresponding setting is left to the user!
+ :param x: A torch.tensor, the input
+ :param number_of_draws: An integer or a list. If an integer
+ `number_of_draws`, will be converted internally to
+ `[number_of_draws,1]`.Numbers of draws to obtain from x via parameter
+ sampling (first element) and noise input sampling (second element).
+ :param remove_graph: If True (default) the output will
+ be detached to save memory
+ :return: mean, std, sigmas
+ """
+ out, sigmas = self.predict(x=x,
+ number_of_draws=number_of_draws,
+ remove_graph=remove_graph,
+ take_average_of_prediction=False)
+ mean = torch.mean(out, dim=1)
+ std = torch.std(out, dim=1)
+ return mean, std, sigmas
+
+
class SmallFNNBer(FNNBer):
"""
diff --git a/EIVPackage/EIVGeneral/coverage_metrices.py b/EIVPackage/EIVGeneral/coverage_metrices.py
index 7187648..1c6ea70 100644
--- a/EIVPackage/EIVGeneral/coverage_metrices.py
+++ b/EIVPackage/EIVGeneral/coverage_metrices.py
@@ -32,8 +32,31 @@ def multivariate_interval_length(dim, q=0.95):
return scipy.stats.norm.ppf(univariate_quantile)
-def epistemic_coverage(mean_unc, y, q=0.95, normalize_errors=False):
- mean, epis_unc, aleat_unc = mean_unc
+def epistemic_coverage(prediction_triple, y, q=0.95, normalize_errors=False):
+ """
+ Returns the average coverage of `y` by the interval
+ "prefactor * (predictions + q-Interval)",
+ where "q-Interval" is the interval of measure `q` under the standard normal,
+ "predictions" is the first component of `prediction_triple` and prefactor is either
+ the epistemic uncertainty, given by the second component of `prediction_triple`, if
+ `normalize_errors` is False, or 1 if it is true. The coverage is returned
+ as given by `y` and as a theoretical_coverage computed from the epistemic
+ uncertainty (second component of `prediction_triple`) and the aleatoric uncertainty
+ (third component of `prediction_triple`)
+ :param prediction_triple: A triple of tensors containing (in this order): the
+ predictions of the neural net (the average under the posterior), the
+ epistemic uncertainty (the standard deviation under the posterior) and
+ the aleatoric uncertainty. All tensors are expected to have two dimensions:
+ a batch and a feature dimension.
+ :param y: A `torch.tensor` of the same shape then the first two components
+ of `prediction_triple`. If the feature dimension is missing, it is added.
+ :param q: A float between 0 and 1. Defaults to 0.95.
+ :param normalize_errors: If True, the deviations between predictions and
+ `y` are normalized by the total uncertainty, computed from the aleatoric
+ and epistemic uncertainty and the coverage w.r.t. q-interval is computed.
+ :returns: numerical_coverage, theoretical_coverage
+ """
+ mean, epis_unc, aleat_unc = prediction_triple
assert epis_unc.shape == aleat_unc.shape
assert mean.shape == epis_unc.shape
# Add feature dimension to y if missing
@@ -65,8 +88,19 @@ def epistemic_coverage(mean_unc, y, q=0.95, normalize_errors=False):
theoretical_coverage = q
return numerical_coverage, theoretical_coverage
-def normalized_std(mean_unc, y):
- mean, epis_unc, aleat_unc = mean_unc
+def normalized_std(prediction_triple, y):
+ """
+ Returns the standard deviation of normalized residuals. In theory this
+ number should be equal to 1.0.
+ :param prediction_triple: A triple of tensors containing (in this order): the
+ predictions of the neural net (the average under the posterior), the
+ epistemic uncertainty (the standard deviation under the posterior) and
+ the aleatoric uncertainty.
+ :param y: A `torch.tensor` of the same shape then the first two components
+ of `prediction_triple`. If the feature dimension is missing, it is added.
+ :returns: numerical_coverage, theoretical_coverage
+ """
+ mean, epis_unc, aleat_unc = prediction_triple
assert epis_unc.shape == aleat_unc.shape
assert mean.shape == epis_unc.shape
# Add feature dimension to y if missing
diff --git a/Experiments/evaluate_tabular.py b/Experiments/evaluate_tabular.py
index d74c553..cfa6664 100644
--- a/Experiments/evaluate_tabular.py
+++ b/Experiments/evaluate_tabular.py
@@ -1,6 +1,5 @@
import importlib
import os
-import matplotlib
import numpy as np
import torch
@@ -10,7 +9,7 @@ from tqdm import tqdm
from EIVArchitectures import Networks
from EIVTrainingRoutines import train_and_store
-
+from EIVGeneral.coverage_metrices import epistemic_coverage, normalized_std
long_dataname = 'energy_efficiency'
short_dataname = 'energy'
@@ -20,8 +19,6 @@ train_noneiv = importlib.import_module(f'train_noneiv_{short_dataname}')
train_eiv = importlib.import_module(f'train_eiv_{short_dataname}')
train_data, test_data = load_data()
-test_dataloader = DataLoader(test_data, batch_size=int(np.max((len(test_data),
- 64))), shuffle=True)
input_dim = train_data[0][0].numel()
output_dim = train_data[0][1].numel()
@@ -29,6 +26,8 @@ def collect_metrics(x,y, seed=0,
noneiv_number_of_draws=100, eiv_number_of_draws=[100,5],
decouple_dimensions=False, device=torch.device('cuda:1')):
"""
+ Compute various metrics for EiV and non-EiV. Will be returned as
+ dictionaries.
:param x: A torch.tensor, taken as input
:param y: A torch.tensor, taken as output
:param seed: Integer. The seed used for loading, defaults to 0.
@@ -40,10 +39,13 @@ def collect_metrics(x,y, seed=0,
of Gal et al. is followed where, in the evaluation of the
log-posterior-predictive, each dimension is treated independently and then
averaged. If False (default), a multivariate distribution is used.
- :returns: noneiv_rmse, noneiv_logdens, noneiv_bias,
- eiv_rmse, eiv_logdens, eiv_bias
+ :returns: Dictionaries noneiv_metrics, eiv_metrics
"""
x,y = x.to(device), y.to(device)
+
+
+ # non-EiV
+ noneiv_metrics = {}
init_std_y = train_noneiv.init_std_y_list[0]
unscaled_reg = train_noneiv.unscaled_reg
p = train_noneiv.p
@@ -61,23 +63,28 @@ def collect_metrics(x,y, seed=0,
# RMSE
training_state = net.training
net.train()
- out = net.predict(x, number_of_draws=noneiv_number_of_draws,
- take_average_of_prediction=True)[0]
+ ######
+ prediction_triple =\
+ net.predict_mean_and_unc(x, number_of_draws=noneiv_number_of_draws)
if len(y.shape) <= 1:
y = y.view((-1,1))
- assert y.shape == out.shape
- res = y-out
+ assert y.shape == prediction_triple[0].shape
+ res = y-prediction_triple[0]
scale = train_data.dataset.std_labels.to(device)
scaled_res = res * scale.view((1,-1))
scaled_res = scaled_res.detach().cpu().numpy().flatten()
- noneiv_rmse = np.sqrt(np.mean(scaled_res**2))
- noneiv_bias = np.mean(scaled_res)
+ noneiv_metrics['rmse'] = np.sqrt(np.mean(scaled_res**2))
+ noneiv_metrics['bias'] = np.mean(scaled_res)
+ noneiv_metrics['coverage_numerical'], noneiv_metrics['coverage_theory'] =\
+ epistemic_coverage(prediction_triple, y, normalize_errors=False)
+ noneiv_metrics['coverage_normalized'],_ =\
+ epistemic_coverage(prediction_triple, y, normalize_errors=True)
+ noneiv_metrics['res_std'] = normalized_std(prediction_triple, y)
+
# NLL
- training_state = net.training
- net.train()
- noneiv_logdens = net.predictive_logdensity(x, y, number_of_draws=100,
+ noneiv_metrics['logdens' ]= net.predictive_logdensity(x, y, number_of_draws=100,
decouple_dimensions=decouple_dimensions,
scale_labels=\
train_data.dataset.std_labels.view((-1,)).to(device)\
@@ -88,6 +95,7 @@ def collect_metrics(x,y, seed=0,
net.eval()
# EiV
+ eiv_metrics = {}
init_std_y = train_eiv.init_std_y_list[0]
unscaled_reg = train_eiv.unscaled_reg
p = train_eiv.p
@@ -103,36 +111,32 @@ def collect_metrics(x,y, seed=0,
fixed_std_x=fixed_std_x).to(device)
train_and_store.open_stored_training(saved_file=saved_file,
net=net)
+
# RMSE
training_state = net.training
noise_state = net.noise_is_on
net.train()
net.noise_on()
- out = net.predict(x, number_of_draws=eiv_number_of_draws,
- take_average_of_prediction=True)[0]
- if len(y.shape) <=1:
+ prediction_triple =\
+ net.predict_mean_and_unc(x, number_of_draws=eiv_number_of_draws)
+ if len(y.shape) <= 1:
y = y.view((-1,1))
- assert y.shape == out.shape
- res = y-out
+ assert y.shape == prediction_triple[0].shape
+ res = y-prediction_triple[0]
scale = train_data.dataset.std_labels.to(device)
scaled_res = res * scale.view((1,-1))
scaled_res = scaled_res.detach().cpu().numpy().flatten()
- eiv_rmse = np.sqrt(np.mean(scaled_res**2))
- eiv_bias = np.mean(scaled_res)
- if training_state:
- net.train()
- else:
- net.eval()
- if noise_state:
- net.noise_on()
- else:
- net.noise_off()
+ eiv_metrics['rmse' ]= np.sqrt(np.mean(scaled_res**2))
+ eiv_metrics['bias' ]= np.mean(scaled_res)
+ eiv_metrics['coverage_numerical'], eiv_metrics['coverage_theory'] =\
+ epistemic_coverage(prediction_triple, y, normalize_errors=False)
+ eiv_metrics['coverage_normalized'],_ =\
+ epistemic_coverage(prediction_triple, y, normalize_errors=True)
+ eiv_metrics['res_std' ]= normalized_std(prediction_triple, y)
# NLL
- training_state = net.training
- net.train()
- eiv_logdens = net.predictive_logdensity(x, y,
+ eiv_metrics['logdens' ]= net.predictive_logdensity(x, y,
number_of_draws=eiv_number_of_draws,
decouple_dimensions=decouple_dimensions,
scale_labels=\
@@ -142,15 +146,19 @@ def collect_metrics(x,y, seed=0,
net.train()
else:
net.eval()
- return noneiv_rmse, noneiv_logdens, noneiv_bias, \
- eiv_rmse, eiv_logdens, eiv_bias
-
-noneiv_rmse_collection = []
-noneiv_logdens_collection = []
-noneiv_bias_collection = []
-eiv_rmse_collection = []
-eiv_logdens_collection = []
-eiv_bias_collection = []
+ if noise_state:
+ net.noise_on()
+ else:
+ net.noise_off()
+ return noneiv_metrics, eiv_metrics
+
+
+collection_keys = ['rmse','logdens','bias','coverage_numerical','coverage_theory','coverage_normalized','res_std']
+noneiv_metrics_collection = {}
+eiv_metrics_collection = {}
+for key in collection_keys:
+ noneiv_metrics_collection[key] = []
+ eiv_metrics_collection[key] = []
num_test_epochs = 10
assert train_noneiv.seed_list == train_eiv.seed_list
seed_list = train_noneiv.seed_list
@@ -158,34 +166,23 @@ max_batch_number = 2
for seed in tqdm(seed_list):
train_data, test_data = load_data(seed=seed)
test_dataloader = DataLoader(test_data,
- batch_size=int(np.max((len(test_data),
+ batch_size=int(np.min((len(test_data),
800))), shuffle=True)
for i in tqdm(range(num_test_epochs)):
for j, (x,y) in enumerate(test_dataloader):
if j > max_batch_number:
break
- noneiv_rmse, noneiv_logdens, noneiv_bias, \
- eiv_rmse, eiv_logdens, eiv_bias =\
- collect_metrics(x,y, seed=seed)
- noneiv_rmse_collection.append(noneiv_rmse)
- noneiv_logdens_collection.append(noneiv_logdens)
- noneiv_bias_collection.append(noneiv_bias)
- eiv_rmse_collection.append(eiv_rmse)
- eiv_logdens_collection.append(eiv_logdens)
- eiv_bias_collection.append(eiv_bias)
-
-
-print('Non-EiV')
-print(f'RMSE {np.mean(noneiv_rmse_collection):.5f}'\
- f'({np.std(noneiv_rmse_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
-print(f'LogDens {np.mean(noneiv_logdens_collection):.5f}'\
- f'({np.std(noneiv_logdens_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
-print(f'Bias {np.mean(noneiv_bias_collection):.5f}'\
- f'({np.std(noneiv_bias_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
-print('EiV')
-print(f'RMSE {np.mean(eiv_rmse_collection):.5f}'\
- f'({np.std(eiv_rmse_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
-print(f'LogDens {np.mean(eiv_logdens_collection):.5f}'\
- f'({np.std(eiv_logdens_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
-print(f'Bias {np.mean(eiv_bias_collection):.5f}'\
- f'({np.std(eiv_bias_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
+
+ noneiv_metrics, eiv_metrics = collect_metrics(x,y, seed=seed)
+ for key in collection_keys:
+ noneiv_metrics_collection[key].append(noneiv_metrics[key])
+ eiv_metrics_collection[key].append(eiv_metrics[key])
+
+print('Non-EiV\n-----')
+for key in collection_keys:
+ print(f'{key} {np.mean(noneiv_metrics_collection[key]):.5f}'\
+ f'({np.std(noneiv_metrics_collection[key])/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
+print('EiV\n-----')
+for key in collection_keys:
+ print(f'{key} {np.mean(eiv_metrics_collection[key]):.5f}'\
+ f'({np.std(eiv_metrics_collection[key])/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
diff --git a/Experiments/train_eiv_california.py b/Experiments/train_eiv_california.py
index 39c95fb..f90a204 100644
--- a/Experiments/train_eiv_california.py
+++ b/Experiments/train_eiv_california.py
@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_eiv_concrete.py b/Experiments/train_eiv_concrete.py
index 0cc42d5..e637d35 100644
--- a/Experiments/train_eiv_concrete.py
+++ b/Experiments/train_eiv_concrete.py
@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_eiv_energy.py b/Experiments/train_eiv_energy.py
index 40be4b2..304501b 100644
--- a/Experiments/train_eiv_energy.py
+++ b/Experiments/train_eiv_energy.py
@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_eiv_kin8nm.py b/Experiments/train_eiv_kin8nm.py
index 58d191c..96b6c79 100644
--- a/Experiments/train_eiv_kin8nm.py
+++ b/Experiments/train_eiv_kin8nm.py
@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_eiv_msd.py b/Experiments/train_eiv_msd.py
index 16e617f..4b533b6 100644
--- a/Experiments/train_eiv_msd.py
+++ b/Experiments/train_eiv_msd.py
@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_eiv_naval.py b/Experiments/train_eiv_naval.py
index 8db266c..f4a9aa7 100644
--- a/Experiments/train_eiv_naval.py
+++ b/Experiments/train_eiv_naval.py
@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_eiv_power.py b/Experiments/train_eiv_power.py
index 2ddf75a..d9be09e 100644
--- a/Experiments/train_eiv_power.py
+++ b/Experiments/train_eiv_power.py
@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_eiv_protein.py b/Experiments/train_eiv_protein.py
index 3801d5e..625fb6a 100644
--- a/Experiments/train_eiv_protein.py
+++ b/Experiments/train_eiv_protein.py
@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_eiv_wine.py b/Experiments/train_eiv_wine.py
index ca561ad..d6c9a41 100644
--- a/Experiments/train_eiv_wine.py
+++ b/Experiments/train_eiv_wine.py
@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_eiv_yacht.py b/Experiments/train_eiv_yacht.py
index 2c9ee88..10e2607 100644
--- a/Experiments/train_eiv_yacht.py
+++ b/Experiments/train_eiv_yacht.py
@@ -70,6 +70,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_noneiv_california.py b/Experiments/train_noneiv_california.py
index 773a30d..f359b43 100644
--- a/Experiments/train_noneiv_california.py
+++ b/Experiments/train_noneiv_california.py
@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_noneiv_concrete.py b/Experiments/train_noneiv_concrete.py
index d4a48a8..5e7c1e3 100644
--- a/Experiments/train_noneiv_concrete.py
+++ b/Experiments/train_noneiv_concrete.py
@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_noneiv_energy.py b/Experiments/train_noneiv_energy.py
index 106bc14..340b81a 100644
--- a/Experiments/train_noneiv_energy.py
+++ b/Experiments/train_noneiv_energy.py
@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_noneiv_kin8nm.py b/Experiments/train_noneiv_kin8nm.py
index 90af665..7070c38 100644
--- a/Experiments/train_noneiv_kin8nm.py
+++ b/Experiments/train_noneiv_kin8nm.py
@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_noneiv_msd.py b/Experiments/train_noneiv_msd.py
index f80c03e..b72644d 100644
--- a/Experiments/train_noneiv_msd.py
+++ b/Experiments/train_noneiv_msd.py
@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_noneiv_naval.py b/Experiments/train_noneiv_naval.py
index f96c66e..fb8c57f 100644
--- a/Experiments/train_noneiv_naval.py
+++ b/Experiments/train_noneiv_naval.py
@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_noneiv_power.py b/Experiments/train_noneiv_power.py
index c63b571..3c87a2d 100644
--- a/Experiments/train_noneiv_power.py
+++ b/Experiments/train_noneiv_power.py
@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_noneiv_protein.py b/Experiments/train_noneiv_protein.py
index ca73d6a..9e2d3f4 100644
--- a/Experiments/train_noneiv_protein.py
+++ b/Experiments/train_noneiv_protein.py
@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_noneiv_wine.py b/Experiments/train_noneiv_wine.py
index dc239f7..a445c1e 100644
--- a/Experiments/train_noneiv_wine.py
+++ b/Experiments/train_noneiv_wine.py
@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
diff --git a/Experiments/train_noneiv_yacht.py b/Experiments/train_noneiv_yacht.py
index d163e54..489a81c 100644
--- a/Experiments/train_noneiv_yacht.py
+++ b/Experiments/train_noneiv_yacht.py
@@ -69,6 +69,8 @@ class UpdatedTrainEpoch(train_and_store.TrainEpoch):
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
+ writer.add_scalar('std_y', self.last_std_y, self.total_count)
+ writer.add_scalar('RMSE:std_y', rmse/self.last_std_y, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
--
GitLab