Skip to content
Snippets Groups Projects
Commit 1e446fa4 authored by Jörg Martin's avatar Jörg Martin
Browse files

Added EiV for concrete, kin8nm and naval 

Added EiV training scripts for the three datasets and moreover included
bias evaluation in `evaluate_tabular`. The inclusion of some coverage
measure is still needed.
parent 3d20eacb
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
EIVPackage/EIVData/concrete_strength.py
+ 1
− 1
View file @ 1e446fa4
......@@ -12,7 +12,7 @@ def load_data(seed=0, splitting_part=0.8, normalize=True):
:normalize: Whether to normalize the data, defaults to True.
:returns: concrete_trainset, concrete_testset
"""
concrete_dataset = CSVData('~/SharedData/AI/datasets/concrete_compression_strength/compresive_strength_concrete.csv',
concrete_dataset = CSVData('~/SharedData/AI/datasets/concrete_compression_strength/compressive_strength_concrete.csv',
class_name='Concrete compressive strength(MPa, megapascals) ',
shuffle_seed=seed,
normalize=normalize)
......
Experiments/evaluate_tabular.py
+ 23
− 7
View file @ 1e446fa4
......@@ -26,7 +26,7 @@ input_dim = train_data[0][0].numel()
output_dim = train_data[0][1].numel()
def collect_metrics(x,y, seed=0,
noneiv_number_of_draws=100, eiv_number_of_draws=[100,1],
noneiv_number_of_draws=100, eiv_number_of_draws=[100,5],
decouple_dimensions=False, device=torch.device('cuda:1')):
"""
:param x: A torch.tensor, taken as input
......@@ -40,7 +40,8 @@ def collect_metrics(x,y, seed=0,
of Gal et al. is followed where, in the evaluation of the
log-posterior-predictive, each dimension is treated independently and then
averaged. If False (default), a multivariate distribution is used.
:returns: noneiv_rmse, noneiv_logdens,eiv_rmse, eiv_logdens
:returns: noneiv_rmse, noneiv_logdens, noneiv_bias,
eiv_rmse, eiv_logdens, eiv_bias
"""
x,y = x.to(device), y.to(device)
init_std_y = train_noneiv.init_std_y_list[0]
......@@ -70,6 +71,7 @@ def collect_metrics(x,y, seed=0,
scaled_res = res * scale.view((1,-1))
scaled_res = scaled_res.detach().cpu().numpy().flatten()
noneiv_rmse = np.sqrt(np.mean(scaled_res**2))
noneiv_bias = np.mean(scaled_res)
# NLL
......@@ -92,7 +94,7 @@ def collect_metrics(x,y, seed=0,
hidden_layers = train_eiv.hidden_layers
fixed_std_x = train_eiv.fixed_std_x
saved_file = os.path.join('saved_networks',
f'eiv_energy'\
f'eiv_{short_dataname}'\
f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_fixed_std_x_{fixed_std_x:.3f}'\
f'_seed_{seed}.pkl')
......@@ -116,6 +118,7 @@ def collect_metrics(x,y, seed=0,
scaled_res = res * scale.view((1,-1))
scaled_res = scaled_res.detach().cpu().numpy().flatten()
eiv_rmse = np.sqrt(np.mean(scaled_res**2))
eiv_bias = np.mean(scaled_res)
if training_state:
net.train()
else:
......@@ -139,28 +142,37 @@ def collect_metrics(x,y, seed=0,
net.train()
else:
net.eval()
return noneiv_rmse, noneiv_logdens, eiv_rmse, eiv_logdens
return noneiv_rmse, noneiv_logdens, noneiv_bias, \
eiv_rmse, eiv_logdens, eiv_bias
noneiv_rmse_collection = []
noneiv_logdens_collection = []
noneiv_bias_collection = []
eiv_rmse_collection = []
eiv_logdens_collection = []
num_test_epochs = 30
eiv_bias_collection = []
num_test_epochs = 10
assert train_noneiv.seed_list == train_eiv.seed_list
seed_list = train_noneiv.seed_list
max_batch_number = 2
for seed in tqdm(seed_list):
train_data, test_data = load_data(seed=seed)
test_dataloader = DataLoader(test_data,
batch_size=int(np.max((len(test_data),
800))), shuffle=True)
for i in tqdm(range(num_test_epochs)):
for x,y in test_dataloader:
noneiv_rmse, noneiv_logdens, eiv_rmse, eiv_logdens =\
for j, (x,y) in enumerate(test_dataloader):
if j > max_batch_number:
break
noneiv_rmse, noneiv_logdens, noneiv_bias, \
eiv_rmse, eiv_logdens, eiv_bias =\
collect_metrics(x,y, seed=seed)
noneiv_rmse_collection.append(noneiv_rmse)
noneiv_logdens_collection.append(noneiv_logdens)
noneiv_bias_collection.append(noneiv_bias)
eiv_rmse_collection.append(eiv_rmse)
eiv_logdens_collection.append(eiv_logdens)
eiv_bias_collection.append(eiv_bias)
print('Non-EiV')
......@@ -168,8 +180,12 @@ print(f'RMSE {np.mean(noneiv_rmse_collection):.5f}'\
f'({np.std(noneiv_rmse_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
print(f'LogDens {np.mean(noneiv_logdens_collection):.5f}'\
f'({np.std(noneiv_logdens_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
print(f'Bias {np.mean(noneiv_bias_collection):.5f}'\
f'({np.std(noneiv_bias_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
print('EiV')
print(f'RMSE {np.mean(eiv_rmse_collection):.5f}'\
f'({np.std(eiv_rmse_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
print(f'LogDens {np.mean(eiv_logdens_collection):.5f}'\
f'({np.std(eiv_logdens_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
print(f'Bias {np.mean(eiv_bias_collection):.5f}'\
f'({np.std(eiv_bias_collection)/np.sqrt(num_test_epochs*len(seed_list)):.5f})')
Experiments/train_eiv_concrete.py 0 → 100644
+ 155
− 0
View file @ 1e446fa4
"""
Train EiV model on concrete strength dataset using different seeds
"""
import random
import os
import numpy as np
import torch
import torch.backends.cudnn
from torch.utils.data import DataLoader
from torch.utils.tensorboard.writer import SummaryWriter
from EIVArchitectures import Networks, initialize_weights
from EIVData.concrete_strength import load_data
from EIVTrainingRoutines import train_and_store, loss_functions
# hyperparameters
lr = 1e-3
batch_size = 32
test_batch_size = 800
number_of_epochs = 100
unscaled_reg = 10
report_point = 5
p = 0.2
lr_update = 20
# pretraining = 300
epoch_offset = 10
init_std_y_list = [0.5]
gamma = 0.5
hidden_layers = [1024, 1024, 1024, 1024]
device = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu')
fixed_std_x = 0.05
# reproducability
def set_seeds(seed):
torch.backends.cudnn.benchmark = False
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
seed_list = range(10)
# to store the RMSE
rmse_chain = []
class UpdatedTrainEpoch(train_and_store.TrainEpoch):
def pre_epoch_update(self, net, epoch):
"""
Overwrites the corresponding method
"""
if epoch == 0:
self.lr = self.initial_lr
self.optimizer = torch.optim.Adam(net.parameters(), lr=self.lr)
self.lr_scheduler = torch.optim.lr_scheduler.StepLR(
self.optimizer, lr_update, gamma)
def post_epoch_update(self, net, epoch):
"""
Overwrites the corresponding method
"""
if epoch >= epoch_offset:
net.std_y_par.requires_grad = True
self.lr_scheduler.step()
def extra_report(self, net, i):
"""
Overwrites the corresponding method
and fed after initialization of this class
"""
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
def rmse(self, net):
"""
Compute the root mean squared error for `net`
"""
net_train_state = net.training
net_noise_state = net.noise_is_on
net.eval()
net.noise_off()
x, y = next(iter(self.test_dataloader))
if len(y.shape) <= 1:
y = y.view((-1,1))
out = net(x.to(device))[0].detach().cpu()
assert out.shape == y.shape
if net_train_state:
net.train()
if net_noise_state:
net.noise_on()
return torch.sqrt(torch.mean((out-y)**2))
def train_on_data(init_std_y, seed):
"""
Sets `seed`, loads data and trains an Bernoulli Modell, starting with
`init_std_y`.
"""
# set seed
set_seeds(seed)
# load Datasets
train_data, test_data = load_data(seed=seed, splitting_part=0.8,
normalize=True)
# make dataloaders
train_dataloader = DataLoader(train_data, batch_size=batch_size,
shuffle=True)
test_dataloader = DataLoader(test_data, batch_size=test_batch_size,
shuffle=True)
# create a net
input_dim = train_data[0][0].numel()
output_dim = train_data[0][1].numel()
net = Networks.FNNEIV(p=p,
init_std_y=init_std_y,
h=[input_dim, *hidden_layers, output_dim],
fixed_std_x=fixed_std_x)
net.apply(initialize_weights.glorot_init)
net = net.to(device)
net.std_y_par.requires_grad = False
std_x_map = lambda: net.get_std_x().detach().cpu().item()
std_y_map = lambda: net.get_std_y().detach().cpu().item()
# regularization
reg = unscaled_reg/len(train_data)
# create epoch_map
criterion = loss_functions.nll_eiv
epoch_map = UpdatedTrainEpoch(train_dataloader=train_dataloader,
test_dataloader=test_dataloader,
criterion=criterion, std_y_map=std_y_map, std_x_map=std_x_map,
lr=lr, reg=reg, report_point=report_point, device=device)
# run and save
save_file = os.path.join('saved_networks',
f'eiv_concrete'\
f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_fixed_std_x_{fixed_std_x:.3f}'\
f'_seed_{seed}.pkl')
train_and_store.train_and_store(net=net,
epoch_map=epoch_map,
number_of_epochs=number_of_epochs,
save_file=save_file)
if __name__ == '__main__':
for seed in seed_list:
# Tensorboard monitoring
writer = SummaryWriter(log_dir=f'/home/martin09/tmp/tensorboard/'\
f'run_eiv_concrete_lr_{lr:.4f}_seed'\
f'_{seed}_uregu_{unscaled_reg:.1f}_p_{p:.2f}'\
f'_fixed_std_x_{fixed_std_x:.3f}')
print(f'>>>>SEED: {seed}')
for init_std_y in init_std_y_list:
print(f'Using init_std_y={init_std_y:.3f}')
train_on_data(init_std_y, seed)
Experiments/train_eiv_kin8nm.py 0 → 100644
+ 155
− 0
View file @ 1e446fa4
"""
Train EiV model on the kin8nm dataset using different seeds
"""
import random
import os
import numpy as np
import torch
import torch.backends.cudnn
from torch.utils.data import DataLoader
from torch.utils.tensorboard.writer import SummaryWriter
from EIVArchitectures import Networks, initialize_weights
from EIVData.kin8nm import load_data
from EIVTrainingRoutines import train_and_store, loss_functions
# hyperparameters
lr = 1e-3
batch_size = 32
test_batch_size = 600
number_of_epochs = 30
unscaled_reg = 10
report_point = 5
p = 0.2
lr_update = 20
# pretraining = 300
epoch_offset = 19
init_std_y_list = [0.5]
gamma = 0.5
hidden_layers = [1024, 1024, 1024, 1024]
device = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu')
fixed_std_x = 0.05
# reproducability
def set_seeds(seed):
torch.backends.cudnn.benchmark = False
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
seed_list = range(10)
# to store the RMSE
rmse_chain = []
class UpdatedTrainEpoch(train_and_store.TrainEpoch):
def pre_epoch_update(self, net, epoch):
"""
Overwrites the corresponding method
"""
if epoch == 0:
self.lr = self.initial_lr
self.optimizer = torch.optim.Adam(net.parameters(), lr=self.lr)
self.lr_scheduler = torch.optim.lr_scheduler.StepLR(
self.optimizer, lr_update, gamma)
def post_epoch_update(self, net, epoch):
"""
Overwrites the corresponding method
"""
if epoch >= epoch_offset:
net.std_y_par.requires_grad = True
self.lr_scheduler.step()
def extra_report(self, net, i):
"""
Overwrites the corresponding method
and fed after initialization of this class
"""
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
def rmse(self, net):
"""
Compute the root mean squared error for `net`
"""
net_train_state = net.training
net_noise_state = net.noise_is_on
net.eval()
net.noise_off()
x, y = next(iter(self.test_dataloader))
if len(y.shape) <= 1:
y = y.view((-1,1))
out = net(x.to(device))[0].detach().cpu()
assert out.shape == y.shape
if net_train_state:
net.train()
if net_noise_state:
net.noise_on()
return torch.sqrt(torch.mean((out-y)**2))
def train_on_data(init_std_y, seed):
"""
Sets `seed`, loads data and trains an Bernoulli Modell, starting with
`init_std_y`.
"""
# set seed
set_seeds(seed)
# load Datasets
train_data, test_data = load_data(seed=seed, splitting_part=0.8,
normalize=True)
# make dataloaders
train_dataloader = DataLoader(train_data, batch_size=batch_size,
shuffle=True)
test_dataloader = DataLoader(test_data, batch_size=test_batch_size,
shuffle=True)
# create a net
input_dim = train_data[0][0].numel()
output_dim = train_data[0][1].numel()
net = Networks.FNNEIV(p=p,
init_std_y=init_std_y,
h=[input_dim, *hidden_layers, output_dim],
fixed_std_x=fixed_std_x)
net.apply(initialize_weights.glorot_init)
net = net.to(device)
net.std_y_par.requires_grad = False
std_x_map = lambda: net.get_std_x().detach().cpu().item()
std_y_map = lambda: net.get_std_y().detach().cpu().item()
# regularization
reg = unscaled_reg/len(train_data)
# create epoch_map
criterion = loss_functions.nll_eiv
epoch_map = UpdatedTrainEpoch(train_dataloader=train_dataloader,
test_dataloader=test_dataloader,
criterion=criterion, std_y_map=std_y_map, std_x_map=std_x_map,
lr=lr, reg=reg, report_point=report_point, device=device)
# run and save
save_file = os.path.join('saved_networks',
f'eiv_kin8nm'\
f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_fixed_std_x_{fixed_std_x:.3f}'\
f'_seed_{seed}.pkl')
train_and_store.train_and_store(net=net,
epoch_map=epoch_map,
number_of_epochs=number_of_epochs,
save_file=save_file)
if __name__ == '__main__':
for seed in seed_list:
# Tensorboard monitoring
writer = SummaryWriter(log_dir=f'/home/martin09/tmp/tensorboard/'\
f'run_eiv_kin8nm_lr_{lr:.4f}_seed'\
f'_{seed}_uregu_{unscaled_reg:.1f}_p_{p:.2f}'\
f'_fixed_std_x_{fixed_std_x:.3f}')
print(f'>>>>SEED: {seed}')
for init_std_y in init_std_y_list:
print(f'Using init_std_y={init_std_y:.3f}')
train_on_data(init_std_y, seed)
Experiments/train_eiv_naval.py 0 → 100644
+ 155
− 0
View file @ 1e446fa4
"""
Train EiV model on the naval propulsion dataset using different seeds
"""
import random
import os
import numpy as np
import torch
import torch.backends.cudnn
from torch.utils.data import DataLoader
from torch.utils.tensorboard.writer import SummaryWriter
from EIVArchitectures import Networks, initialize_weights
from EIVData.naval_propulsion import load_data
from EIVTrainingRoutines import train_and_store, loss_functions
# hyperparameters
lr = 1e-3
batch_size = 32
test_batch_size = 600
number_of_epochs = 30
unscaled_reg = 10
report_point = 5
p = 0.2
lr_update = 20
# pretraining = 300
epoch_offset = 20
init_std_y_list = [0.5]
gamma = 0.5
hidden_layers = [1024, 1024, 1024, 1024]
device = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu')
fixed_std_x = 0.05
# reproducability
def set_seeds(seed):
torch.backends.cudnn.benchmark = False
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
seed_list = range(10)
# to store the RMSE
rmse_chain = []
class UpdatedTrainEpoch(train_and_store.TrainEpoch):
def pre_epoch_update(self, net, epoch):
"""
Overwrites the corresponding method
"""
if epoch == 0:
self.lr = self.initial_lr
self.optimizer = torch.optim.Adam(net.parameters(), lr=self.lr)
self.lr_scheduler = torch.optim.lr_scheduler.StepLR(
self.optimizer, lr_update, gamma)
def post_epoch_update(self, net, epoch):
"""
Overwrites the corresponding method
"""
if epoch >= epoch_offset:
net.std_y_par.requires_grad = True
self.lr_scheduler.step()
def extra_report(self, net, i):
"""
Overwrites the corresponding method
and fed after initialization of this class
"""
rmse = self.rmse(net).item()
rmse_chain.append(rmse)
writer.add_scalar('RMSE', rmse, self.total_count)
writer.add_scalar('train loss', self.last_train_loss, self.total_count)
writer.add_scalar('test loss', self.last_test_loss, self.total_count)
print(f'RMSE {rmse:.3f}')
def rmse(self, net):
"""
Compute the root mean squared error for `net`
"""
net_train_state = net.training
net_noise_state = net.noise_is_on
net.eval()
net.noise_off()
x, y = next(iter(self.test_dataloader))
if len(y.shape) <= 1:
y = y.view((-1,1))
out = net(x.to(device))[0].detach().cpu()
assert out.shape == y.shape
if net_train_state:
net.train()
if net_noise_state:
net.noise_on()
return torch.sqrt(torch.mean((out-y)**2))
def train_on_data(init_std_y, seed):
"""
Sets `seed`, loads data and trains an Bernoulli Modell, starting with
`init_std_y`.
"""
# set seed
set_seeds(seed)
# load Datasets
train_data, test_data = load_data(seed=seed, splitting_part=0.8,
normalize=True)
# make dataloaders
train_dataloader = DataLoader(train_data, batch_size=batch_size,
shuffle=True)
test_dataloader = DataLoader(test_data, batch_size=test_batch_size,
shuffle=True)
# create a net
input_dim = train_data[0][0].numel()
output_dim = train_data[0][1].numel()
net = Networks.FNNEIV(p=p,
init_std_y=init_std_y,
h=[input_dim, *hidden_layers, output_dim],
fixed_std_x=fixed_std_x)
net.apply(initialize_weights.glorot_init)
net = net.to(device)
net.std_y_par.requires_grad = False
std_x_map = lambda: net.get_std_x().detach().cpu().item()
std_y_map = lambda: net.get_std_y().detach().cpu().item()
# regularization
reg = unscaled_reg/len(train_data)
# create epoch_map
criterion = loss_functions.nll_eiv
epoch_map = UpdatedTrainEpoch(train_dataloader=train_dataloader,
test_dataloader=test_dataloader,
criterion=criterion, std_y_map=std_y_map, std_x_map=std_x_map,
lr=lr, reg=reg, report_point=report_point, device=device)
# run and save
save_file = os.path.join('saved_networks',
f'eiv_naval'\
f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_fixed_std_x_{fixed_std_x:.3f}'\
f'_seed_{seed}.pkl')
train_and_store.train_and_store(net=net,
epoch_map=epoch_map,
number_of_epochs=number_of_epochs,
save_file=save_file)
if __name__ == '__main__':
for seed in seed_list:
# Tensorboard monitoring
writer = SummaryWriter(log_dir=f'/home/martin09/tmp/tensorboard/'\
f'run_eiv_naval_lr_{lr:.4f}_seed'\
f'_{seed}_uregu_{unscaled_reg:.1f}_p_{p:.2f}'\
f'_fixed_std_x_{fixed_std_x:.3f}')
print(f'>>>>SEED: {seed}')
for init_std_y in init_std_y_list:
print(f'Using init_std_y={init_std_y:.3f}')
train_on_data(init_std_y, seed)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment