diff --git a/Experiments/evaluate_california.py b/Experiments/evaluate_california.py
new file mode 100644
index 0000000000000000000000000000000000000000..be631efaf4ca6cd6851eebeac5a5c5dcc7d13e8d
--- /dev/null
+++ b/Experiments/evaluate_california.py
@@ -0,0 +1,58 @@
+import os
+import numpy as np
+import torch
+import torch.backends.cudnn
+from torch.utils.data import DataLoader
+from torch.utils.tensorboard.writer import SummaryWriter
+
+from EIVArchitectures import Networks, initialize_weights
+from EIVData.california_housing import load_data
+from EIVTrainingRoutines import train_and_store, loss_functions
+
+from train_noneiv_california import p, init_std_y_list, seed_list, unscaled_reg, hidden_layers
+
+
+train_data, test_data = load_data()
+test_dataloader = DataLoader(test_data, batch_size=int(np.max((len(test_data), 800))))
+
+seed = seed_list[0]
+init_std_y = init_std_y_list[0]
+saved_file = os.path.join('saved_networks',
+ f'noneiv_california'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_p_{p:.2f}_seed_{seed}.pkl')
+
+input_dim = train_data[0][0].numel()
+output_dim = train_data[0][1].numel()
+net = Networks.FNNBer(p=p, init_std_y=init_std_y,
+ h=[input_dim, *hidden_layers, output_dim])
+train_and_store.open_stored_training(saved_file=saved_file,
+ net=net)
+
+
+# RMSE
+x,y = next(iter(test_dataloader))
+out = net(x)[0]
+if len(y.shape) <=1:
+ y = y.view((-1,1))
+assert y.shape == out.shape
+res = y-out
+scale = train_data.dataset.std_labels
+scaled_res = res * scale.view((1,-1))
+scaled_res = scaled_res.detach().cpu().numpy().flatten()
+rmse = np.sqrt(np.mean(scaled_res**2))
+print(f'RMSE {rmse:.3f}')
+
+
+# NLL
+x,y = next(iter(test_dataloader))
+training_state = net.training
+net.train()
+logdens = net.predictive_logdensity(x, y, number_of_draws=100,
+ decouple_dimensions=True,
+ scale_labels=train_data.dataset.std_labels.view((-1,))).mean()
+if training_state:
+ net.train()
+else:
+ net.eval()
+print(f'Dropout predictive {logdens:.3f}')
diff --git a/Experiments/evaluate_energy.py b/Experiments/evaluate_energy.py
index 548596afccefd1336fa6e034f33252ed3003a6cb..e9d74e3d6f9351d54221a41ef51c7b8d4f4f318c 100644
--- a/Experiments/evaluate_energy.py
+++ b/Experiments/evaluate_energy.py
@@ -19,7 +19,7 @@ seed = seed_list[0]
init_std_y = init_std_y_list[0]
saved_file = os.path.join('saved_networks',
f'noneiv_energy'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
input_dim = train_data[0][0].numel()
diff --git a/Experiments/evaluate_kin8nm.py b/Experiments/evaluate_kin8nm.py
index e11f441d81a1b394b1833935333d3e92a428d370..a9d8ae6ab77700b1f32b127c89c5a2f9cf7c0ae2 100644
--- a/Experiments/evaluate_kin8nm.py
+++ b/Experiments/evaluate_kin8nm.py
@@ -19,7 +19,7 @@ seed = seed_list[0]
init_std_y = init_std_y_list[0]
saved_file = os.path.join('saved_networks',
f'noneiv_kin8nm'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
input_dim = train_data[0][0].numel()
diff --git a/Experiments/evaluate_msd.py b/Experiments/evaluate_msd.py
index abf951d86cd46743b6511d37b88afe6c035019fe..041b7556da3eb8d7f206284f20ec0baf4eec2167 100644
--- a/Experiments/evaluate_msd.py
+++ b/Experiments/evaluate_msd.py
@@ -19,7 +19,7 @@ seed = seed_list[0]
init_std_y = init_std_y_list[0]
saved_file = os.path.join('saved_networks',
f'noneiv_msd'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
input_dim = train_data[0][0].numel()
diff --git a/Experiments/evaluate_naval.py b/Experiments/evaluate_naval.py
index 55e22d3c4a6d780dece45a36dbd7fa19bf01ae71..63c88a34f6b7ec45193602838b2dc2568756f9ac 100644
--- a/Experiments/evaluate_naval.py
+++ b/Experiments/evaluate_naval.py
@@ -19,7 +19,7 @@ seed = seed_list[0]
init_std_y = init_std_y_list[0]
saved_file = os.path.join('saved_networks',
f'noneiv_naval'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
input_dim = train_data[0][0].numel()
diff --git a/Experiments/evaluate_power.py b/Experiments/evaluate_power.py
index fdf3e6e5f59baccd530e19201a1686d4a231a28b..bc4d328af5cd624d5e2b56b66fd5dbbe4ba08d4a 100644
--- a/Experiments/evaluate_power.py
+++ b/Experiments/evaluate_power.py
@@ -19,7 +19,7 @@ seed = seed_list[0]
init_std_y = init_std_y_list[0]
saved_file = os.path.join('saved_networks',
f'noneiv_power'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
input_dim = train_data[0][0].numel()
diff --git a/Experiments/evaluate_protein.py b/Experiments/evaluate_protein.py
index 985eee2e9c81df3b385a7dad813cdb147eb3c81d..de32d3cfbb99e949cda31e1451ec2e827adf4c66 100644
--- a/Experiments/evaluate_protein.py
+++ b/Experiments/evaluate_protein.py
@@ -19,7 +19,7 @@ seed = seed_list[0]
init_std_y = init_std_y_list[0]
saved_file = os.path.join('saved_networks',
f'noneiv_protein'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
input_dim = train_data[0][0].numel()
diff --git a/Experiments/evaluate_wine.py b/Experiments/evaluate_wine.py
index 3031b8483b8ff380554eb1bfba649f5eb9206c7e..02be10de8141a299c02450b91deb87312aca4fb7 100644
--- a/Experiments/evaluate_wine.py
+++ b/Experiments/evaluate_wine.py
@@ -19,7 +19,7 @@ seed = seed_list[0]
init_std_y = init_std_y_list[0]
saved_file = os.path.join('saved_networks',
f'noneiv_wine'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
input_dim = train_data[0][0].numel()
diff --git a/Experiments/evaluate_yacht.py b/Experiments/evaluate_yacht.py
index 6db62c48594069597f9e009ab3e9f0903263531b..842674dd5643d7032b32b487b0ac90c558121d72 100644
--- a/Experiments/evaluate_yacht.py
+++ b/Experiments/evaluate_yacht.py
@@ -19,7 +19,7 @@ seed = seed_list[0]
init_std_y = init_std_y_list[0]
saved_file = os.path.join('saved_networks',
f'noneiv_yacht'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
input_dim = train_data[0][0].numel()
diff --git a/Experiments/train_noneiv_california.py b/Experiments/train_noneiv_california.py
index 87ad6c4ca5295ff0dc4fce19d1905df68fba286e..773a30ddf5d50d3a7d2fff2c58e6a9579ba24992 100644
--- a/Experiments/train_noneiv_california.py
+++ b/Experiments/train_noneiv_california.py
@@ -36,7 +36,7 @@ def set_seeds(seed):
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
-seed_list = [0,]
+seed_list = range(10)
# to store the RMSE
rmse_chain = []
@@ -125,7 +125,7 @@ def train_on_data(init_std_y, seed):
# run and save
save_file = os.path.join('saved_networks',
f'noneiv_california'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
train_and_store.train_and_store(net=net,
epoch_map=epoch_map,
diff --git a/Experiments/train_noneiv_concrete.py b/Experiments/train_noneiv_concrete.py
index 625db7da14c08f8dd93a9126dbbbc6ca0ea3a40f..d4a48a8dc6fc7c0273ea4f894d63d38acc931d7a 100644
--- a/Experiments/train_noneiv_concrete.py
+++ b/Experiments/train_noneiv_concrete.py
@@ -36,7 +36,7 @@ def set_seeds(seed):
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
-seed_list = [0,]
+seed_list = range(10)
# to store the RMSE
rmse_chain = []
@@ -125,7 +125,7 @@ def train_on_data(init_std_y, seed):
# run and save
save_file = os.path.join('saved_networks',
f'noneiv_concrete'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
train_and_store.train_and_store(net=net,
epoch_map=epoch_map,
diff --git a/Experiments/train_noneiv_energy.py b/Experiments/train_noneiv_energy.py
index a635741c8a28a0b2f4806df4020d78a951ddc97a..04f299af8f0c92aeffd393f03379319844c248c8 100644
--- a/Experiments/train_noneiv_energy.py
+++ b/Experiments/train_noneiv_energy.py
@@ -36,7 +36,7 @@ def set_seeds(seed):
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
-seed_list = [0,]
+seed_list = range(10)
# to store the RMSE
rmse_chain = []
@@ -125,7 +125,7 @@ def train_on_data(init_std_y, seed):
# run and save
save_file = os.path.join('saved_networks',
f'noneiv_energy'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
train_and_store.train_and_store(net=net,
epoch_map=epoch_map,
diff --git a/Experiments/train_noneiv_kin8nm.py b/Experiments/train_noneiv_kin8nm.py
index 0baa2572ec56d985a5538b42c151d8989c645c95..90af665059e0eca165cfb97fd991ba877fcfbad0 100644
--- a/Experiments/train_noneiv_kin8nm.py
+++ b/Experiments/train_noneiv_kin8nm.py
@@ -36,7 +36,7 @@ def set_seeds(seed):
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
-seed_list = [0,]
+seed_list = range(10)
# to store the RMSE
rmse_chain = []
@@ -125,7 +125,7 @@ def train_on_data(init_std_y, seed):
# run and save
save_file = os.path.join('saved_networks',
f'noneiv_kin8nm'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
train_and_store.train_and_store(net=net,
epoch_map=epoch_map,
diff --git a/Experiments/train_noneiv_msd.py b/Experiments/train_noneiv_msd.py
index a38b5215ffc0d6827290abe44160e50f1c0b4bc8..f80c03e44ba37c0028806725e2beed1001ab0699 100644
--- a/Experiments/train_noneiv_msd.py
+++ b/Experiments/train_noneiv_msd.py
@@ -36,7 +36,7 @@ def set_seeds(seed):
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
-seed_list = [0,]
+seed_list = range(10)
# to store the RMSE
rmse_chain = []
@@ -125,7 +125,7 @@ def train_on_data(init_std_y, seed):
# run and save
save_file = os.path.join('saved_networks',
f'noneiv_msd'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
train_and_store.train_and_store(net=net,
epoch_map=epoch_map,
diff --git a/Experiments/train_noneiv_naval.py b/Experiments/train_noneiv_naval.py
index 2b03843f8c9f4277bf9f67b55329a58a8aa2ca3f..f96c66e1c20425042726f3c22c27285ed8b085d1 100644
--- a/Experiments/train_noneiv_naval.py
+++ b/Experiments/train_noneiv_naval.py
@@ -36,7 +36,7 @@ def set_seeds(seed):
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
-seed_list = [0,]
+seed_list = range(10)
# to store the RMSE
rmse_chain = []
@@ -125,7 +125,7 @@ def train_on_data(init_std_y, seed):
# run and save
save_file = os.path.join('saved_networks',
f'noneiv_naval'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
train_and_store.train_and_store(net=net,
epoch_map=epoch_map,
diff --git a/Experiments/train_noneiv_power.py b/Experiments/train_noneiv_power.py
index 14ba1200a6f870f426c2f2fa0be4c27773b7b486..c63b5713cb6fd1a8dd1b9d0ea0d5e6a4bcec510f 100644
--- a/Experiments/train_noneiv_power.py
+++ b/Experiments/train_noneiv_power.py
@@ -36,7 +36,7 @@ def set_seeds(seed):
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
-seed_list = [0,]
+seed_list = range(10)
# to store the RMSE
rmse_chain = []
@@ -125,7 +125,7 @@ def train_on_data(init_std_y, seed):
# run and save
save_file = os.path.join('saved_networks',
f'noneiv_power'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
train_and_store.train_and_store(net=net,
epoch_map=epoch_map,
diff --git a/Experiments/train_noneiv_protein.py b/Experiments/train_noneiv_protein.py
index 55db23c306d5e46f07926edd9ba53f2aa5b2d759..ca73d6af733bffd469da898d8bd43c958068586b 100644
--- a/Experiments/train_noneiv_protein.py
+++ b/Experiments/train_noneiv_protein.py
@@ -36,7 +36,7 @@ def set_seeds(seed):
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
-seed_list = [0,]
+seed_list = range(10)
# to store the RMSE
rmse_chain = []
@@ -125,7 +125,7 @@ def train_on_data(init_std_y, seed):
# run and save
save_file = os.path.join('saved_networks',
f'noneiv_protein'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
train_and_store.train_and_store(net=net,
epoch_map=epoch_map,
diff --git a/Experiments/train_noneiv_wine.py b/Experiments/train_noneiv_wine.py
index 837eb720461dbc35267f1cd46a05e86699e86f09..dc239f74fda492662853d92859e355585f476db0 100644
--- a/Experiments/train_noneiv_wine.py
+++ b/Experiments/train_noneiv_wine.py
@@ -36,7 +36,7 @@ def set_seeds(seed):
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
-seed_list = [0,]
+seed_list = range(10)
# to store the RMSE
rmse_chain = []
@@ -125,7 +125,7 @@ def train_on_data(init_std_y, seed):
# run and save
save_file = os.path.join('saved_networks',
f'noneiv_wine'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
train_and_store.train_and_store(net=net,
epoch_map=epoch_map,
diff --git a/Experiments/train_noneiv_yacht.py b/Experiments/train_noneiv_yacht.py
index 7640817821e90ec82854ac9b92db7ad4291ad744..d163e544f50bdc09283d138ba7149b5290f2a515 100644
--- a/Experiments/train_noneiv_yacht.py
+++ b/Experiments/train_noneiv_yacht.py
@@ -36,7 +36,7 @@ def set_seeds(seed):
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
-seed_list = [0,]
+seed_list = range(10)
# to store the RMSE
rmse_chain = []
@@ -125,7 +125,7 @@ def train_on_data(init_std_y, seed):
# run and save
save_file = os.path.join('saved_networks',
f'noneiv_yacht'\
- f'init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
+ f'_init_std_y_{init_std_y:.3f}_ureg_{unscaled_reg:.1f}'\
f'_p_{p:.2f}_seed_{seed}.pkl')
train_and_store.train_and_store(net=net,
epoch_map=epoch_map,
diff --git a/README.md b/README.md
index 912aa64caf3702a6adcf85f7b93cb634531ad3f2..7551f09f52640d3d685e875c5b36ddd3d99c1fef 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@ This directory lists the source code for the article `Errors-in-Variables for de
## Requirements
-The software used to produce the results from the preprint was written in [Python 3](https://www.python.org/). If not already installed, the easiest way to set up Python is usually via [Anaconda](https://www.anaconda.com/). To use the software, the installation of some additional packages is required. This is discussed below. To avoid any global impacts on the Python install, especially if the system interpreter is used, it might be preferable to do the following in a virtual environment, either in [Anaconda](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html) or by using the [venv](https://docs.python.org/3/tutorial/venv.html) module. The Python version used for the results in the preprint is 3.8.5.
+The software used to produce the results from the preprint was written in [Python 3](https://www.python.org/). If not already installed, the easiest way to set up Python is usually via [Anaconda](https://www.anaconda.com/). To use the software, the installation of some additional packages is required. This is discussed below. To avoid any global impacts on the Python install, especially if the system interpreter is used, it might be preferable to do the following in a virtual environment, either in [Anaconda](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html) or by using the [venv](https://docs.python.org/3/tutorial/venv.html) module. The Python version used for the results in the preprint is 3.9.7.
### Installing additional packages (except PyTorch)
The Python packages to use this software, except for PyTorch which we will discuss below, can be installed by using the file `requirements.txt`
@@ -13,7 +13,7 @@ The Python packages to use this software, except for PyTorch which we will discu
pip install -r requirements.txt
```
-When using Anaconda, make sure that `python` is installed in the virtual environment. If not, use `conda install python` or `conda install python=3.8.5` before running the `pip` command.
+When using Anaconda, make sure that `python` is installed in the virtual environment. If not, use `conda install python` or `conda install python=3.9.7` before running the `pip` command.
**When running into version issues**: