From 8b1adc0fe1f06c0a8c62b409038c0df59c630d0a Mon Sep 17 00:00:00 2001 From: Joerg Martin <joerg.martin@ptb.de> Date: Mon, 7 Feb 2022 15:33:39 +0000 Subject: [PATCH] simulated datasets revised to show EiV effect --- EIVPackage/EIVData/cubic.py | 4 ++-- EIVPackage/EIVData/linear.py | 2 +- EIVPackage/EIVData/sine.py | 4 ++-- Experiments/configurations/eiv_cubic.json | 4 ++-- Experiments/configurations/eiv_linear.json | 2 +- Experiments/configurations/eiv_quadratic.json | 2 +- Experiments/configurations/eiv_sine.json | 4 ++-- Experiments/configurations/noneiv_cubic.json | 2 +- Experiments/configurations/noneiv_linear.json | 2 +- Experiments/configurations/noneiv_quadratic.json | 2 +- Experiments/configurations/noneiv_sine.json | 2 +- Experiments/plot_coverage_vs_q.py | 7 +++---- Experiments/plot_summary.py | 6 +++--- 13 files changed, 21 insertions(+), 22 deletions(-) diff --git a/EIVPackage/EIVData/cubic.py b/EIVPackage/EIVData/cubic.py index dce6ea4..9d5e397 100644 --- a/EIVPackage/EIVData/cubic.py +++ b/EIVPackage/EIVData/cubic.py @@ -9,8 +9,8 @@ total_number_of_datapoints = 1000 input_range = [-1,1] slope = 1.0 intercept = 0.0 -x_noise_strength = 0.1 -y_noise_strength = 0.1 +x_noise_strength = 0.2 +y_noise_strength = 0.05 func = lambda true_x: slope * true_x**3 + intercept def load_data(seed=0, splitting_part=0.8, normalize=True, diff --git a/EIVPackage/EIVData/linear.py b/EIVPackage/EIVData/linear.py index 97d2cd9..2a1b0d7 100644 --- a/EIVPackage/EIVData/linear.py +++ b/EIVPackage/EIVData/linear.py @@ -10,7 +10,7 @@ input_range = [-1,1] slope = 1.0 intercept = 0.0 x_noise_strength = 0.1 -y_noise_strength = 0.1 +y_noise_strength = 0.2 func = lambda true_x: slope * true_x + intercept def load_data(seed=0, splitting_part=0.8, normalize=True, diff --git a/EIVPackage/EIVData/sine.py b/EIVPackage/EIVData/sine.py index cde80d3..2e9eabc 100644 --- a/EIVPackage/EIVData/sine.py +++ b/EIVPackage/EIVData/sine.py @@ -8,8 +8,8 @@ from EIVGeneral.manipulate_tensors import add_noise, normalize_tensor,\ total_number_of_datapoints = 2000 input_range = [-0.2,0.8] intercept = 0.0 -x_noise_strength = 0.02 -y_noise_strength = 0.02 +x_noise_strength = 0.04 +y_noise_strength = 0.01 func = lambda true_x: true_x +\ torch.sin(2 * torch.pi * true_x) +\ torch.sin(4 * torch.pi * true_x) diff --git a/Experiments/configurations/eiv_cubic.json b/Experiments/configurations/eiv_cubic.json index 0e9f9e8..59c37c1 100644 --- a/Experiments/configurations/eiv_cubic.json +++ b/Experiments/configurations/eiv_cubic.json @@ -13,10 +13,10 @@ "std_y_update_points": [1,40], "eiv_prediction_number_of_draws": [100,5], "eiv_prediction_number_of_batches": 10, - "init_std_y_list": [0.5], + "init_std_y_list": [0.05], "gamma": 0.5, "hidden_layers": [128, 128, 128, 128], - "fixed_std_x": 0.10, + "fixed_std_x": 0.20, "seed_range": [0,10], "gpu_number": 1 } diff --git a/Experiments/configurations/eiv_linear.json b/Experiments/configurations/eiv_linear.json index 8b7ebe0..ce84949 100644 --- a/Experiments/configurations/eiv_linear.json +++ b/Experiments/configurations/eiv_linear.json @@ -13,7 +13,7 @@ "std_y_update_points": [1,40], "eiv_prediction_number_of_draws": [100,5], "eiv_prediction_number_of_batches": 10, - "init_std_y_list": [0.5], + "init_std_y_list": [0.1], "gamma": 0.5, "hidden_layers": [128, 128, 128, 128], "fixed_std_x": 0.10, diff --git a/Experiments/configurations/eiv_quadratic.json b/Experiments/configurations/eiv_quadratic.json index c774bbe..dc8a9cd 100644 --- a/Experiments/configurations/eiv_quadratic.json +++ b/Experiments/configurations/eiv_quadratic.json @@ -13,7 +13,7 @@ "std_y_update_points": [1,40], "eiv_prediction_number_of_draws": [100,5], "eiv_prediction_number_of_batches": 10, - "init_std_y_list": [0.5], + "init_std_y_list": [0.1], "gamma": 0.5, "hidden_layers": [128, 128, 128, 128], "fixed_std_x": 0.10, diff --git a/Experiments/configurations/eiv_sine.json b/Experiments/configurations/eiv_sine.json index 1195ea0..b6bf6d0 100644 --- a/Experiments/configurations/eiv_sine.json +++ b/Experiments/configurations/eiv_sine.json @@ -14,10 +14,10 @@ "eiv_number_of_forward_draws": 10, "eiv_prediction_number_of_draws": [100,5], "eiv_prediction_number_of_batches": 10, - "init_std_y_list": [0.1], + "init_std_y_list": [0.01], "gamma": 0.5, "hidden_layers": [128, 128, 128, 128], - "fixed_std_x": 0.02, + "fixed_std_x": 0.04, "seed_range": [0,10], "gpu_number": 1 } diff --git a/Experiments/configurations/noneiv_cubic.json b/Experiments/configurations/noneiv_cubic.json index 6f71af1..a325665 100644 --- a/Experiments/configurations/noneiv_cubic.json +++ b/Experiments/configurations/noneiv_cubic.json @@ -13,7 +13,7 @@ "std_y_update_points": [1,40] , "noneiv_prediction_number_of_draws": 100, "noneiv_prediction_number_of_batches": 10, - "init_std_y_list": [0.5], + "init_std_y_list": [0.05], "gamma": 0.5, "hidden_layers": [128, 128, 128, 128], "seed_range": [0,10], diff --git a/Experiments/configurations/noneiv_linear.json b/Experiments/configurations/noneiv_linear.json index ae3040e..ff57b56 100644 --- a/Experiments/configurations/noneiv_linear.json +++ b/Experiments/configurations/noneiv_linear.json @@ -13,7 +13,7 @@ "std_y_update_points": [1,40] , "noneiv_prediction_number_of_draws": 100, "noneiv_prediction_number_of_batches": 10, - "init_std_y_list": [0.5], + "init_std_y_list": [0.1], "gamma": 0.5, "hidden_layers": [128, 128, 128, 128], "seed_range": [0,10], diff --git a/Experiments/configurations/noneiv_quadratic.json b/Experiments/configurations/noneiv_quadratic.json index 4052637..d858851 100644 --- a/Experiments/configurations/noneiv_quadratic.json +++ b/Experiments/configurations/noneiv_quadratic.json @@ -13,7 +13,7 @@ "std_y_update_points": [1,40] , "noneiv_prediction_number_of_draws": 100, "noneiv_prediction_number_of_batches": 10, - "init_std_y_list": [0.5], + "init_std_y_list": [0.1], "gamma": 0.5, "hidden_layers": [128, 128, 128, 128], "seed_range": [0,10], diff --git a/Experiments/configurations/noneiv_sine.json b/Experiments/configurations/noneiv_sine.json index cc88919..5191847 100644 --- a/Experiments/configurations/noneiv_sine.json +++ b/Experiments/configurations/noneiv_sine.json @@ -13,7 +13,7 @@ "std_y_update_points": [1,40] , "noneiv_prediction_number_of_draws": 100, "noneiv_prediction_number_of_batches": 10, - "init_std_y_list": [0.1], + "init_std_y_list": [0.01], "gamma": 0.5, "hidden_layers": [128, 128, 128, 128], "seed_range": [0,10], diff --git a/Experiments/plot_coverage_vs_q.py b/Experiments/plot_coverage_vs_q.py index 9bc59a3..5cb7f8f 100644 --- a/Experiments/plot_coverage_vs_q.py +++ b/Experiments/plot_coverage_vs_q.py @@ -13,6 +13,7 @@ import torch.backends.cudnn from torch.utils.data import DataLoader from matplotlib.pyplot import cm import matplotlib.pyplot as plt +from tqdm import tqdm from EIVArchitectures import Networks from EIVTrainingRoutines import train_and_store @@ -94,7 +95,7 @@ plt.ylabel('coverage') # datasets to plot and their coloring datasets = ['linear', 'quadratic','cubic','sine'] -colors = ['cyan', 'magenta', 'yellow', 'green'] +colors = ['#084519', '#7D098D', '#77050C', '#09017F'] def compute_coverages(data, eiv, number_of_draws): """ @@ -148,7 +149,6 @@ def compute_coverages(data, eiv, number_of_draws): = load_data(seed=0, return_ground_truth=True, normalize=normalize) - print(f"Computing {'EiV' if eiv else 'non-EiV'} coverage for {long_dataname}") # train_data only used for finding dimensions input_dim = train_data[0][0].numel() @@ -233,14 +233,13 @@ def compute_coverages(data, eiv, number_of_draws): return numerical_coverage, theoretical_coverage # loop through data -for data, color in zip(datasets, colors): +for data, color in tqdm(zip(datasets, colors)): # compute coverages eiv_coverages = compute_coverages(data=data, eiv=True, number_of_draws=[100,5]) noneiv_coverages = compute_coverages(data=data, eiv=False, number_of_draws=100) # create plots - plt.figure(1) coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color=color, against_theoretical=False, label=data) diff --git a/Experiments/plot_summary.py b/Experiments/plot_summary.py index c3d8c0d..c0863c6 100644 --- a/Experiments/plot_summary.py +++ b/Experiments/plot_summary.py @@ -80,7 +80,7 @@ for i, ([(eiv_metric_mean, eiv_metric_std), bottom = noneiv_metric_mean - k* noneiv_metric_std, color=colors[1], alpha=0.5) -plt.ylim(bottom=0, top=y_max) +plt.ylim(bottom=0, top=ymax) ax = plt.gca() ax.set_xticks(np.arange(1,len(data_list)+1)) ax.set_xticklabels(data_list, rotation='vertical') @@ -127,7 +127,7 @@ for i, ([(eiv_metric_mean, eiv_metric_std), color=colors[1], alpha=0.5) plt.axhline(0.95,0.0,1.0,color='k', linestyle='dashed') -plt.ylim(bottom=0, top=y_max) +plt.ylim(bottom=0, top=ymax) ax = plt.gca() ax.set_xticks(np.arange(1,len(data_list)+1)) ax.set_xticklabels(data_list, rotation='vertical') @@ -173,7 +173,7 @@ for i, ([(eiv_metric_mean, eiv_metric_std), bottom = noneiv_metric_mean - k* noneiv_metric_std, color=colors[1], alpha=0.5) -plt.ylim(bottom=0, top=y_max) +plt.ylim(bottom=0, top=ymax) ax = plt.gca() ax.set_xticks(np.arange(1,len(data_list)+1)) ax.set_xticklabels(data_list, rotation='vertical') -- GitLab