diff --git a/Experiments/plot_coverage.py b/Experiments/plot_coverage.py index f4a93a4c8776b4101bc6277e885d18e3fdf786b3..298603ce4b400606fa21e16c7997410183b7973f 100644 --- a/Experiments/plot_coverage.py +++ b/Experiments/plot_coverage.py @@ -12,6 +12,7 @@ import numpy as np import torch import torch.backends.cudnn from torch.utils.data import DataLoader +from matplotlib.pyplot import cm import matplotlib.pyplot as plt from EIVArchitectures import Networks @@ -24,7 +25,7 @@ q_range = np.linspace(0.1, 0.95) # load hyperparameters from JSON file def compute_coverages(data, eiv, number_of_draws, - use_ground_truth=False, noisy_y=True): + use_ground_truth=False): """ Create network and dataloader iterators for `data` (short dataname) and feed them into `get_coverage_distribution`. @@ -191,32 +192,44 @@ def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color, """ eiv_numerical_coverage, eiv_theoretical_coverage = eiv_coverages noneiv_numerical_coverage, noneiv_theoretical_coverage = noneiv_coverages - if eiv_numerical_coverage is not None and eiv_theoretical_coverage is not None: - mean_eiv_numerical_coverage = np.mean(eiv_numerical_coverage, axis=-1) - std_eiv_numerical_coverage = np.std(eiv_numerical_coverage, axis=-1) - if against_theoretical: - x_values = np.mean(eiv_theoretical_coverage, axis=-1) - else: - x_values = np.array(q_range) - plt.plot(x_values, mean_eiv_numerical_coverage, - color=color, linestyle='solid', label=f'eiv_{label}') - plt.fill_between(x_values, - mean_eiv_numerical_coverage - std_eiv_numerical_coverage, - mean_eiv_numerical_coverage + std_eiv_numerical_coverage, - color=color, alpha=0.5) - if noneiv_numerical_coverage is not None and noneiv_theoretical_coverage is not None: - mean_noneiv_numerical_coverage = np.mean(noneiv_numerical_coverage, axis=-1) - std_noneiv_numerical_coverage = np.std(noneiv_numerical_coverage, axis=-1) - if against_theoretical: - x_values = np.mean(noneiv_theoretical_coverage, axis=-1) - else: - x_values = np.array(q_range) - plt.plot(x_values, mean_noneiv_numerical_coverage, - color=color, linestyle='dashed', label=f'noneiv_{label}') - plt.fill_between(x_values, - mean_noneiv_numerical_coverage - std_noneiv_numerical_coverage, - mean_noneiv_numerical_coverage + std_noneiv_numerical_coverage, - color=color, alpha=0.3) + assert (len(eiv_numerical_coverage.shape)) == 2 + assert (len(noneiv_numerical_coverage.shape)) == 2 + # EiV + # take mean/std over seed dimension + mean_eiv_numerical_coverage = np.mean(eiv_numerical_coverage, axis=-1) + std_eiv_numerical_coverage = np.std(eiv_numerical_coverage, axis=-1) + if against_theoretical: + # show theoretical coverage on x-axis + x_values = np.mean(eiv_theoretical_coverage, axis=-1) + else: + # show q-range on x-axis + x_values = np.array(q_range) + # plot mean + plt.plot(x_values, mean_eiv_numerical_coverage, + color=color, linestyle='solid', label=f'eiv_{label}') + # plot std + plt.fill_between(x_values, + mean_eiv_numerical_coverage - std_eiv_numerical_coverage, + mean_eiv_numerical_coverage + std_eiv_numerical_coverage, + color=color, alpha=0.5) + # non-EiV + # take mean/std over seed dimension + mean_noneiv_numerical_coverage = np.mean(noneiv_numerical_coverage, axis=-1) + std_noneiv_numerical_coverage = np.std(noneiv_numerical_coverage, axis=-1) + if against_theoretical: + # show theoretical coverage on x-axis + x_values = np.mean(noneiv_theoretical_coverage, axis=-1) + else: + # show q-range on x-axis + x_values = np.array(q_range) + # plot mean + plt.plot(x_values, mean_noneiv_numerical_coverage, + color=color, linestyle='dashed', label=f'noneiv_{label}') + # plot std + plt.fill_between(x_values, + mean_noneiv_numerical_coverage - std_noneiv_numerical_coverage, + mean_noneiv_numerical_coverage + std_noneiv_numerical_coverage, + color=color, alpha=0.3) def coverage_residual_plot(eiv_coverages, noneiv_coverages, color, absolute_values=True, against_theoretical=False, label=''): @@ -275,7 +288,7 @@ def coverage_residual_plot(eiv_coverages, noneiv_coverages, color, # create figures, together with title and axis labels plt.figure(1) plt.clf() -plt.title('Coverage for datasets with ground truth') +plt.title('Coverage for datasets with ground truth (vs. q)') plt.xlabel('q') plt.ylabel('coverage') plt.figure(2) @@ -285,23 +298,24 @@ plt.xlabel('th. coverage') plt.ylabel('coverage') plt.figure(3) plt.clf() -plt.title('Coverage of noisy labels (vs. q)') +plt.title('q Deviation of coverage of noisy labels') plt.xlabel('q') plt.ylabel('coverage') plt.figure(4) plt.clf() -plt.title('Theoretical deviation Coverage for datasets with ground truth') +plt.title('q Deviation of coverage for datasets with ground truth') plt.xlabel('q') plt.ylabel('deviation cov from q') plt.figure(5) plt.clf() -plt.title('Deviation (theory) of noisy Coverage') +plt.title('Theory deviation of noisy coverage') plt.xlabel('q') plt.ylabel('deviation cov from theor. cov') # datasets to plot and their coloring -datasets = ['linear', 'quadratic','yacht','wine','power'] -colors = ['red', 'blue','green','purple','orange','cyan'] +datasets = ['linear', 'quadratic','yacht','wine','power', + 'protein','concrete','california','energy','kin8nm','msd','naval'] +colors = cm.rainbow(np.linspace(0,1,len(datasets))) # loop through data for use_ground_truth in [False, True]: @@ -354,5 +368,17 @@ for fig_nr in range(1,6): plt.figure(fig_nr) plt.legend() -# show plots +plt.figure(1) +plt.savefig('results/figures/summary_coverage_ground_truth.pdf') +plt.figure(2) +plt.savefig('results/figures/summary_coverage_noisy.pdf') +plt.figure(3) +plt.savefig('results/figures/summary_q_deviation_coverage_noisy.pdf') +plt.figure(4) +plt.savefig('results/figures/summary_q_deviation_coverage_ground_truth.pdf') +plt.figure(5) +plt.savefig('results/figures/summary_theory_deviation_coverage_ground_truth.pdf') + +# show plot plt.show() + diff --git a/README.md b/README.md index 7547a9ddf61f21fe3abacba7584910cedd0dfe34..11bb840651f4a26d5b4fc09d78555994dc92f5b5 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ pip install EIVPackage/ Installing this package will make 3 modules available to the python environment: `EIVArchitectures` (for building EiV Models), `EIVTrainingRoutines` (containing a general training framework), `EIVGeneral` (containing a single module needed for repeated sampling). ## Missing -+ Tell to create folders, like `Experiments/results` ++ Tell to create folders, like `Experiments/results`, `Experiments/results/figures/` ## Contributing