diff --git a/Experiments/plot_coverage.py b/Experiments/plot_coverage.py index 298603ce4b400606fa21e16c7997410183b7973f..ba21039aacbf8a02fc65303d4400c54a53938218 100644 --- a/Experiments/plot_coverage.py +++ b/Experiments/plot_coverage.py @@ -178,7 +178,7 @@ def compute_coverages(data, eiv, number_of_draws, #### def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color, - against_theoretical = False, label = ''): + against_theoretical = False, label = '', mean_error=True): """ Plot numerical coverages against q (used coverage value), if against_theoretical is False, or the theoretical coverage, if @@ -187,8 +187,10 @@ def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color, :param noneiv_coverages: The output of `compute_coverages` with `eiv=False` :param color: String, denoting the color. :param against_theoretical: Boolean, see above. - :param label: String. Will be combined with "eiv_"/"noneiv_" and included - as label in the plot. + :param label: String. Will be included as label in the plot. + :param mean_error: Boolean. If True the standard deviation is divided + by the square root of the number of elements, to display the error + of the mean (and not the dispersion). """ eiv_numerical_coverage, eiv_theoretical_coverage = eiv_coverages noneiv_numerical_coverage, noneiv_theoretical_coverage = noneiv_coverages @@ -198,6 +200,8 @@ def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color, # take mean/std over seed dimension mean_eiv_numerical_coverage = np.mean(eiv_numerical_coverage, axis=-1) std_eiv_numerical_coverage = np.std(eiv_numerical_coverage, axis=-1) + if mean_error: + std_eiv_numerical_coverage /= np.sqrt(eiv_numerical_coverage.shape[1]) if against_theoretical: # show theoretical coverage on x-axis x_values = np.mean(eiv_theoretical_coverage, axis=-1) @@ -206,7 +210,7 @@ def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color, x_values = np.array(q_range) # plot mean plt.plot(x_values, mean_eiv_numerical_coverage, - color=color, linestyle='solid', label=f'eiv_{label}') + color=color, linestyle='solid', label=label) # plot std plt.fill_between(x_values, mean_eiv_numerical_coverage - std_eiv_numerical_coverage, @@ -216,6 +220,9 @@ def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color, # take mean/std over seed dimension mean_noneiv_numerical_coverage = np.mean(noneiv_numerical_coverage, axis=-1) std_noneiv_numerical_coverage = np.std(noneiv_numerical_coverage, axis=-1) + if mean_error: + std_noneiv_numerical_coverage /= \ + np.sqrt(noneiv_numerical_coverage.shape[1]) if against_theoretical: # show theoretical coverage on x-axis x_values = np.mean(noneiv_theoretical_coverage, axis=-1) @@ -224,7 +231,7 @@ def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color, x_values = np.array(q_range) # plot mean plt.plot(x_values, mean_noneiv_numerical_coverage, - color=color, linestyle='dashed', label=f'noneiv_{label}') + color=color, linestyle='dashed') # plot std plt.fill_between(x_values, mean_noneiv_numerical_coverage - std_noneiv_numerical_coverage, @@ -232,7 +239,8 @@ def coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color, color=color, alpha=0.3) def coverage_residual_plot(eiv_coverages, noneiv_coverages, color, - absolute_values=True, against_theoretical=False, label=''): + absolute_values=True, against_theoretical=False, label='', + mean_error=True): """ Plot deviation of numerical coverages to q (used coverage value), if against_theoretical is False, or to the theoretical coverage, if @@ -244,8 +252,10 @@ def coverage_residual_plot(eiv_coverages, noneiv_coverages, color, :param absolute_values:Boolean. If True (default) the absolute value of the differences will be plotted. :param against_theoretical: Boolean, see above. - :param label: String. Will be combined with "eiv_"/"noneiv_" and included - as label in the plot. + :param label: String. Will be included as label in the plot. + :param mean_error: Boolean. If True the standard deviation is divided + by the square root of the number of elements, to display the error + of the mean (and not the dispersion). """ if not against_theoretical: eiv_coverages = eiv_coverages[0] @@ -266,29 +276,59 @@ def coverage_residual_plot(eiv_coverages, noneiv_coverages, color, noneiv_cov_res = np.abs(noneiv_cov_res) mean_eiv_cov_res = np.mean(eiv_cov_res, axis=-1) std_eiv_cov_res = np.std(eiv_cov_res, axis=-1) + if mean_error: + std_eiv_cov_res /= np.sqrt(eiv_cov_res.shape[1]) mean_noneiv_cov_res = np.mean(noneiv_cov_res, axis=-1) std_noneiv_cov_res = np.std(noneiv_cov_res, axis=-1) + if mean_error: + std_noneiv_cov_res /= np.sqrt(noneiv_cov_res.shape[1]) plt.plot(q_range, mean_eiv_cov_res, - color=color, linestyle='solid', label=f'eiv_{label}') + color=color, linestyle='solid', label=label) plt.fill_between(q_range, mean_eiv_cov_res - std_eiv_cov_res, mean_eiv_cov_res + std_eiv_cov_res, color=color, alpha=0.5) plt.plot(q_range, mean_noneiv_cov_res, - color=color, linestyle='dashed', label=f'noneiv_{label}') + color=color, linestyle='dashed') plt.fill_between(q_range, mean_noneiv_cov_res - std_noneiv_cov_res, mean_noneiv_cov_res + std_noneiv_cov_res, color=color, alpha=0.3) +def coverage_difference_plot(eiv_coverages, noneiv_coverages, color, label='', + mean_error=True): + """ + Plot difference of numerical coverages from EiV to the one of non-EiV. + On the x axis q is plotted. + :param eiv_coverages: The output of `compute_coverages` with `eiv=True` + :param noneiv_coverages: The output of `compute_coverages` with `eiv=False` + :param color: String, denoting the color. + :param label: String. Will be included as label in the plot. + :param mean_error: Boolean. If True the standard deviation is divided + by the square root of the number of elements, to display the error + of the mean (and not the dispersion). + """ + cov_diff = eiv_coverages[0] - noneiv_coverages[0] + + mean_cov_diff = np.mean(cov_diff, axis=-1) + std_cov_diff = np.std(cov_diff, axis=-1) + if mean_error: + std_cov_diff /= np.sqrt(cov_diff.shape[1]) + plt.plot(q_range, mean_cov_diff, + color=color, linestyle='solid', label=label) + plt.fill_between(q_range, mean_cov_diff - std_cov_diff, + mean_cov_diff + std_cov_diff, + color=color, alpha=0.5) + + # create figures, together with title and axis labels plt.figure(1) plt.clf() -plt.title('Coverage for datasets with ground truth (vs. q)') +plt.title('Coverage for datasets with ground truth') plt.xlabel('q') plt.ylabel('coverage') plt.figure(2) @@ -298,20 +338,29 @@ plt.xlabel('th. coverage') plt.ylabel('coverage') plt.figure(3) plt.clf() -plt.title('q Deviation of coverage of noisy labels') +plt.title('Coverage of noisy labels (vs. q)') plt.xlabel('q') plt.ylabel('coverage') plt.figure(4) plt.clf() +plt.title('q Deviation of coverage of noisy labels') +plt.xlabel('q') +plt.ylabel('coverage') +plt.figure(5) +plt.clf() plt.title('q Deviation of coverage for datasets with ground truth') plt.xlabel('q') plt.ylabel('deviation cov from q') -plt.figure(5) +plt.figure(6) plt.clf() plt.title('Theory deviation of noisy coverage') plt.xlabel('q') plt.ylabel('deviation cov from theor. cov') - +plt.figure(7) +plt.clf() +plt.title('Difference coverages') +plt.xlabel('q') +plt.ylabel('EiV cov - nonEiV-cov') # datasets to plot and their coloring datasets = ['linear', 'quadratic','yacht','wine','power', 'protein','concrete','california','energy','kin8nm','msd','naval'] @@ -335,21 +384,27 @@ for use_ground_truth in [False, True]: # create plots if use_ground_truth: plt.figure(1) - coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color=color, - against_theoretical=False, label=data) - plt.figure(4) - coverage_residual_plot(eiv_coverages, noneiv_coverages, color=color, - against_theoretical=False, label=data) + coverage_diagonal_plot(eiv_coverages, noneiv_coverages, + color=color, against_theoretical=False, label=data) + plt.figure(5) + coverage_residual_plot(eiv_coverages, noneiv_coverages, + color=color, against_theoretical=False, label=data) else: plt.figure(2) - coverage_diagonal_plot(eiv_coverages, noneiv_coverages, color=color, - against_theoretical=True, label=data) + coverage_diagonal_plot(eiv_coverages, noneiv_coverages, + color=color, against_theoretical=True, label=data) plt.figure(3) - coverage_residual_plot(eiv_coverages, noneiv_coverages, color=color, - against_theoretical=False, label=data) - plt.figure(5) - coverage_residual_plot(eiv_coverages, noneiv_coverages, color=color, - against_theoretical=True, label=data) + coverage_diagonal_plot(eiv_coverages, noneiv_coverages, + color=color, against_theoretical=False, label=data) + plt.figure(4) + coverage_residual_plot(eiv_coverages, noneiv_coverages, + color=color, against_theoretical=False, label=data) + plt.figure(6) + coverage_residual_plot(eiv_coverages, noneiv_coverages, + color=color, against_theoretical=True, label=data) + plt.figure(7) + coverage_difference_plot(eiv_coverages, noneiv_coverages, + color=color, label=data) # add diagonals, where meaningful plt.figure(1) @@ -364,21 +419,26 @@ plt.plot(x_diag, x_diag, color='k', linestyle='dotted' ) # add legends -for fig_nr in range(1,6): +for fig_nr in range(1,8): plt.figure(fig_nr) plt.legend() plt.figure(1) plt.savefig('results/figures/summary_coverage_ground_truth.pdf') plt.figure(2) -plt.savefig('results/figures/summary_coverage_noisy.pdf') +plt.savefig('results/figures/summary_coverage_noisy_vs_th.pdf') plt.figure(3) -plt.savefig('results/figures/summary_q_deviation_coverage_noisy.pdf') +plt.savefig('results/figures/summary_coverage_noisy_vs_q.pdf') plt.figure(4) -plt.savefig('results/figures/summary_q_deviation_coverage_ground_truth.pdf') +plt.savefig('results/figures/summary_q_deviation_coverage_noisy.pdf') plt.figure(5) -plt.savefig('results/figures/summary_theory_deviation_coverage_ground_truth.pdf') +plt.savefig('results/figures/summary_q_deviation_coverage_ground_truth.pdf') +plt.figure(6) +plt.savefig('results/figures/summary_th_deviation_coverage_ground_truth.pdf') +plt.figure(7) +plt.savefig('results/figures/summary_difference_coverage.pdf') + -# show plot -plt.show() +##### +plt.show()