6
$\begingroup$

When I run

from scipy.stats import anderson_ksamp
a = [-1.8, -2.4, -2.4, -0.0, -1.5, -2.7, -1.8, -3.0, -1.8, -1.2, -3.0, -3.0, -2.8, -3.0, -2.1, -0.0, 0.6, -2.5, -2.4, -0.0, -2.7, -0.0, -2.5, -2.1, -0.9, -3.0, -0.6, -0.6, -1.5, -2.2, -1.2, -2.4, -2.4, -3.0, 1.5, -1.8, 1.5, -2.7, -3.0, -2.5, -2.5, -1.5, -1.5, -2.1, -2.1, -3.0, -0.6, -2.7, -3.0, -1.5, -0.6, -0.0, -2.1, 0.6, -2.0, -3.0, -3.0, -2.4, -3.0, -1.8, -0.0, -0.0, -3.0, -1.5, -3.0, -3.0, -1.5, -2.5, -3.0, -2.8, -3.0, -2.2, -0.6, -0.0, -1.5, -2.7, -2.1, -2.1, -2.2, -2.1, -0.6, -0.0, -2.5, -2.1, -1.5, -3.0, -2.2, -1.8, -2.7, -2.4, -1.5, -2.1, -2.9, -2.4, -0.9, -0.0, -0.0, -2.4, -2.7, -0.6, -2.2, -3.0, -1.5, -0.9, -3.0, -3.0, -0.0, -2.7, -2.7, -1.5, -2.2, -3.0, -0.0, 1.5, -3.0, -2.7, -2.2, -2.9, -2.2, -3.0, -1.8, -0.0, -3.0, -1.5, -2.7, -3.0, -3.0, -2.9, -3.0, -3.0, -3.0, -3.0, -2.5, -0.0, 0.9, -3.0, -0.0, -3.0, 3.0, -3.0, -3.0, -1.2, -2.1, 1.5, -0.0, -0.9, -3.0, -2.7, -1.5, -2.4, -2.1, -3.0, -0.9, -3.0, -0.8, -1.5, -2.1, -2.7, -0.0, -0.0, -2.2, -1.8, -2.1, -2.2, -3.0, 0.6, -2.4, -2.2, -2.4, -2.5, -1.5, -0.0, -2.7, -3.0, -3.0, -2.1, -0.0, -2.4, -2.4, -0.0, -2.0, -0.9, -2.4, -3.0, -1.4, -2.7, -2.7, -3.0, -3.0, -2.7, -1.2, -2.1, -3.0, -0.0, -3.0, -2.7, -2.7, -3.0, -3.0, -2.5, -3.0, -1.8, -1.5, -2.7, -2.4, -1.8, -3.0, -2.7, 2.1, -3.0, -2.2, -2.2, 0.6, -0.9, 6.0, -3.0, -2.1, -3.0, -2.1, -2.5, -3.0, -1.5, -2.5, 3.0, -2.1, -3.0, -3.0, -1.5, -2.1, -2.7, -2.5, 1.5, -2.1, -0.0, -0.0, -3.0, -0.6, 1.5, -2.7, -2.4, -2.1, -3.0, -2.7, -3.0, 9.0, -3.0, -1.7, -3.0, -0.0, -3.0, -2.2, -0.0, -0.6, -2.7, -2.7, -3.0, -3.0, -1.7, -2.1, -2.0, -3.0, -2.1, -3.0, -1.1, -3.0, -3.0, -2.4, -1.5, -3.0, -2.2, -3.0, -1.5, -2.7, -3.0, -3.0, -3.0, -2.2, -3.0, -2.1, -2.1, -2.4, -3.0, -3.0, -0.0, -3.0, -3.0, -2.1, -2.0, 1.5, -3.0, -3.0, -2.1, -2.9, -2.4, -3.0, -3.0, -1.5, -2.2, -0.9, -1.8, -2.1, -1.8, -1.5, -3.0, -1.5, -3.0, -1.5, -3.0, -2.4, 1.5, -2.7, -3.0, -1.8, -1.8, -1.5, -2.1, -2.7, -2.7, -2.7, -3.0, -1.5, -2.7, -3.0, -2.7, -3.0, -1.5, -1.5, -3.0, 0.6, -0.6, -3.0, -2.1, -2.4, -2.1, -3.0, -2.2, -3.0, -1.8, -1.2, -3.0, -0.8, -2.4, -2.5, -3.0, -1.5, -1.2, -0.0, -2.7, -2.4, -3.0, -3.0, -2.1, -2.1, -2.1, -3.0, -2.7, -2.4, -2.1, -1.5, -2.1, -0.6, -3.0, -3.0, -3.0, -3.0]
b = [-1.2, 6.0, -3.0, 1.5, 12.0, 1.5, 3.0, -0.0, -1.5, -0.0, 6.0, -0.0, 1.8, -1.5, -3.0, -3.0, 1.5, -0.0, 1.5, -3.0, -3.0, -0.0, -1.2, -3.0, 22.5, -0.0, -3.0, -3.0, -2.1, 3.0, 2.4, 1.5, -2.1, 4.5, -3.0, -3.0, 12.0, 6.0, -3.0, 3.0, -3.0, 12.0, -2.8, 0.6, 4.5, 3.0, -0.0, -3.0, -1.0, -3.0, -0.0, 3.0, 2.1, -0.6, -3.0, -3.0, 1.5, 1.5, -0.6, -0.0, 1.5, 6.0, -2.2, -2.1, -0.0, -0.8, 6.0, -3.0, -3.0, 9.0, -3.0, -0.9, 15.0, 1.5, -2.9, 19.5, 4.5, -3.0, 1.5, -1.8, 1.5, 0.9, -3.0, -3.0, 9.0, -0.3, 9.0, -2.1, -2.2, -3.0, 12.0, -2.1, -1.2, -3.0, 4.5, -1.2, -0.0, 12.0, -0.9, -3.0, -0.6, -3.0, -1.5, -0.0, 27.0, 4.5, -2.4, -2.7, -0.0, 3.0, 1.5, 3.0, -3.0, 16.5, -2.1, -0.6, -3.0, -1.5, -1.2, -3.0, 4.4, -2.5, -2.1, 3.0, -1.2, -3.0, 12.0, -1.8, -0.9, -3.0, -3.0, -0.0, -3.0, -1.2, -3.0, -3.0, -0.0, 1.5, -3.0, 7.5, -3.0, -2.1, 3.0, 1.5, 3.0, -3.0, 1.5, -3.0, 19.5, -3.0, -2.2, 27.0, 3.0, -1.5, -3.0, -3.0, 4.5, -1.2, 12.0, 3.0, 3.0, -2.1, 6.0, -2.2, -3.0, -2.4, 6.0, -3.0, -1.9, -0.6, -0.0, 3.6, 15.0, -3.0, -3.0, 7.5, -0.0, 4.5, -2.4, -3.0, -3.0, -3.0, -2.0, -3.0, -3.0, -3.0, -2.6, 3.0, -0.0, -3.0, 4.5, -1.2, -3.0, -3.0, 3.0, -3.0, 1.6, 1.5, -3.0, -3.0, -3.0, -3.0, 3.0, -3.0, -1.5, -3.0, -1.5, 12.0, -1.5, 3.0, 9.6, -0.0, -1.8, -3.0, -2.1, -0.6, 9.0, -3.0, 19.5, -2.4, -1.8, 15.0, -3.0, -3.0, 27.0, 7.5, 12.0, -3.0, -2.0, -3.0, 12.0, -3.0, -3.0, 0.9, -3.0, -2.5, 12.6, -1.5, -0.6, -3.0, -3.0, -0.0, -0.8, -3.0, 3.0, -0.0, 4.5, -3.0, -0.0, 0.6, 0.3, -2.7, -0.0, 4.5, -3.0, 3.0, -1.5, 9.0, -3.0, 1.5, -3.0, 6.0, -2.2, -0.0, -3.0, -3.0, 3.0, -3.0, -2.7, -0.0, -3.0, 0.6, 4.5, 1.5, 1.5, 1.5, -2.1, 7.5, -3.0, -3.0, -3.0, -2.1, -3.0, 6.0, -1.5, -2.0, -3.0, -0.6, 6.0, -1.5, -3.0, 6.0, -3.0, 3.0, 3.0, -3.0, 7.5, -3.0, -0.0, -3.0, -0.0, -3.0, -3.0, 0.6, -3.0, -3.0, -3.0, -3.0, -2.4, -0.0, 1.5, 3.0, 2.1, -3.0, 3.0, 4.5, -3.0, -2.4, -3.0, -2.2, 7.5, 2.1, -3.0, -0.0, -2.0, -3.0, -3.0, -1.8, -3.0, 4.5, -1.5, -3.0, 4.5, 3.0, 15.0, 4.5, -2.1, 12.0, 6.0, 4.5, 27.0, -3.0, 3.0, 0.6, -3.0, 0.6, -2.4, 4.5, -1.5, -2.2, 12.0, -2.0, 1.5, 9.0, -1.5, -1.2, -2.0, 1.5, -1.2, -1.8, -3.0, -0.0, -0.0, -3.0, -0.9]
print(anderson_ksamp([a,b]))

I get the following output:

Anderson_ksampResult(statistic=53.560696338122263, critical_values=array([ 0.325,  1.226,  1.961,  2.718,  3.752]), significance_level=7238105.535194747)

The "significance level" is far greater than 1. Why is this?

I'm mostly wondering whether it's an issue with the scipy implementation or is this issue inherent to numerical implementations of the Anderson-Darling test (as I don't know the details of how the p-value is actually computed). I'd also be interested in some info on how to get around this bug.



I'm aware of this issue, but that seems to be a slight error in extrapolation due to data from the same distribution (scipy.stats.ks_2samp gives a p-value of 0.962305 whereas anderson_ksamp gives 1.075969). On the other hand, with the above data we can run

from scipy.stats import ks_2samp
from statsmodels.distributions.empirical_distribution import ECDF
import matplotlib.pyplot as plt
import numpy as np
a = [-1.8, -2.4, -2.4, -0.0, -1.5, -2.7, -1.8, -3.0, -1.8, -1.2, -3.0, -3.0, -2.8, -3.0, -2.1, -0.0, 0.6, -2.5, -2.4, -0.0, -2.7, -0.0, -2.5, -2.1, -0.9, -3.0, -0.6, -0.6, -1.5, -2.2, -1.2, -2.4, -2.4, -3.0, 1.5, -1.8, 1.5, -2.7, -3.0, -2.5, -2.5, -1.5, -1.5, -2.1, -2.1, -3.0, -0.6, -2.7, -3.0, -1.5, -0.6, -0.0, -2.1, 0.6, -2.0, -3.0, -3.0, -2.4, -3.0, -1.8, -0.0, -0.0, -3.0, -1.5, -3.0, -3.0, -1.5, -2.5, -3.0, -2.8, -3.0, -2.2, -0.6, -0.0, -1.5, -2.7, -2.1, -2.1, -2.2, -2.1, -0.6, -0.0, -2.5, -2.1, -1.5, -3.0, -2.2, -1.8, -2.7, -2.4, -1.5, -2.1, -2.9, -2.4, -0.9, -0.0, -0.0, -2.4, -2.7, -0.6, -2.2, -3.0, -1.5, -0.9, -3.0, -3.0, -0.0, -2.7, -2.7, -1.5, -2.2, -3.0, -0.0, 1.5, -3.0, -2.7, -2.2, -2.9, -2.2, -3.0, -1.8, -0.0, -3.0, -1.5, -2.7, -3.0, -3.0, -2.9, -3.0, -3.0, -3.0, -3.0, -2.5, -0.0, 0.9, -3.0, -0.0, -3.0, 3.0, -3.0, -3.0, -1.2, -2.1, 1.5, -0.0, -0.9, -3.0, -2.7, -1.5, -2.4, -2.1, -3.0, -0.9, -3.0, -0.8, -1.5, -2.1, -2.7, -0.0, -0.0, -2.2, -1.8, -2.1, -2.2, -3.0, 0.6, -2.4, -2.2, -2.4, -2.5, -1.5, -0.0, -2.7, -3.0, -3.0, -2.1, -0.0, -2.4, -2.4, -0.0, -2.0, -0.9, -2.4, -3.0, -1.4, -2.7, -2.7, -3.0, -3.0, -2.7, -1.2, -2.1, -3.0, -0.0, -3.0, -2.7, -2.7, -3.0, -3.0, -2.5, -3.0, -1.8, -1.5, -2.7, -2.4, -1.8, -3.0, -2.7, 2.1, -3.0, -2.2, -2.2, 0.6, -0.9, 6.0, -3.0, -2.1, -3.0, -2.1, -2.5, -3.0, -1.5, -2.5, 3.0, -2.1, -3.0, -3.0, -1.5, -2.1, -2.7, -2.5, 1.5, -2.1, -0.0, -0.0, -3.0, -0.6, 1.5, -2.7, -2.4, -2.1, -3.0, -2.7, -3.0, 9.0, -3.0, -1.7, -3.0, -0.0, -3.0, -2.2, -0.0, -0.6, -2.7, -2.7, -3.0, -3.0, -1.7, -2.1, -2.0, -3.0, -2.1, -3.0, -1.1, -3.0, -3.0, -2.4, -1.5, -3.0, -2.2, -3.0, -1.5, -2.7, -3.0, -3.0, -3.0, -2.2, -3.0, -2.1, -2.1, -2.4, -3.0, -3.0, -0.0, -3.0, -3.0, -2.1, -2.0, 1.5, -3.0, -3.0, -2.1, -2.9, -2.4, -3.0, -3.0, -1.5, -2.2, -0.9, -1.8, -2.1, -1.8, -1.5, -3.0, -1.5, -3.0, -1.5, -3.0, -2.4, 1.5, -2.7, -3.0, -1.8, -1.8, -1.5, -2.1, -2.7, -2.7, -2.7, -3.0, -1.5, -2.7, -3.0, -2.7, -3.0, -1.5, -1.5, -3.0, 0.6, -0.6, -3.0, -2.1, -2.4, -2.1, -3.0, -2.2, -3.0, -1.8, -1.2, -3.0, -0.8, -2.4, -2.5, -3.0, -1.5, -1.2, -0.0, -2.7, -2.4, -3.0, -3.0, -2.1, -2.1, -2.1, -3.0, -2.7, -2.4, -2.1, -1.5, -2.1, -0.6, -3.0, -3.0, -3.0, -3.0]
b = [-1.2, 6.0, -3.0, 1.5, 12.0, 1.5, 3.0, -0.0, -1.5, -0.0, 6.0, -0.0, 1.8, -1.5, -3.0, -3.0, 1.5, -0.0, 1.5, -3.0, -3.0, -0.0, -1.2, -3.0, 22.5, -0.0, -3.0, -3.0, -2.1, 3.0, 2.4, 1.5, -2.1, 4.5, -3.0, -3.0, 12.0, 6.0, -3.0, 3.0, -3.0, 12.0, -2.8, 0.6, 4.5, 3.0, -0.0, -3.0, -1.0, -3.0, -0.0, 3.0, 2.1, -0.6, -3.0, -3.0, 1.5, 1.5, -0.6, -0.0, 1.5, 6.0, -2.2, -2.1, -0.0, -0.8, 6.0, -3.0, -3.0, 9.0, -3.0, -0.9, 15.0, 1.5, -2.9, 19.5, 4.5, -3.0, 1.5, -1.8, 1.5, 0.9, -3.0, -3.0, 9.0, -0.3, 9.0, -2.1, -2.2, -3.0, 12.0, -2.1, -1.2, -3.0, 4.5, -1.2, -0.0, 12.0, -0.9, -3.0, -0.6, -3.0, -1.5, -0.0, 27.0, 4.5, -2.4, -2.7, -0.0, 3.0, 1.5, 3.0, -3.0, 16.5, -2.1, -0.6, -3.0, -1.5, -1.2, -3.0, 4.4, -2.5, -2.1, 3.0, -1.2, -3.0, 12.0, -1.8, -0.9, -3.0, -3.0, -0.0, -3.0, -1.2, -3.0, -3.0, -0.0, 1.5, -3.0, 7.5, -3.0, -2.1, 3.0, 1.5, 3.0, -3.0, 1.5, -3.0, 19.5, -3.0, -2.2, 27.0, 3.0, -1.5, -3.0, -3.0, 4.5, -1.2, 12.0, 3.0, 3.0, -2.1, 6.0, -2.2, -3.0, -2.4, 6.0, -3.0, -1.9, -0.6, -0.0, 3.6, 15.0, -3.0, -3.0, 7.5, -0.0, 4.5, -2.4, -3.0, -3.0, -3.0, -2.0, -3.0, -3.0, -3.0, -2.6, 3.0, -0.0, -3.0, 4.5, -1.2, -3.0, -3.0, 3.0, -3.0, 1.6, 1.5, -3.0, -3.0, -3.0, -3.0, 3.0, -3.0, -1.5, -3.0, -1.5, 12.0, -1.5, 3.0, 9.6, -0.0, -1.8, -3.0, -2.1, -0.6, 9.0, -3.0, 19.5, -2.4, -1.8, 15.0, -3.0, -3.0, 27.0, 7.5, 12.0, -3.0, -2.0, -3.0, 12.0, -3.0, -3.0, 0.9, -3.0, -2.5, 12.6, -1.5, -0.6, -3.0, -3.0, -0.0, -0.8, -3.0, 3.0, -0.0, 4.5, -3.0, -0.0, 0.6, 0.3, -2.7, -0.0, 4.5, -3.0, 3.0, -1.5, 9.0, -3.0, 1.5, -3.0, 6.0, -2.2, -0.0, -3.0, -3.0, 3.0, -3.0, -2.7, -0.0, -3.0, 0.6, 4.5, 1.5, 1.5, 1.5, -2.1, 7.5, -3.0, -3.0, -3.0, -2.1, -3.0, 6.0, -1.5, -2.0, -3.0, -0.6, 6.0, -1.5, -3.0, 6.0, -3.0, 3.0, 3.0, -3.0, 7.5, -3.0, -0.0, -3.0, -0.0, -3.0, -3.0, 0.6, -3.0, -3.0, -3.0, -3.0, -2.4, -0.0, 1.5, 3.0, 2.1, -3.0, 3.0, 4.5, -3.0, -2.4, -3.0, -2.2, 7.5, 2.1, -3.0, -0.0, -2.0, -3.0, -3.0, -1.8, -3.0, 4.5, -1.5, -3.0, 4.5, 3.0, 15.0, 4.5, -2.1, 12.0, 6.0, 4.5, 27.0, -3.0, 3.0, 0.6, -3.0, 0.6, -2.4, 4.5, -1.5, -2.2, 12.0, -2.0, 1.5, 9.0, -1.5, -1.2, -2.0, 1.5, -1.2, -1.8, -3.0, -0.0, -0.0, -3.0, -0.9]
ecdf1, ecdf2 = ECDF(a), ECDF(b)
xs = np.linspace(min(a+b),max(a+b), num=10000)
plt.figure(figsize=(12,8))
plt.plot(xs,ecdf1(xs), xs,ecdf2(xs))
plt.show()
print(ks_2samp(a,b))

to get a p-value of 2.6169e-17, with the following ECDF plot showing that the distributions which a and b were sampled from are very different: ECDF Plot

$\endgroup$
  • 2
    $\begingroup$ It's obviously a bug and appears to be a result of gross extrapolation. Are you looking for any other kind of answer? $\endgroup$ – whuber Jul 25 '16 at 20:20
  • 2
    $\begingroup$ I'm mostly wondering whether it's an issue with the scipy implementation or is this issue inherent to numerical implementations of the Anderson-Darling test (as I don't know the details of how the p-value is actually computed). I'd also be interested in some info on how to get around this bug. $\endgroup$ – Esteemator Jul 25 '16 at 20:24
  • 1
    $\begingroup$ Please include that clarification prominently in an edit to the question, lest others vote to close it (as being solely a computing issue). $\endgroup$ – whuber Jul 25 '16 at 20:26
  • 2
    $\begingroup$ Okay, added it midway through. $\endgroup$ – Esteemator Jul 25 '16 at 20:32
  • 1
    $\begingroup$ In case someone encounters the same issue, I'd like to update that as of Sept 2018 the extrapolation problem mentioned in this post has been fixed in the current scipy github. The critical value is now capped/floored at confidence interval 25% and 0.1%. This update should be available in the next version (1.2.0) $\endgroup$ – user12075 Sep 29 '18 at 1:59

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.