Skip to content

Commit

Permalink
Adjust some of the [rand.dist] critical values that are too strict
Browse files Browse the repository at this point in the history
 - Most critical values are determined empirically by running each test 51
   times with a different PRNG seed and finding the smallest symmetric interval
   around the median that contains 90% of the sample means, variances, etc.

 - For the Kolmogorov-Smirnov tests, the alpha=0.1 critical value for large N
   is 1.224/sqrt(N).

 - For normally distributed variates, the sample kurtosis is distributed as
   Normal(0, 24/N). For N=1e5, this gives a 90% confidence interval of
   0+/-0.0255. For Binomial(40, 0.25), which is approximately normal, the
   kurtosis is -0.0167, so the relative 90% CI is large, on the order of
   0.0255/0.0167 = 153%. In most cases the distribution of the sample kurtosis
   isn't known analytically, but similarly large relative tolerances can be
   expected if the kurtosis is near zero.
  • Loading branch information
MattStephanson committed Apr 27, 2024
1 parent 5927492 commit 788fc37
Show file tree
Hide file tree
Showing 28 changed files with 134 additions and 134 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ int main(int, char**)
double x_kurtosis = (6 * sqr(d.p()) - 6 * d.p() + 1)/x_var;
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.05);
}
{
typedef std::bernoulli_distribution D;
Expand Down Expand Up @@ -99,8 +99,8 @@ int main(int, char**)
double x_kurtosis = (6 * sqr(d.p()) - 6 * d.p() + 1)/x_var;
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.05);
}

return 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ int main(int, char**)
double x_kurtosis = (6 * sqr(p.p()) - 6 * p.p() + 1)/x_var;
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.05);
}
{
typedef std::bernoulli_distribution D;
Expand Down Expand Up @@ -103,8 +103,8 @@ int main(int, char**)
double x_kurtosis = (6 * sqr(p.p()) - 6 * p.p() + 1)/x_var;
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.05);
}

return 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ int main(int, char**) {
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.04);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.06);

return 0;
}
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ void test1() {
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.04);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.08);
}

template <class T>
Expand Down Expand Up @@ -109,8 +109,8 @@ void test2() {
double x_kurtosis = (1-6*d.p()*(1-d.p())) / x_var;
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.08);
}

template <class T>
Expand Down Expand Up @@ -151,8 +151,8 @@ void test3() {
double x_kurtosis = (1-6*d.p()*(1-d.p())) / x_var;
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.03);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.3);
assert(std::abs((skew - x_skew) / x_skew) < 0.07);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 2.0);
}

template <class T>
Expand Down Expand Up @@ -292,7 +292,7 @@ void test6() {
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs(skew - x_skew) < 0.02);
assert(std::abs(kurtosis - x_kurtosis) < 0.01);
assert(std::abs(kurtosis - x_kurtosis) < 0.03);
}

template <class T>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ int main(int, char**)
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.04);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.08);
}
{
typedef std::binomial_distribution<> D;
Expand Down Expand Up @@ -113,8 +113,8 @@ int main(int, char**)
double x_kurtosis = (1-6*p.p()*(1-p.p())) / x_var;
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.08);
}
{
typedef std::binomial_distribution<> D;
Expand Down Expand Up @@ -155,8 +155,8 @@ int main(int, char**)
double x_kurtosis = (1-6*p.p()*(1-p.p())) / x_var;
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.04);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.3);
assert(std::abs((skew - x_skew) / x_skew) < 0.07);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 2.0);
}

return 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ void test1() {
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
}

template <class T>
Expand Down Expand Up @@ -161,7 +161,7 @@ void test3() {
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
}

template <class T>
Expand Down Expand Up @@ -203,7 +203,7 @@ void test4() {
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
}

template <class T>
Expand Down Expand Up @@ -245,7 +245,7 @@ void test5() {
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
}

template <class T>
Expand Down Expand Up @@ -284,8 +284,8 @@ void test6() {
double x_var = x_mean / d.p();
double x_skew = (2 - d.p()) / std::sqrt((1 - d.p()));
double x_kurtosis = 6 + sqr(d.p()) / (1 - d.p());
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((mean - x_mean) / x_mean) < 0.02);
assert(std::abs((var - x_var) / x_var) < 0.02);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ int main(int, char**)
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
}
{
typedef std::geometric_distribution<> D;
Expand Down Expand Up @@ -156,7 +156,7 @@ int main(int, char**)
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
}

return 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ void test1() {
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
}

template <class T>
Expand Down Expand Up @@ -115,8 +115,8 @@ void test2() {
double x_kurtosis = 6. / d.k() + sqr(d.p()) / (d.k() * (1 - d.p()));
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.1);
}

template <class T>
Expand Down Expand Up @@ -157,8 +157,8 @@ void test3() {
double x_kurtosis = 6. / d.k() + sqr(d.p()) / (d.k() * (1 - d.p()));
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.08);
}

template <class T>
Expand Down Expand Up @@ -243,8 +243,8 @@ void test5() {
double x_kurtosis = 6. / d.k() + sqr(d.p()) / (d.k() * (1 - d.p()));
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.04);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.05);
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.3);
}

template <class T>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ int main(int, char**)
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
}
{
typedef std::negative_binomial_distribution<> D;
Expand Down Expand Up @@ -113,8 +113,8 @@ int main(int, char**)
double x_kurtosis = 6. / p.k() + sqr(p.p()) / (p.k() * (1 - p.p()));
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.1);
}
{
typedef std::negative_binomial_distribution<> D;
Expand Down Expand Up @@ -155,8 +155,8 @@ int main(int, char**)
double x_kurtosis = 6. / p.k() + sqr(p.p()) / (p.k() * (1 - p.p()));
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.08);
}

return 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ int main(int, char**)
u.push_back(d(g));
std::sort(u.begin(), u.end());
for (int i = 0; i < N; ++i)
assert(std::abs(f(u[i], a, b) - double(i)/N) < .001);
assert(std::abs(f(u[i], a, b) - double(i)/N) < .0013);
}
{
typedef std::cauchy_distribution<> D;
Expand All @@ -60,7 +60,7 @@ int main(int, char**)
u.push_back(d(g));
std::sort(u.begin(), u.end());
for (int i = 0; i < N; ++i)
assert(std::abs(f(u[i], a, b) - double(i)/N) < .001);
assert(std::abs(f(u[i], a, b) - double(i)/N) < .0013);
}
{
typedef std::cauchy_distribution<> D;
Expand All @@ -75,7 +75,7 @@ int main(int, char**)
u.push_back(d(g));
std::sort(u.begin(), u.end());
for (int i = 0; i < N; ++i)
assert(std::abs(f(u[i], a, b) - double(i)/N) < .001);
assert(std::abs(f(u[i], a, b) - double(i)/N) < .0013);
}

return 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ int main(int, char**)
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.04);
}
{
typedef std::chi_squared_distribution<> D;
Expand Down Expand Up @@ -109,7 +109,7 @@ int main(int, char**)
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
}
{
typedef std::chi_squared_distribution<> D;
Expand Down Expand Up @@ -148,7 +148,7 @@ int main(int, char**)
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
}

return 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ int main(int, char**)
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
}
{
typedef std::chi_squared_distribution<> D;
Expand Down Expand Up @@ -113,7 +113,7 @@ int main(int, char**)
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
}
{
typedef std::chi_squared_distribution<> D;
Expand Down Expand Up @@ -154,7 +154,7 @@ int main(int, char**)
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.04);
}

return 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,8 @@ test1()
3*std::exp(2*sqr(d.s())) - 6;
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.05);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.25);
assert(std::abs((skew - x_skew) / x_skew) < 0.1);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 1.9);
}

void
Expand Down Expand Up @@ -115,7 +115,7 @@ test2()
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.04);
}

void
Expand Down Expand Up @@ -159,7 +159,7 @@ test3()
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.01);
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.05);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.06);
}

void
Expand Down Expand Up @@ -202,8 +202,8 @@ test4()
3*std::exp(2*sqr(d.s())) - 6;
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.02);
assert(std::abs((skew - x_skew) / x_skew) < 0.08);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.4);
assert(std::abs((skew - x_skew) / x_skew) < 0.1);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.5);
}

void
Expand Down Expand Up @@ -245,9 +245,9 @@ test5()
double x_kurtosis = std::exp(4*sqr(d.s())) + 2*std::exp(3*sqr(d.s())) +
3*std::exp(2*sqr(d.s())) - 6;
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
assert(std::abs((var - x_var) / x_var) < 0.04);
assert(std::abs((skew - x_skew) / x_skew) < 0.2);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.7);
assert(std::abs((var - x_var) / x_var) < 0.05);
assert(std::abs((skew - x_skew) / x_skew) < 0.3);
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 1.0);
}

int main(int, char**)
Expand Down
Loading

0 comments on commit 788fc37

Please sign in to comment.