diff --git a/allel/stats/hw.py b/allel/stats/hw.py index 1ef7e20f..1212584e 100644 --- a/allel/stats/hw.py +++ b/allel/stats/hw.py @@ -93,7 +93,8 @@ def heterozygosity_expected(af, ploidy, fill=np.nan): out = 1 - np.sum(np.power(af, ploidy), axis=1) # fill values where allele frequencies could not be calculated - af_sum = np.sum(af, axis=1) + precision = np.finfo(af.dtype).precision + af_sum = np.round(np.sum(af, axis=1), precision) with ignore_invalid(): out[(af_sum < 1) | np.isnan(af_sum)] = fill diff --git a/allel/test/test_stats.py b/allel/test/test_stats.py index b2ee4b6e..2825e28a 100644 --- a/allel/test/test_stats.py +++ b/allel/test/test_stats.py @@ -425,7 +425,10 @@ def refimpl(f, ploidy, fill=0): """Limited reference implementation for testing purposes.""" # check allele frequencies sum to 1 - af_sum = np.sum(f, axis=1) + # round to suitable precision for the dtype + af_sum = (np.sum(f, axis=1)) + precision = np.finfo(af_sum.dtype).precision + af_sum = np.round(np.sum(f, axis=1), precision) # assume three alleles p = f[:, 0]