From 8fcc7a60c7aa828ef3f043bfe6e0e5e54c17e27a Mon Sep 17 00:00:00 2001 From: Jacob Schreiber Date: Fri, 16 Oct 2026 10:14:30 -0400 Subject: [PATCH 1/3] Add a pytest suite for the bowtie2, bowtie2-build and bowtie2-inspect CLIs 612 tests in scripts/test/test_*.py covering input formats and read preprocessing, alignment, scoring, reporting and effort options, paired-end semantics, output and SAM options, the wrapper scripts, and index building and inspection. Expected values are derived from reads cut at known positions from fixed random references, using the MANUAL's scoring rules; tests that pin current output say so in their docstring. Run after building, from the repository root: make -j4 allall python3 -m pytest scripts/test Optional tools (zstd, lz4, bzip2, samtools) and the -debug/-sanitized binaries are skipped when absent. BOWTIE2_BIN_DIR points the suite at binaries built elsewhere. Co-Authored-By: Claude Opus 5.5 --- scripts/test/bt2_helpers.py | 256 ++++++ scripts/test/conftest.py | 87 ++ scripts/test/test_align.py | 1179 ++++++++++++++++++++++++ scripts/test/test_build_inspect.py | 938 +++++++++++++++++++ scripts/test/test_input.py | 995 ++++++++++++++++++++ scripts/test/test_output.py | 1358 ++++++++++++++++++++++++++++ scripts/test/test_paired.py | 1056 +++++++++++++++++++++ 7 files changed, 5869 insertions(+) create mode 100644 scripts/test/bt2_helpers.py create mode 100644 scripts/test/conftest.py create mode 100644 scripts/test/test_align.py create mode 100644 scripts/test/test_build_inspect.py create mode 100644 scripts/test/test_input.py create mode 100644 scripts/test/test_output.py create mode 100644 scripts/test/test_paired.py diff --git a/scripts/test/bt2_helpers.py b/scripts/test/bt2_helpers.py new file mode 100644 index 00000000..38065ab7 --- /dev/null +++ b/scripts/test/bt2_helpers.py @@ -0,0 +1,256 @@ +"""Helpers shared by the pytest suite in scripts/test (test_*.py). + +Everything here is plain Python so that expected values in the tests can be +derived independently of bowtie2: references are generated from a fixed +seed, reads are cut from them at known positions, and SAM output is parsed +into records whose fields the tests compare against those positions. +""" + +import gzip +import bz2 +import os +import random +import shutil +import subprocess + + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +BIN_DIR = os.path.abspath(os.environ.get('BOWTIE2_BIN_DIR', ROOT)) +EXAMPLE_DIR = os.path.join(ROOT, 'example') + +COMPLEMENT = str.maketrans('ACGTNacgtn', 'TGCANtgcan') + +# SAM FLAG bits +PAIRED = 0x1 +PROPER_PAIR = 0x2 +UNMAPPED = 0x4 +MATE_UNMAPPED = 0x8 +REVERSE = 0x10 +MATE_REVERSE = 0x20 +FIRST_IN_PAIR = 0x40 +SECOND_IN_PAIR = 0x80 +SECONDARY = 0x100 +QC_FAIL = 0x200 + + +def revcomp(seq): + return seq.translate(COMPLEMENT)[::-1] + + +def random_dna(length, rng): + return ''.join(rng.choice('ACGT') for _ in range(length)) + + +def mutate(seq, positions): + """Substitute the base at each 0-based position with a different base. + + The replacement is the next base in ACGT order, so the result is + deterministic and always differs from the original. + """ + seq = list(seq) + for p in positions: + seq[p] = 'ACGT'[('ACGT'.index(seq[p]) + 1) % 4] + return ''.join(seq) + + +def write_fasta(path, records, width=60): + """Write (name, sequence) pairs to a FASTA file; returns path.""" + with open(path, 'w') as f: + for name, seq in records: + f.write('>%s\n' % name) + for i in range(0, len(seq), width): + f.write(seq[i:i + width] + '\n') + return path + + +def fastq_text(reads): + """FASTQ text for (name, seq, qual) triples; qual defaults to all 'I'.""" + out = [] + for read in reads: + name, seq = read[0], read[1] + qual = read[2] if len(read) > 2 and read[2] is not None else 'I' * len(seq) + out.append('@%s\n%s\n+\n%s\n' % (name, seq, qual)) + return ''.join(out) + + +def write_fastq(path, reads, compress=None): + """Write reads to FASTQ, optionally gzip or bzip2 compressed; returns path.""" + text = fastq_text(reads).encode() + if compress == 'gz': + with gzip.open(path, 'wb') as f: + f.write(text) + elif compress == 'bz2': + with bz2.open(path, 'wb') as f: + f.write(text) + else: + with open(path, 'wb') as f: + f.write(text) + return path + + +def write_fasta_reads(path, reads): + """Write (name, seq) read pairs as multi-FASTA, one line per sequence.""" + with open(path, 'w') as f: + for read in reads: + f.write('>%s\n%s\n' % (read[0], read[1])) + return path + + +def read_fastq(path): + """Parse a FASTQ file (plain, .gz or .bz2) into (name, seq, qual) triples.""" + opener = open + if path.endswith('.gz'): + opener = gzip.open + elif path.endswith('.bz2'): + opener = bz2.open + with opener(path, 'rt') as f: + lines = [line.rstrip('\n') for line in f] + reads = [] + for i in range(0, len(lines) - 3, 4): + assert lines[i].startswith('@'), lines[i] + reads.append((lines[i][1:], lines[i + 1], lines[i + 3])) + return reads + + +class SamRecord(object): + """One SAM alignment line. Optional fields are in `tags`, keyed by name, + with integer values for type i and strings otherwise.""" + + __slots__ = ('qname', 'flag', 'rname', 'pos', 'mapq', 'cigar', 'rnext', + 'pnext', 'tlen', 'seq', 'qual', 'tags', 'tag_order', 'line') + + def __init__(self, line): + fields = line.rstrip('\n').split('\t') + assert len(fields) >= 11, line + self.line = line.rstrip('\n') + self.qname = fields[0] + self.flag = int(fields[1]) + self.rname = fields[2] + self.pos = int(fields[3]) + self.mapq = int(fields[4]) + self.cigar = fields[5] + self.rnext = fields[6] + self.pnext = int(fields[7]) + self.tlen = int(fields[8]) + self.seq = fields[9] + self.qual = fields[10] + self.tags = {} + self.tag_order = [] + for field in fields[11:]: + name, typ, value = field.split(':', 2) + self.tags[name] = int(value) if typ == 'i' else value + self.tag_order.append(name) + + @property + def is_unmapped(self): + return bool(self.flag & UNMAPPED) + + @property + def is_reverse(self): + return bool(self.flag & REVERSE) + + @property + def is_secondary(self): + return bool(self.flag & SECONDARY) + + def __repr__(self): + return 'SamRecord(%r)' % self.line + + +def parse_sam(text): + """Split SAM text into (header lines, list of SamRecord).""" + header, records = [], [] + for line in text.splitlines(): + if not line: + continue + if line.startswith('@'): + header.append(line) + else: + records.append(SamRecord(line)) + return header, records + + +def cigar_ops(cigar): + """'10M2I5M' -> [(10, 'M'), (2, 'I'), (5, 'M')].""" + ops, num = [], '' + for c in cigar: + if c.isdigit(): + num += c + else: + ops.append((int(num), c)) + num = '' + return ops + + +def cigar_ref_length(cigar): + return sum(n for n, op in cigar_ops(cigar) if op in 'MDN=X') + + +def cigar_query_length(cigar): + return sum(n for n, op in cigar_ops(cigar) if op in 'MIS=X') + + +def mismatch_penalty(qual_char, mx=6, mn=2, phred_offset=33): + """bowtie2's quality-aware mismatch penalty, from the MANUAL's --mp: + MN + floor((MX - MN) * (min(Q, 40) / 40)).""" + q = min(ord(qual_char) - phred_offset, 40) + return mn + (mx - mn) * q // 40 + + +class Bowtie2(object): + """Runs the bowtie2, bowtie2-build and bowtie2-inspect wrappers.""" + + def __init__(self, bin_dir=BIN_DIR): + self.bin_dir = bin_dir + self.bowtie2 = os.path.join(bin_dir, 'bowtie2') + self.build = os.path.join(bin_dir, 'bowtie2-build') + self.inspect = os.path.join(bin_dir, 'bowtie2-inspect') + + def has_binary(self, name): + return os.access(os.path.join(self.bin_dir, name), os.X_OK) + + def run(self, prog, args, input=None, check=True, cwd=None, env=None, + timeout=300): + """Run one of the three wrappers with a list of arguments. + + Returns the CompletedProcess with text stdout and stderr. With + check=True a non-zero exit raises AssertionError showing stderr. + """ + exe = {'bowtie2': self.bowtie2, 'bowtie2-build': self.build, + 'bowtie2-inspect': self.inspect}.get(prog, prog) + cmd = [exe] + [str(a) for a in args] + proc = subprocess.run(cmd, input=input, capture_output=True, + text=True, cwd=cwd, env=env, timeout=timeout) + if check: + assert proc.returncode == 0, 'exit %d from %s\nstderr:\n%s' % ( + proc.returncode, ' '.join(cmd), proc.stderr) + return proc + + def align(self, args, input=None, check=True, cwd=None, env=None): + """Run bowtie2 with SAM on stdout; returns (header, records, proc).""" + proc = self.run('bowtie2', args, input=input, check=check, cwd=cwd, + env=env) + header, records = parse_sam(proc.stdout) if proc.returncode == 0 else ([], []) + return header, records, proc + + def build_index(self, fasta, prefix, extra=(), check=True): + """bowtie2-build a FASTA (or comma-separated list) into prefix.""" + return self.run('bowtie2-build', list(extra) + ['--quiet', fasta, prefix], + check=check) + + +def by_name(records): + """Map qname -> list of records, in output order.""" + out = {} + for r in records: + out.setdefault(r.qname, []).append(r) + return out + + +def primary(records): + """Records that are neither secondary nor supplementary.""" + return [r for r in records if not r.flag & (SECONDARY | 0x800)] + + +def which(name): + return shutil.which(name) diff --git a/scripts/test/conftest.py b/scripts/test/conftest.py new file mode 100644 index 00000000..8d0eb84b --- /dev/null +++ b/scripts/test/conftest.py @@ -0,0 +1,87 @@ +"""Fixtures for the bowtie2 command-line test suite (scripts/test/test_*.py). + +Build first, then run from the repository root: + + make -j4 allall + python3 -m pytest scripts/test + +Set BOWTIE2_BIN_DIR to test binaries that were built somewhere else. +""" + +import os +import random + +import pytest + +import bt2_helpers as h + + +def pytest_report_header(config): + return 'bowtie2 binaries: %s' % h.BIN_DIR + + +@pytest.fixture(scope='session') +def bt2(): + runner = h.Bowtie2() + for prog in ('bowtie2', 'bowtie2-build', 'bowtie2-inspect', 'bowtie2-align-s', + 'bowtie2-build-s', 'bowtie2-inspect-s'): + if not runner.has_binary(prog): + pytest.fail('%s not found in %s: build bowtie2 first (make), or set ' + 'BOWTIE2_BIN_DIR' % (prog, runner.bin_dir), pytrace=False) + return runner + + +@pytest.fixture(scope='session') +def genome(): + """Three random chromosomes, fixed by seed, as an ordered list of + (name, sequence). At these lengths every 20-mer is unique with high + probability, so a read cut from one place aligns only there.""" + rng = random.Random(20261005) + return [('chr1', h.random_dna(5000, rng)), ('chr2', h.random_dna(3000, rng)), + ('chr3', h.random_dna(2000, rng))] + + +@pytest.fixture(scope='session') +def genome_dict(genome): + return dict(genome) + + +@pytest.fixture(scope='session') +def genome_fasta(genome, tmp_path_factory): + path = tmp_path_factory.mktemp('genome') / 'genome.fa' + return str(h.write_fasta(str(path), genome)) + + +@pytest.fixture(scope='session') +def genome_index(bt2, genome_fasta, tmp_path_factory): + """Small (.bt2) index of `genome`.""" + prefix = str(tmp_path_factory.mktemp('genome_index') / 'genome') + bt2.build_index(genome_fasta, prefix, extra=['--seed', '0']) + return prefix + + +@pytest.fixture(scope='session') +def genome_index_large(bt2, genome_fasta, tmp_path_factory): + """Large (.bt2l) index of `genome`, built with --large-index.""" + prefix = str(tmp_path_factory.mktemp('genome_index_large') / 'genome') + bt2.build_index(genome_fasta, prefix, extra=['--large-index', '--seed', '0']) + return prefix + + +@pytest.fixture(scope='session') +def lambda_fasta(): + return os.path.join(h.EXAMPLE_DIR, 'reference', 'lambda_virus.fa') + + +@pytest.fixture(scope='session') +def lambda_index(): + """The index shipped in example/index.""" + return os.path.join(h.EXAMPLE_DIR, 'index', 'lambda_virus') + + +@pytest.fixture(scope='session') +def example_reads(): + d = os.path.join(h.EXAMPLE_DIR, 'reads') + return {'reads_1': os.path.join(d, 'reads_1.fq'), 'reads_2': os.path.join(d, 'reads_2.fq'), + 'longreads': os.path.join(d, 'longreads.fq'), + 'bam': os.path.join(d, 'combined_reads.bam')} diff --git a/scripts/test/test_align.py b/scripts/test/test_align.py new file mode 100644 index 00000000..75797932 --- /dev/null +++ b/scripts/test/test_align.py @@ -0,0 +1,1179 @@ +"""Alignment, scoring, reporting and effort options of bowtie2. + +Reads are cut from fixed random references at known positions, with +substitutions, Ns, insertions and deletions placed at known offsets, so the +expected POS, strand, CIGAR, MD, NM, XM, XO, XG and AS of each alignment are +computed here from the MANUAL's scoring rules before bowtie2 runs: + +- end-to-end: mismatch penalty MN + floor((MX - MN) * min(Q, 40) / 40) with + MX = 6, MN = 2; N penalty 1; read gap (deletion, CIGAR D) and reference gap + (insertion, CIGAR I) of length n cost 5 + 3n; the best score is 0. +- local: as above, plus a bonus of 2 for each matching base. + +Repeats come from a second, small reference (`rep`) with a segment present +three times exactly and once with two substitutions, and from a third +(`family`) holding 800 slightly diverged copies of one element, used where an +option only changes behaviour for reads with repetitive seeds (-D, -R). +""" + +import random +import re + +import pytest + +import bt2_helpers as h + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def write_reads(tmp_path, reads, name='reads.fq'): + return h.write_fastq(str(tmp_path / name), reads) + + +def align(bt2, index, fq, args=(), check=True): + """Align unpaired FASTQ `fq` with -p 1; returns the SAM records.""" + _, records, _ = bt2.align(['-p', '1', '-x', index, '-U', fq] + [str(a) for a in args], + check=check) + return records + + +def body(records): + """SAM lines without the header (whose @PG line holds the command line).""" + return [r.line for r in records] + + +def same_records(a, b, nreads): + """Records a and b are identical and cover all `nreads` reads, so the + comparison cannot pass on two empty or truncated outputs.""" + assert len(set(r.qname for r in a)) == nreads + assert body(a) == body(b) + + +def one_per_read(records): + """Map qname -> its single record; fails if a read has several.""" + out = {} + for r in records: + assert r.qname not in out, 'more than one record for %s' % r.qname + out[r.qname] = r + return out + + +def other_base(*avoid): + return [b for b in 'ACGT' if b not in avoid][0] + + +def gaps_unambiguous(ref, start, ops): + """True if every gap in `ops`, applied at `start`, has one placement only + (it cannot slide left or right through equal bases).""" + pos = start + for op in ops: + kind = op[0] + if kind == 'D': + n = op[1] + if ref[pos - 1] == ref[pos + n - 1] or ref[pos] == ref[pos + n]: + return False + pos += n + elif kind == 'I': + pass # Edited chooses inserted bases that cannot slide + elif kind == '=': + pos += op[1] + else: + pos += 1 + return True + + +def place(ref, start, ops): + """First offset >= start where the gaps in `ops` are unambiguous.""" + while not gaps_unambiguous(ref, start, ops): + start += 1 + return start + + +class Edited(object): + """A read built from a reference and an edit script, with the alignment + fields bowtie2 should report for it on the forward strand.""" + + def __init__(self, ref, start, ops): + seq, cigar, md = [], [], [] + self.sub_offsets = [] # read offsets of substitutions + self.n_offsets = [] # read offsets of read Ns + self.dels, self.inss = [], [] + self.matches = 0 + run, pos = 0, start + + def add(op, n): + if cigar and cigar[-1][1] == op: + cigar[-1] = (cigar[-1][0] + n, op) + else: + cigar.append((n, op)) + + for op in ops: + kind = op[0] + if kind == '=': + n = op[1] + seq.append(ref[pos:pos + n]) + add('M', n) + run += n + pos += n + self.matches += n + elif kind in ('X', 'N'): + offset = sum(len(s) for s in seq) + if kind == 'X': + seq.append(h.mutate(ref[pos], [0])) + self.sub_offsets.append(offset) + else: + seq.append('N') + self.n_offsets.append(offset) + add('M', 1) + md.append('%d%s' % (run, ref[pos])) + run = 0 + pos += 1 + elif kind == 'D': + n = op[1] + add('D', n) + md.append('%d^%s' % (run, ref[pos:pos + n])) + run = 0 + pos += n + self.dels.append(n) + elif kind == 'I': + # The first inserted base differs from the next reference base + # and the last from the previous one, so the gap cannot slide. + n = op[1] + if n == 1: + seq.append(other_base(ref[pos - 1], ref[pos])) + else: + seq.append(other_base(ref[pos]) + 'A' * (n - 2) + other_base(ref[pos - 1])) + add('I', n) + self.inss.append(n) + else: + raise ValueError(op) + md.append(str(run)) + self.seq = ''.join(seq) + self.pos = start + 1 + self.cigar = ''.join('%d%s' % (n, op) for n, op in cigar) + self.md = ''.join(md) + self.xm = len(self.sub_offsets) + len(self.n_offsets) + self.xo = len(self.dels) + len(self.inss) + self.xg = sum(self.dels) + sum(self.inss) + self.nm = self.xm + self.xg + + def penalty(self, qual, mx=6, mn=2, np=1, rdg=(5, 3), rfg=(5, 3), ignore_quals=False): + """Sum of the MANUAL's penalties for this read's edits.""" + pen = 0 + for off in self.sub_offsets: + pen += mx if ignore_quals else h.mismatch_penalty(qual[off], mx=mx, mn=mn) + pen += np * len(self.n_offsets) + pen += sum(rdg[0] + rdg[1] * n for n in self.dels) + pen += sum(rfg[0] + rfg[1] * n for n in self.inss) + return pen + + def e2e_score(self, qual, **kw): + return -self.penalty(qual, **kw) + + def local_score(self, qual, ma=2, **kw): + return ma * self.matches - self.penalty(qual, **kw) + + +def fastq_record(name, edited, qual, reverse): + """(name, seq, qual) as sequenced from the forward or reverse strand.""" + if reverse: + return (name, h.revcomp(edited.seq), qual[::-1]) + return (name, edited.seq, qual) + + +def check_fields(rec, rname, edited, qual, reverse, score): + """Assert every alignment field of `rec` against the read's construction.""" + assert not rec.is_unmapped, rec + assert rec.flag == (h.REVERSE if reverse else 0), rec + assert rec.rname == rname + assert rec.pos == edited.pos + assert rec.cigar == edited.cigar + assert rec.seq == edited.seq + assert rec.qual == qual + assert rec.tags['AS'] == score + assert rec.tags['MD'] == edited.md + assert rec.tags['NM'] == edited.nm + assert rec.tags['XM'] == edited.xm + assert rec.tags['XO'] == edited.xo + assert rec.tags['XG'] == edited.xg + assert rec.tags['XN'] == 0 + assert rec.tags['YT'] == 'UU' + assert 'XS' not in rec.tags + assert 'YF' not in rec.tags + + +def qual_with(length, changes): + q = ['I'] * length + for off, c in changes.items(): + q[off] = c + return ''.join(q) + + +# --------------------------------------------------------------------------- +# Reads with known edits against the shared `genome` +# --------------------------------------------------------------------------- + +# name: (chromosome, approximate start, edit script, {read offset: quality}) +EDIT_READS = { + 'perfect': ('chr1', 100, [('=', 100)], {}), + # Q40 ('I'), Q20 ('5') and Q10 ('+') substitutions: penalties 6, 4, 3. + 'subs': ('chr1', 1000, [('=', 20), ('X',), ('=', 39), ('X',), ('=', 24), ('X',), + ('=', 14)], {60: '5', 85: '+'}), + 'del3': ('chr1', 1500, [('=', 50), ('D', 3), ('=', 50)], {}), + 'ins2': ('chr2', 1000, [('=', 40), ('I', 2), ('=', 58)], {}), + 'combo': ('chr2', 2000, [('=', 8), ('X',), ('=', 16), ('X',), ('=', 20), ('D', 2), + ('=', 46), ('I', 1), ('=', 7)], {25: '5'}), + 'ns': ('chr3', 200, [('=', 30), ('N',), ('=', 19), ('N',), ('=', 49)], {}), +} + + +@pytest.fixture(scope='module') +def edit_reads(genome_dict): + """name -> (rname, Edited, qual) for EDIT_READS, gaps placed unambiguously.""" + out = {} + for name, (rname, start, ops, quals) in EDIT_READS.items(): + ref = genome_dict[rname] + edited = Edited(ref, place(ref, start, ops), ops) + out[name] = (rname, edited, qual_with(len(edited.seq), quals)) + return out + + +@pytest.fixture(scope='module') +def edit_fastq(edit_reads, tmp_path_factory): + """FASTQ of every EDIT_READS read, from both strands (suffix _fw / _rc).""" + reads = [] + for name, (rname, edited, qual) in sorted(edit_reads.items()): + for strand, rev in (('fw', False), ('rc', True)): + reads.append(fastq_record('%s_%s' % (name, strand), edited, qual, rev)) + path = tmp_path_factory.mktemp('edit_reads') / 'reads.fq' + return h.write_fastq(str(path), reads) + + +@pytest.fixture(scope='module') +def edit_e2e(bt2, genome_index, edit_fastq): + return one_per_read(align(bt2, genome_index, edit_fastq)) + + +@pytest.mark.parametrize('name', sorted(EDIT_READS)) +@pytest.mark.parametrize('strand', ['fw', 'rc']) +def test_fields_end_to_end(edit_reads, edit_e2e, name, strand): + """POS, FLAG, CIGAR, SEQ, QUAL, AS, MD, NM, XM, XO, XG, XN, YT match the read's construction.""" + rname, edited, qual = edit_reads[name] + rec = edit_e2e['%s_%s' % (name, strand)] + check_fields(rec, rname, edited, qual, strand == 'rc', edited.e2e_score(qual)) + + +def test_fields_expected_values_spelled_out(edit_reads, edit_e2e): + """The combined read's fields, written out literally as a check on the helper.""" + rname, edited, qual = edit_reads['combo'] + rec = edit_e2e['combo_fw'] + # Q40 substitution (6) + Q20 substitution (4) + 2 bp deletion (5 + 2*3) + # + 1 bp insertion (5 + 3). + assert rec.tags['AS'] == -(6 + 4 + 11 + 8) + assert rec.cigar == '46M2D46M1I7M' + assert (rec.tags['XM'], rec.tags['XO'], rec.tags['XG'], rec.tags['NM']) == (2, 2, 3, 5) + assert re.match(r'^8[ACGT]16[ACGT]20\^[ACGT]{2}53$', rec.tags['MD']), rec.tags['MD'] + + +def test_end_to_end_is_default(bt2, genome_index, edit_fastq): + """--end-to-end gives the same SAM records as the default mode.""" + same_records(align(bt2, genome_index, edit_fastq, ['--end-to-end']), + align(bt2, genome_index, edit_fastq), 2 * len(EDIT_READS)) + + +def test_end_to_end_never_clips(edit_e2e): + """In --end-to-end mode every aligned read is aligned over its full length.""" + assert len(edit_e2e) == 2 * len(EDIT_READS) + for rec in edit_e2e.values(): + assert 'S' not in rec.cigar + assert h.cigar_query_length(rec.cigar) == len(rec.seq) + assert rec.tags['AS'] <= 0 + + +def test_ns_in_read_count_as_mismatches(edit_e2e): + """Read Ns cost --np each, appear in MD as the reference base and count in XM (pins current XM).""" + rec = edit_e2e['ns_fw'] + assert re.match(r'^30[ACGT]19[ACGT]49$', rec.tags['MD']), rec.tags['MD'] + assert rec.tags['AS'] == -2 + assert rec.tags['XM'] == 2 + assert rec.tags['NM'] == 2 + + +# --------------------------------------------------------------------------- +# Local mode +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize('ma', [None, 3]) +def test_local_scores_whole_reads(bt2, genome_index, edit_reads, edit_fastq, ma): + """--local (and --ma) adds the match bonus per matching base and keeps reads that align end to end unclipped.""" + args = ['--local'] + (['--ma', ma] if ma else []) + recs = one_per_read(align(bt2, genome_index, edit_fastq, args)) + for name, (rname, edited, qual) in edit_reads.items(): + for strand in ('fw', 'rc'): + rec = recs['%s_%s' % (name, strand)] + check_fields(rec, rname, edited, qual, strand == 'rc', + edited.local_score(qual, ma=ma or 2)) + + +@pytest.fixture(scope='module') +def clip_reads(genome_dict): + """Reads whose 10-base ends differ from the reference at every base, + around an 80-base core (exact, or with one Q40 substitution).""" + ref = genome_dict['chr3'] + s = 1000 + left = h.mutate(ref[s - 10:s], range(10)) + right = h.mutate(ref[s + 80:s + 90], range(10)) + exact = Edited(ref, s, [('=', 80)]) + sub = Edited(ref, s, [('=', 40), ('X',), ('=', 39)]) + return { + 'clip': (left + exact.seq + right, exact), + 'clip_sub': (left + sub.seq + right, sub), + } + + +@pytest.mark.parametrize('strand', ['fw', 'rc']) +def test_local_soft_clips_mismatched_ends(bt2, genome_index, clip_reads, tmp_path, strand): + """--local soft clips ends that do not match: CIGAR 10S..10S, POS at the core, AS = 2 * matches - penalties.""" + reads = [] + for name, (seq, _) in sorted(clip_reads.items()): + reads.append((name, h.revcomp(seq) if strand == 'rc' else seq)) + fq = write_reads(tmp_path, reads) + recs = one_per_read(align(bt2, genome_index, fq, ['--local'])) + for name, (seq, core) in clip_reads.items(): + rec = recs[name] + assert rec.flag == (h.REVERSE if strand == 'rc' else 0) + assert (rec.rname, rec.pos) == ('chr3', core.pos) + assert rec.cigar == '10S%s10S' % core.cigar + assert rec.seq == seq + assert rec.tags['MD'] == core.md + assert rec.tags['NM'] == core.nm + assert rec.tags['AS'] == core.local_score('I' * 100) + assert recs['clip'].tags['AS'] == 160 + assert recs['clip_sub'].tags['AS'] == 2 * 79 - 6 + # The same reads cannot align end to end: 20 mismatched ends cost far + # more than the default minimum score allows. + e2e = align(bt2, genome_index, fq) + assert len(e2e) == 2 + for rec in e2e: + assert rec.is_unmapped + + +@pytest.mark.parametrize('order', ['before', 'after']) +@pytest.mark.parametrize('preset', ['very-fast', 'fast', 'sensitive', 'very-sensitive']) +def test_local_with_preset_is_local_preset(bt2, lambda_index, example_reads, preset, order): + """--local combined with an end-to-end preset name equals the -local preset (MANUAL, --local).""" + fq = example_reads['reads_1'] + args = ['--local', '--' + preset] if order == 'before' else ['--' + preset, '--local'] + same_records(align(bt2, lambda_index, fq, args + ['-u', 1000]), + align(bt2, lambda_index, fq, ['--%s-local' % preset, '-u', 1000]), 1000) + + +# --------------------------------------------------------------------------- +# Presets +# --------------------------------------------------------------------------- + +PRESETS = { + 'very-fast': '-D 5 -R 1 -N 0 -L 22 -i S,0,2.50', + 'fast': '-D 10 -R 2 -N 0 -L 22 -i S,0,2.50', + 'sensitive': '-D 15 -R 2 -N 0 -L 22 -i S,1,1.15', + 'very-sensitive': '-D 20 -R 3 -N 0 -L 20 -i S,1,0.50', + 'very-fast-local': '-D 5 -R 1 -N 0 -L 25 -i S,1,2.00', + 'fast-local': '-D 10 -R 2 -N 0 -L 22 -i S,1,1.75', + 'sensitive-local': '-D 15 -R 2 -N 0 -L 20 -i S,1,0.75', + 'very-sensitive-local': '-D 20 -R 3 -N 0 -L 20 -i S,1,0.50', +} + + +def family(seed, elen, ncopies, ndiv, spacer): + """One random element, `ncopies` copies each with `ndiv` substitutions, + separated by random spacers. Returns (sequence, copy start offsets).""" + rng = random.Random(seed) + element = h.random_dna(elen, rng) + parts, origins, pos = [], [], 0 + for _ in range(ncopies): + parts.append(h.random_dna(spacer, rng)) + pos += spacer + parts.append(h.mutate(element, sorted(rng.sample(range(elen), ndiv)))) + origins.append(pos) + pos += elen + parts.append(h.random_dna(spacer, rng)) + return ''.join(parts), origins + + +@pytest.fixture(scope='module') +def family_index(bt2, tmp_path_factory): + """Index of 800 copies of a 200 bp element, 2 substitutions per copy.""" + seq, _ = family(11, 200, 800, 2, 20) + d = tmp_path_factory.mktemp('family') + fa = h.write_fasta(str(d / 'family.fa'), [('fam', seq)]) + prefix = str(d / 'family') + bt2.build_index(fa, prefix, extra=['--seed', '0']) + return prefix + + +@pytest.fixture(scope='module') +def family_fastq(tmp_path_factory): + """300 reads of 100 bp from random copies, 3 substitutions each, either strand. + Their 22-mer seeds hit hundreds of copies, so they are repetitive reads.""" + seq, origins = family(11, 200, 800, 2, 20) + rng = random.Random(3) + reads = [] + for i in range(300): + s = rng.choice(origins) + rng.randrange(0, 100) + r = h.mutate(seq[s:s + 100], sorted(rng.sample(range(100), 3))) + if rng.random() < 0.5: + r = h.revcomp(r) + reads.append(('f%d' % i, r)) + path = tmp_path_factory.mktemp('family_reads') / 'reads.fq' + return h.write_fastq(str(path), reads) + + +@pytest.fixture(scope='module') +def preset_inputs(lambda_index, example_reads, family_index, family_fastq): + """(index, reads, extra args, number of reads): the shipped lambda reads + (non-repetitive, sensitive to -N/-L/-i) and the family reads (sensitive + to -D/-R).""" + return { + 'lambda': (lambda_index, example_reads['reads_1'], ['-u', 3000], 3000), + 'family': (family_index, family_fastq, [], 300), + } + + +@pytest.mark.parametrize('dataset', ['lambda', 'family']) +@pytest.mark.parametrize('preset', sorted(PRESETS)) +def test_preset_equals_expansion(bt2, preset_inputs, preset, dataset): + """Each preset gives the same SAM records as its documented -D/-R/-N/-L/-i expansion.""" + index, fq, extra, nreads = preset_inputs[dataset] + mode = ['--local'] if preset.endswith('-local') else [] + same_records(align(bt2, index, fq, ['--' + preset] + extra), + align(bt2, index, fq, mode + PRESETS[preset].split() + extra), nreads) + + +@pytest.mark.parametrize('mode', ['end-to-end', 'local']) +def test_presets_differ_from_each_other(bt2, preset_inputs, mode): + """The four presets of a mode give four different outputs, so the equivalence tests are not vacuous.""" + index, fq, extra, _ = preset_inputs['family'] + names = [p for p in PRESETS if p.endswith('-local') == (mode == 'local')] + outputs = set() + for p in names: + outputs.add(tuple(body(align(bt2, index, fq, ['--' + p] + extra)))) + assert len(outputs) == 4 + + +@pytest.mark.parametrize('mode,preset', [('end-to-end', 'sensitive'), ('local', 'sensitive-local')]) +def test_default_preset_is_sensitive(bt2, preset_inputs, mode, preset): + """With no preset, --end-to-end uses --sensitive and --local uses --sensitive-local.""" + for index, fq, extra, nreads in preset_inputs.values(): + same_records(align(bt2, index, fq, ['--' + mode] + extra), + align(bt2, index, fq, ['--' + preset] + extra), nreads) + + +@pytest.mark.parametrize('preset', ['very-fast', 'very-sensitive', 'fast-local']) +def test_preset_option_equals_preset_flag(bt2, preset_inputs, preset): + """The undocumented --preset equals -- (with --local for a -local name).""" + index, fq, extra, nreads = preset_inputs['family'] + mode = ['--local'] if preset.endswith('-local') else [] + same_records(align(bt2, index, fq, mode + ['--preset', preset] + extra), + align(bt2, index, fq, ['--' + preset] + extra), nreads) + + +# --------------------------------------------------------------------------- +# Seed options: -N, -L, -i, --no-1mm-upfront +# --------------------------------------------------------------------------- + +def mapped(bt2, index, tmp_path, seq, args, qual=None): + """Align one read; returns its record (mapped or not).""" + fq = write_reads(tmp_path, [('r', seq, qual)]) + recs = align(bt2, index, fq, args) + assert len(recs) == 1 + return recs[0] + + +def test_seed_mismatches_N(bt2, genome_index, genome_dict, tmp_path): + """-N 1 finds a read whose every seed has one mismatch; -N 0 cannot.""" + # 50 bp, seeds of 20 every 20 bases (offsets 0 and 20), one substitution + # in each seed; two substitutions in total, so no 1-mismatch end-to-end hit. + edited = Edited(genome_dict['chr1'], 2000, [('=', 10), ('X',), ('=', 19), ('X',), ('=', 19)]) + seeds = ['-L', 20, '-i', 'C,20,0'] + assert mapped(bt2, genome_index, tmp_path, edited.seq, seeds + ['-N', 0]).is_unmapped + rec = mapped(bt2, genome_index, tmp_path, edited.seq, seeds + ['-N', 1]) + assert (rec.rname, rec.pos, rec.cigar) == ('chr1', edited.pos, '50M') + assert rec.tags['AS'] == -12 + + +def test_seed_mismatches_N_range(bt2, genome_index, genome_dict, tmp_path): + """-N accepts only 0 or 1.""" + fq = write_reads(tmp_path, [('r', genome_dict['chr1'][:50])]) + proc = bt2.run('bowtie2', ['-x', genome_index, '-U', fq, '-N', 2], check=False) + assert proc.returncode != 0 + assert '-N argument must be within the interval [0,1]' in proc.stderr + + +@pytest.mark.parametrize('length,aligns', [(10, True), (14, True), (15, False), (22, False)]) +def test_seed_length_L(bt2, genome_index, genome_dict, tmp_path, length, aligns): + """With seeds at every offset, a read aligns iff -L <= its longest exact stretch (14).""" + # 60 bp with substitutions every 15 bases: exact runs of 7, 14, 14, 14, 7. + edited = Edited(genome_dict['chr1'], 2100, [('=', 7), ('X',), ('=', 14), ('X',), ('=', 14), + ('X',), ('=', 14), ('X',), ('=', 7)]) + rec = mapped(bt2, genome_index, tmp_path, edited.seq, ['-L', length, '-i', 'C,1,0']) + assert rec.is_unmapped != aligns + if aligns: + assert (rec.pos, rec.tags['AS'], rec.tags['MD']) == (edited.pos, -24, edited.md) + + +@pytest.mark.parametrize('ival,aligns', [ + ('C,1,0', True), ('C,5,0', True), ('C,30,0', False), + ('C,0,0', True), # below 1, rounded up to 1 + ('L,0,0.1', True), ('L,0,0.5', False), # 6, 30 + ('S,0,0.6', True), ('S,0,4', False), # 4.6, 31.0 + ('G,0,1', True), ('G,0,8', False), # 4.1, 32.8 +]) +def test_seed_interval_i(bt2, genome_index, genome_dict, tmp_path, ival, aligns): + """-i sets the seed spacing: only the exact 20-mers at offsets 3-8 can seed this 60 bp read.""" + # Substitutions at 2, 28, 40 and 55: the only exact 20-base windows start + # at offsets 3..8. Spacing <= 8 puts a seed there; spacing 30 or more puts + # seeds only at 0 and 30, which both hold a substitution. + edited = Edited(genome_dict['chr1'], 2200, [('=', 2), ('X',), ('=', 25), ('X',), ('=', 11), + ('X',), ('=', 14), ('X',), ('=', 4)]) + rec = mapped(bt2, genome_index, tmp_path, edited.seq, ['-L', 20, '-i', ival]) + assert rec.is_unmapped != aligns + if aligns: + assert (rec.pos, rec.tags['AS']) == (edited.pos, -24) + + +def test_no_1mm_upfront(bt2, genome_index, genome_dict, tmp_path): + """With -N 0 and -L equal to the read length, a 1-mismatch read aligns only via the 1-mismatch upfront search.""" + edited = Edited(genome_dict['chr3'], 1500, [('=', 16), ('X',), ('=', 15)]) + rec = mapped(bt2, genome_index, tmp_path, edited.seq, ['-N', 0, '-L', 32]) + assert (rec.pos, rec.cigar, rec.tags['AS'], rec.tags['MD']) == (edited.pos, '32M', -6, edited.md) + assert mapped(bt2, genome_index, tmp_path, edited.seq, + ['-N', 0, '-L', 32, '--no-1mm-upfront']).is_unmapped + rec = mapped(bt2, genome_index, tmp_path, edited.seq, ['-N', 1, '-L', 32, '--no-1mm-upfront']) + assert (rec.pos, rec.tags['AS']) == (edited.pos, -6) + + +# --------------------------------------------------------------------------- +# N ceiling and N penalty +# --------------------------------------------------------------------------- + +@pytest.fixture(scope='module') +def three_n_read(genome_dict): + """50 bp with Ns at offsets 4, 9 and 14 (seeds after offset 15 are clean).""" + return Edited(genome_dict['chr1'], 3000, [('=', 4), ('N',), ('=', 4), ('N',), ('=', 4), ('N',), + ('=', 35)]) + + +@pytest.mark.parametrize('func,filtered', [ + (None, False), # default L,0,0.15 -> 7.5 + ('L,0,0.07', False), ('L,0,0.05', True), # 3.5, 2.5 + ('C,3,0', False), ('C,2,0', True), + ('S,0,0.45', False), ('S,0,0.4', True), # 3.18, 2.83 + ('G,0,0.8', False), ('G,0,0.7', True), # 3.13, 2.74 +]) +def test_n_ceil(bt2, genome_index, three_n_read, tmp_path, func, filtered): + """A read with 3 Ns is filtered (YF:Z:NS) iff the --n-ceil function is below 3.""" + args = ['--n-ceil', func] if func else [] + rec = mapped(bt2, genome_index, tmp_path, three_n_read.seq, args) + if filtered: + assert rec.is_unmapped + assert rec.tags['YF'] == 'NS' + assert rec.tag_order == ['YT', 'YF'] # field order pins current output + else: + assert (rec.rname, rec.pos, rec.cigar) == ('chr1', three_n_read.pos, '50M') + assert rec.tags['MD'] == three_n_read.md + assert rec.tags['AS'] == -3 + assert 'YF' not in rec.tags + + +@pytest.mark.parametrize('np', [0, 1, 3]) +def test_np(bt2, genome_index, three_n_read, tmp_path, np): + """--np sets the penalty per read N.""" + args = ['--np', np] if np != 1 else [] + rec = mapped(bt2, genome_index, tmp_path, three_n_read.seq, args) + assert rec.tags['AS'] == -3 * np + + +@pytest.fixture(scope='module') +def ref_n_index(bt2, tmp_path_factory): + """Index of one 1000 bp sequence with NN at 500-501 and 10 Ns at 700-709.""" + rng = random.Random(5) + seq = list(h.random_dna(1000, rng)) + seq[500:502] = ['N'] * 2 + seq[700:710] = ['N'] * 10 + seq = ''.join(seq) + d = tmp_path_factory.mktemp('ref_n') + prefix = str(d / 'refn') + bt2.build_index(h.write_fasta(str(d / 'refn.fa'), [('chrN', seq)]), prefix, extra=['--seed', '0']) + return prefix, seq + + +@pytest.mark.parametrize('local', [False, True]) +def test_reference_ns_xn(bt2, ref_n_index, tmp_path, local): + """XN counts reference Ns under the alignment; each costs --np and shows as N in MD.""" + prefix, seq = ref_n_index + reads = [ + ('n2', seq[450:550].replace('N', 'A')), + ('n10', seq[650:760].replace('N', 'C')), + ('n0', seq[300:400]), + ('n0_adjacent', h.mutate(seq[600:695], [20, 50, 80])), + ] + recs = one_per_read(align(bt2, prefix, write_reads(tmp_path, reads), + ['--local'] if local else [])) + for name, read in reads: + rec = recs[name] + start = {'n2': 450, 'n10': 650, 'n0': 300, 'n0_adjacent': 600}[name] + nref = seq[start:start + len(read)].count('N') + nsub = 3 if name == 'n0_adjacent' else 0 + assert (rec.pos, rec.cigar) == (start + 1, '%dM' % len(read)) + assert rec.tags['XN'] == nref + assert rec.tags['MD'].count('N') == nref + pen = nref + 6 * nsub + assert rec.tags['AS'] == (2 * (len(read) - nref - nsub) - pen if local else -pen) + assert recs['n10'].tags['MD'] == '50' + 'N0' * 9 + 'N50' + + +# --------------------------------------------------------------------------- +# Gaps: --rdg, --rfg, --dpad, --gbar +# --------------------------------------------------------------------------- + +def test_rdg_rfg(bt2, genome_index, edit_reads, edit_fastq): + """--rdg prices deletions (read gaps, CIGAR D) and --rfg insertions (reference gaps, CIGAR I) as open + n * extend.""" + recs = one_per_read(align(bt2, genome_index, edit_fastq, ['--rdg', '4,2', '--rfg', '7,1'])) + for name in ('del3', 'ins2', 'combo'): + rname, edited, qual = edit_reads[name] + for strand in ('fw', 'rc'): + check_fields(recs['%s_%s' % (name, strand)], rname, edited, qual, strand == 'rc', + edited.e2e_score(qual, rdg=(4, 2), rfg=(7, 1))) + assert recs['del3_fw'].tags['AS'] == -(4 + 2 * 3) + assert recs['ins2_fw'].tags['AS'] == -(7 + 1 * 2) + + +@pytest.fixture(scope='module') +def long_deletion(genome_dict): + """100 bp read with a 20 bp deletion after 50 bases.""" + ops = [('=', 50), ('D', 20), ('=', 50)] + ref = genome_dict['chr2'] + return Edited(ref, place(ref, 1000, ops), ops) + + +def test_long_deletion_needs_cheap_gaps(bt2, genome_index, long_deletion, tmp_path): + """A 20 bp deletion costs 5 + 60 by default, below the minimum score of a 100 bp read; --rdg 1,1 makes it 21.""" + assert mapped(bt2, genome_index, tmp_path, long_deletion.seq, []).is_unmapped + rec = mapped(bt2, genome_index, tmp_path, long_deletion.seq, ['--rdg', '1,1']) + assert (rec.pos, rec.cigar, rec.tags['MD']) == \ + (long_deletion.pos, '50M20D50M', long_deletion.md) + assert rec.tags['AS'] == -21 + assert (rec.tags['XO'], rec.tags['XG'], rec.tags['NM']) == (1, 20, 20) + + +@pytest.mark.parametrize('dpad,aligns', [(0, False), (5, False), (20, True)]) +def test_dpad(bt2, genome_index, long_deletion, tmp_path, dpad, aligns): + """--dpad bounds the gaps dynamic programming can open: none with 0 columns, a 20 bp deletion with 20.""" + rec = mapped(bt2, genome_index, tmp_path, long_deletion.seq, ['--rdg', '1,1', '--dpad', dpad]) + assert rec.is_unmapped != aligns + if aligns: + assert (rec.pos, rec.cigar, rec.tags['AS']) == (long_deletion.pos, '50M20D50M', -21) + + +def e2e_score_from_fields(rec): + """End-to-end AS implied by a record's XM, CIGAR gaps (default penalties, + all bases Q40).""" + gaps = [n for n, op in h.cigar_ops(rec.cigar) if op in 'ID'] + return -(6 * rec.tags['XM'] + sum(5 + 3 * n for n in gaps)) + + +@pytest.mark.parametrize('end', ['start', 'end']) +def test_gbar(bt2, genome_index, genome_dict, tmp_path, end): + """A 2 bp deletion 8 bases from a read end is used by default (--gbar 4) but not with --gbar 10, which forces a costlier alignment with no gap near the ends.""" + if end == 'start': + ops = [('=', 8), ('D', 2), ('=', 92)] + else: + ops = [('=', 92), ('D', 2), ('=', 8)] + ref = genome_dict['chr3'] + edited = Edited(ref, place(ref, 600, ops), ops) + rec = mapped(bt2, genome_index, tmp_path, edited.seq, []) + assert (rec.pos, rec.cigar, rec.tags['AS'], rec.tags['MD']) == \ + (edited.pos, edited.cigar, -11, edited.md) + rec = mapped(bt2, genome_index, tmp_path, edited.seq, ['--gbar', 10]) + assert not rec.is_unmapped + assert rec.rname == 'chr3' and abs(rec.pos - edited.pos) <= 2 + assert h.cigar_query_length(rec.cigar) == 100 + assert rec.tags['AS'] < -11 + assert rec.tags['AS'] == e2e_score_from_fields(rec) + if end == 'end': + # No reported gap within 10 bases of either read end. Not checked for + # 'start': there the current version reports 9M2D91M, a gap the + # barrier forbids. + off = 0 + for n, op in h.cigar_ops(rec.cigar): + if op in 'ID': + assert 10 <= off <= 90, rec.cigar + if op in 'MIS=X': + off += n + + +# --------------------------------------------------------------------------- +# Mismatch scoring: --mp, --ignore-quals +# --------------------------------------------------------------------------- + +@pytest.fixture(scope='module') +def qual_read(genome_dict): + """60 bp, substitutions at a Q10 base (15), a Q20 base (45) and a Q40 base (52).""" + edited = Edited(genome_dict['chr3'], 500, [('=', 15), ('X',), ('=', 29), ('X',), ('=', 6), + ('X',), ('=', 7)]) + return edited, qual_with(60, {15: '+', 45: '5'}) + + +@pytest.mark.parametrize('args,mx,mn,ignore', [ + ([], 6, 2, False), + (['--ignore-quals'], 6, 2, True), + (['--mp', '4'], 4, 2, False), # MX only (not in the MANUAL); MN stays 2 + (['--mp', '8,4'], 8, 4, False), + (['--mp', '8,4', '--ignore-quals'], 8, 4, True), + (['--mp', '5,5'], 5, 5, False), +]) +def test_mismatch_penalty(bt2, genome_index, qual_read, tmp_path, args, mx, mn, ignore): + """Mismatch penalty is MN + floor((MX - MN) * min(Q, 40) / 40), or MX with --ignore-quals.""" + edited, qual = qual_read + rec = mapped(bt2, genome_index, tmp_path, edited.seq, args, qual=qual) + assert (rec.pos, rec.tags['MD'], rec.tags['XM']) == (edited.pos, edited.md, 3) + assert rec.tags['AS'] == edited.e2e_score(qual, mx=mx, mn=mn, ignore_quals=ignore) + + +def test_mismatch_penalty_spelled_out(bt2, genome_index, qual_read, tmp_path): + """Default penalties at Q10, Q20 and Q40 are 3, 4 and 6; with --mp 8,4 they are 5, 6 and 8.""" + edited, qual = qual_read + assert mapped(bt2, genome_index, tmp_path, edited.seq, [], qual=qual).tags['AS'] == -(3 + 4 + 6) + assert mapped(bt2, genome_index, tmp_path, edited.seq, ['--mp', '8,4'], + qual=qual).tags['AS'] == -(5 + 6 + 8) + + +def test_mp_max_below_min_is_an_error(bt2, genome_index, qual_read, tmp_path): + """--mp with MX < MN is rejected.""" + edited, qual = qual_read + fq = write_reads(tmp_path, [('r', edited.seq, qual)]) + proc = bt2.run('bowtie2', ['-x', genome_index, '-U', fq, '--mp', '3,5'], check=False) + assert proc.returncode != 0 + assert 'Maximum mismatch penalty (3) is less than minimum penalty (5' in proc.stderr + + +# --------------------------------------------------------------------------- +# --score-min, --ma and the YF filters +# --------------------------------------------------------------------------- + +@pytest.fixture(scope='module') +def score18_read(genome_dict): + """64 bp with three Q40 substitutions (AS -18); seeds at 10 and 20 are clean.""" + return Edited(genome_dict['chr2'], 2500, [('=', 5), ('X',), ('=', 39), ('X',), ('=', 12), + ('X',), ('=', 5)]) + + +@pytest.mark.parametrize('func,aligns', [ + ('C,-18,0', True), ('C,-17,0', False), + ('L,0,-0.28125', True), ('L,1,-0.28125', False), # -18, -17 + ('S,0,-2.25', True), ('S,0,-2.125', False), # -18, -17 + ('G,0,-5', True), ('G,0,-4', False), # -20.8, -16.6 +]) +def test_score_min_end_to_end(bt2, genome_index, score18_read, tmp_path, func, aligns): + """A read scoring -18 is reported iff the --score-min function of its length is <= -18.""" + rec = mapped(bt2, genome_index, tmp_path, score18_read.seq, ['--score-min', func]) + assert rec.is_unmapped != aligns + if aligns: + assert (rec.pos, rec.tags['AS']) == (score18_read.pos, -18) + else: + assert 'YF' not in rec.tags + + +@pytest.mark.parametrize('func,aligns', [ + (None, True), # default G,20,8 -> 56.8 + ('G,1,34', True), ('G,1,35', False), # 157.6, 162.2 + ('L,0,1.5', True), ('L,0,1.7', False), + ('C,160,0', True), ('C,161,0', False), +]) +def test_score_min_local(bt2, genome_index, clip_reads, tmp_path, func, aligns): + """In --local mode the soft-clipped read scoring 160 is reported iff --score-min is <= 160.""" + seq, core = clip_reads['clip'] + args = ['--local'] + (['--score-min', func] if func else []) + rec = mapped(bt2, genome_index, tmp_path, seq, args) + assert rec.is_unmapped != aligns + if aligns: + assert (rec.pos, rec.tags['AS']) == (core.pos, 160) + + +@pytest.mark.parametrize('args,filtered', [ + (['--local', '--ma', '0'], True), # best possible 0 < 56.8 + (['--local', '--score-min', 'C,201,0'], True), # best possible 200 + (['--local', '--score-min', 'C,200,0'], False), +]) +def test_yf_sc_filter(bt2, genome_index, genome_dict, tmp_path, args, filtered): + """A read that cannot reach --score-min even with a perfect match is filtered with YF:Z:SC.""" + rec = mapped(bt2, genome_index, tmp_path, genome_dict['chr1'][100:200], args) + if filtered: + assert rec.is_unmapped + assert rec.tags['YF'] == 'SC' + else: + assert (rec.pos, rec.cigar, rec.tags['AS']) == (101, '100M', 200) + + +def test_yf_ln_filter(bt2, genome_index, genome_dict, tmp_path): + """A read no longer than -N is filtered with YF:Z:LN; a longer one is not.""" + fq = write_reads(tmp_path, [('one', 'A'), ('two', genome_dict['chr1'][300:302])]) + recs = one_per_read(align(bt2, genome_index, fq, ['-N', 1])) + assert recs['one'].is_unmapped + assert recs['one'].tags['YF'] == 'LN' + assert 'YF' not in recs['two'].tags + + +# --------------------------------------------------------------------------- +# Strands +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize('args,fw_aligns,rc_aligns', [ + ([], True, True), + (['--nofw'], False, True), + (['--norc'], True, False), + (['--nofw', '--norc'], False, False), +]) +def test_nofw_norc_unpaired(bt2, genome_index, genome_dict, tmp_path, args, fw_aligns, rc_aligns): + """--nofw skips the forward strand and --norc the reverse strand for unpaired reads.""" + seg = genome_dict['chr3'][1000:1060] + fq = write_reads(tmp_path, [('fw', seg), ('rc', h.revcomp(seg))]) + recs = one_per_read(align(bt2, genome_index, fq, args)) + assert recs['fw'].is_unmapped != fw_aligns + assert recs['rc'].is_unmapped != rc_aligns + if fw_aligns: + assert (recs['fw'].flag, recs['fw'].pos) == (0, 1001) + if rc_aligns: + assert (recs['rc'].flag, recs['rc'].pos) == (h.REVERSE, 1001) + + +@pytest.mark.parametrize('opt,watson,crick', [ + (None, True, True), ('--nofw', False, True), ('--norc', True, False), +]) +def test_nofw_norc_paired_fragments(bt2, genome_index, genome_dict, tmp_path, opt, watson, crick): + """For pairs, --nofw/--norc apply to the fragment's strand: --nofw keeps only Crick fragments.""" + ref = genome_dict['chr1'] + frags = {'watson': ref[2000:2300], 'crick': h.revcomp(ref[3000:3300])} + m1 = [(n, f[:50]) for n, f in sorted(frags.items())] + m2 = [(n, h.revcomp(f[-50:])) for n, f in sorted(frags.items())] + f1 = write_reads(tmp_path, m1, '1.fq') + f2 = write_reads(tmp_path, m2, '2.fq') + args = ['-p', '1', '-x', genome_index, '-1', f1, '-2', f2] + ([opt] if opt else []) + _, recs, _ = bt2.align(args) + by = h.by_name(recs) + for name, expect in (('watson', watson), ('crick', crick)): + assert len(by[name]) == 2 + for rec in by[name]: + assert rec.tags['YT'] == ('CP' if expect else 'UP') + assert rec.is_unmapped != expect + if watson: + assert sorted(r.pos for r in by['watson']) == [2001, 2251] + if crick: + assert sorted(r.pos for r in by['crick']) == [3001, 3251] + + +# --------------------------------------------------------------------------- +# Repeats: -k, -a, XS, MAPQ, -l, -d, --seed, --non-deterministic +# --------------------------------------------------------------------------- + +REP_S_EXACT = {('rep1', 301, False), ('rep1', 1301, False), ('rep2', 501, True)} +REP_S_NEAR = ('rep2', 1201, False) +REP_T = {('rep1', 2001, False), ('rep1', 2601, False), ('rep2', 1701, False)} + + +@pytest.fixture(scope='module') +def rep(): + """Two sequences with repeats of 100 bp segments S and T: + S exactly at rep1:300 and rep1:1300 (forward) and rep2:500 (reverse + complement), and with substitutions at 30 and 70 at rep2:1200; T exactly + at rep1:2000, rep1:2600 and rep2:1700, all forward. U = rep1:700-800 is + unique.""" + rng = random.Random(7) + S = h.random_dna(100, rng) + T = h.random_dna(100, rng) + r1 = list(h.random_dna(3000, rng)) + r2 = list(h.random_dna(2000, rng)) + + def put(chrom, off, seg): + chrom[off:off + len(seg)] = list(seg) + + put(r1, 300, S) + put(r1, 1300, S) + put(r1, 2000, T) + put(r1, 2600, T) + put(r2, 500, h.revcomp(S)) + put(r2, 1200, h.mutate(S, [30, 70])) + put(r2, 1700, T) + seqs = [('rep1', ''.join(r1)), ('rep2', ''.join(r2))] + return {'seqs': seqs, 'S': S, 'T': T, 'U': seqs[0][1][700:800], + 'Snear': h.mutate(S, [30, 70])} + + +@pytest.fixture(scope='module') +def rep_index(bt2, rep, tmp_path_factory): + d = tmp_path_factory.mktemp('rep_index') + fa = h.write_fasta(str(d / 'rep.fa'), rep['seqs']) + prefix = str(d / 'rep') + bt2.build_index(fa, prefix, extra=['--seed', '0']) + return prefix + + +@pytest.fixture(scope='module') +def rep_fastq(rep, tmp_path_factory): + reads = [('S', rep['S']), ('T', rep['T']), ('U', rep['U']), ('Snear', rep['Snear'])] + path = tmp_path_factory.mktemp('rep_reads') / 'reads.fq' + return h.write_fastq(str(path), reads) + + +def loc(rec): + return (rec.rname, rec.pos, rec.is_reverse) + + +def test_repeat_default_mode(bt2, rep_index, rep_fastq): + """Default mode reports one best alignment; XS is the second-best score and absent for a unique read.""" + recs = one_per_read(align(bt2, rep_index, rep_fastq)) + s = recs['S'] + assert loc(s) in REP_S_EXACT + assert (s.tags['AS'], s.tags['XS']) == (0, 0) + assert loc(recs['T']) in REP_T + assert (recs['T'].tags['AS'], recs['T'].tags['XS']) == (0, 0) + near = recs['Snear'] + assert loc(near) == REP_S_NEAR + assert (near.tags['AS'], near.tags['XS']) == (0, -12) + assert near.tags['MD'] == '100' + u = recs['U'] + assert (u.rname, u.pos, u.tags['AS']) == ('rep1', 701, 0) + assert 'XS' not in u.tags + for rec in recs.values(): + assert not rec.flag & h.SECONDARY + + +def test_repeat_mapq(bt2, rep_index, rep_fastq): + """MAPQ is 42 for a unique perfect hit, 1 for a tie and 30 when the second best is 12 lower (pins the current values). + + The MANUAL gives no MAPQ formula; the values follow BowtieMapq2 in + unique.h for a 100 bp read, whose minimum score -60.6 truncates to -60 + (diff = 60): no second best and AS - min >= 0.8 * diff gives 42; AS == + XS with AS - min >= 0.67 * diff gives 1; AS 0 and XS -12 fall just below + the 0.2f * diff band (12 < 12.0000002) into the 0.1 band, where AS == + perfect gives 30.""" + recs = one_per_read(align(bt2, rep_index, rep_fastq)) + assert recs['U'].mapq == 42 + assert recs['S'].mapq == 1 + assert recs['T'].mapq == 1 + assert (recs['Snear'].tags['AS'], recs['Snear'].tags['XS']) == (0, -12) + assert recs['Snear'].mapq == 30 + + +def check_k_records(recs, expected_locs, n): + """n distinct alignments from expected_locs, AS descending, secondary flag + and MAPQ 255 on all but the first.""" + assert len(recs) == n + locs = [loc(r) for r in recs] + assert len(set(locs)) == n + assert set(locs) <= set(expected_locs) + scores = [r.tags['AS'] for r in recs] + assert scores == sorted(scores, reverse=True) + assert not recs[0].flag & h.SECONDARY + for r in recs[1:]: + assert r.flag & h.SECONDARY + assert r.mapq == 255 + + +@pytest.mark.parametrize('k', [1, 2, 3, 4, 10]) +def test_k(bt2, rep_index, rep_fastq, rep, k): + """-k N reports min(N, number of valid alignments) distinct alignments in descending AS order.""" + by = h.by_name(align(bt2, rep_index, rep_fastq, ['-k', k])) + s_all = REP_S_EXACT | {REP_S_NEAR} + check_k_records(by['S'], s_all, min(k, 4)) + if k >= 4: + near = by['S'][-1] + assert loc(near) == REP_S_NEAR + # MD gives the reference (near copy) bases at the two substitutions. + assert (near.tags['AS'], near.tags['XM'], near.tags['MD']) == (-12, 2, '30%s39%s29' % ( + rep['Snear'][30], rep['Snear'][70])) + check_k_records(by['T'], REP_T, min(k, 3)) + check_k_records(by['U'], {('rep1', 701, False)}, 1) + for r in by['S']: + # SEQ is reverse complemented for reverse-strand alignments, secondary or not. + assert r.seq == (h.revcomp(rep['S']) if r.is_reverse else rep['S']) + + +@pytest.mark.parametrize('opt', ['-a', '--all']) +def test_all(bt2, rep_index, rep_fastq, opt): + """-a/--all reports every valid alignment: S at 4 places, T at 3, U at 1.""" + by = h.by_name(align(bt2, rep_index, rep_fastq, [opt])) + check_k_records(by['S'], REP_S_EXACT | {REP_S_NEAR}, 4) + assert by['S'][-1].tags['AS'] == -12 + check_k_records(by['T'], REP_T, 3) + check_k_records(by['U'], {('rep1', 701, False)}, 1) + for r in by['S'] + by['T']: + assert 'XS' in r.tags + + +def test_all_alias(bt2, rep_index, rep_fastq): + """-a and --all give identical records.""" + same_records(align(bt2, rep_index, rep_fastq, ['-a']), + align(bt2, rep_index, rep_fastq, ['--all']), 4) + + +NO_UPFRONT = ['--no-exact-upfront', '--no-1mm-upfront'] + + +@pytest.mark.parametrize('lowseeds,t_aligns', [ + ('0', True), # 0 disables the cut + ('2', False), ('3', True), + # Relative forms: the cut is ceil(R * value / divisor), where R is the + # number of reference sequences in the index (2 here) and the divisor is + # 100 for %, 1000 for m ("mili" in the usage text). + ('100%', False), ('101%', True), ('150%', True), + ('1000m', False), ('1001m', True), + # n is "nano" in the usage text but the current version divides by + # 10^6; these two cases come out the same with + # 10^6 and 10^9: cuts of 2 and 1, then 2001 and 3. + ('1000000n', False), ('1000000001n', True), +]) +def test_lowseeds(bt2, rep_index, rep_fastq, lowseeds, t_aligns): + """-l drops seeds whose hit range exceeds the cut: T's seeds hit 3 places, U's 1. + + -l is not in the MANUAL. The usage text says the relative forms are + "relative to idx size"; that the base is the number of reference + sequences (not the index length) pins the current output (bt2_search.cpp, + lowseeds_ncut).""" + recs = one_per_read(align(bt2, rep_index, rep_fastq, ['-l', lowseeds] + NO_UPFRONT)) + assert recs['T'].is_unmapped != t_aligns + if t_aligns: + assert loc(recs['T']) in REP_T + assert (recs['U'].rname, recs['U'].pos) == ('rep1', 701) + + +def test_lowseeds_does_not_affect_upfront_search(bt2, rep_index, rep_fastq): + """With the exact upfront search on, -l 1 still finds T, whose end-to-end exact hit needs no seed.""" + recs = one_per_read(align(bt2, rep_index, rep_fastq, ['-l', '1'])) + assert loc(recs['T']) in REP_T + + +@pytest.mark.parametrize('opt', ['-d', '--deterministic-seeds']) +def test_deterministic_seeds(bt2, rep_index, rep_fastq, opt): + """-d/--deterministic-seeds with -a and both upfront searches off finds every alignment.""" + by = h.by_name(align(bt2, rep_index, rep_fastq, [opt, '-a'] + NO_UPFRONT)) + check_k_records(by['S'], REP_S_EXACT | {REP_S_NEAR}, 4) + check_k_records(by['T'], REP_T, 3) + check_k_records(by['U'], {('rep1', 701, False)}, 1) + + +def test_deterministic_seeds_alias(bt2, rep_index, rep_fastq): + """-d and --deterministic-seeds give identical records.""" + args = ['-a'] + NO_UPFRONT + same_records(align(bt2, rep_index, rep_fastq, ['-d'] + args), + align(bt2, rep_index, rep_fastq, ['--deterministic-seeds'] + args), 4) + + +@pytest.mark.parametrize('args', [['-a'], ['-a', '--no-1mm-upfront'], ['-a', '--no-exact-upfront']]) +def test_deterministic_seeds_requires_no_upfront(bt2, rep_index, rep_fastq, args): + """-d without both --no-exact-upfront and --no-1mm-upfront is rejected, as the MANUAL requires.""" + proc = bt2.run('bowtie2', ['-x', rep_index, '-U', rep_fastq, '-d'] + args, check=False) + assert proc.returncode != 0 + assert '-d must be used with --no-exact-upfront and --no-1mm-upfront' in proc.stderr + + +@pytest.fixture(scope='module') +def tied_reads(rep, tmp_path_factory): + """30 differently named copies of S, which has three equally good hits.""" + path = tmp_path_factory.mktemp('tied') / 'reads.fq' + return h.write_fastq(str(path), [('s%02d' % i, rep['S']) for i in range(30)]) + + +def test_seed(bt2, rep_index, tied_reads): + """--seed changes how ties are broken, the default is --seed 0, and a fixed seed reproduces the output.""" + runs = {} + for seed in (None, 0, 1, 2): + args = ['--seed', seed] if seed is not None else [] + runs[seed] = align(bt2, rep_index, tied_reads, args) + assert body(runs[None]) == body(runs[0]) + assert body(align(bt2, rep_index, tied_reads, ['--seed', 1])) == body(runs[1]) + picks = {} + for seed, recs in runs.items(): + assert len(recs) == 30 + for r in recs: + assert loc(r) in REP_S_EXACT + assert r.tags['AS'] == 0 + picks[seed] = [loc(r) for r in recs] + assert len(set(picks[seed])) > 1 # ties go different ways across reads + assert picks[0] != picks[1] + assert picks[1] != picks[2] + + +def test_identical_reads_identical_alignments(bt2, rep_index, rep, tmp_path): + """Reads with the same name, sequence and quality get the same alignment among ties.""" + fq = write_reads(tmp_path, [('dup', rep['S'])] * 30) + recs = align(bt2, rep_index, fq) + assert len(recs) == 30 + assert len(set(loc(r) for r in recs)) == 1 + assert loc(recs[0]) in REP_S_EXACT + + +def test_non_deterministic(bt2, rep_index, rep, tmp_path): + """--non-deterministic breaks ties differently for identical reads.""" + fq = write_reads(tmp_path, [('dup', rep['S'])] * 30) + recs = align(bt2, rep_index, fq, ['--non-deterministic']) + assert len(recs) == 30 + locs = [loc(r) for r in recs] + assert set(locs) <= REP_S_EXACT + assert len(set(locs)) > 1 + assert all(r.tags['AS'] == 0 for r in recs) + + +# --------------------------------------------------------------------------- +# Effort: -D, -R +# --------------------------------------------------------------------------- + +def total_score(records): + assert all(not r.is_unmapped for r in records) + return sum(r.tags['AS'] for r in records) + + +def test_D_more_extensions_find_better_alignments(bt2, family_index, family_fastq): + """On repetitive reads, raising -D finds better alignments (higher total AS); the size of the gain pins current output.""" + scores = [total_score(align(bt2, family_index, family_fastq, ['-D', d, '-R', 1])) + for d in (1, 5, 15)] + assert scores[0] < scores[1] < scores[2] + + +def test_D_irrelevant_for_unique_reads(bt2, lambda_index, example_reads): + """On the non-repetitive lambda reads, -D 1 and -D 40 give the same records.""" + fq = example_reads['reads_1'] + same_records(align(bt2, lambda_index, fq, ['-D', 1, '-u', 2000]), + align(bt2, lambda_index, fq, ['-D', 40, '-u', 2000]), 2000) + + +def test_R_reseeding_helps_repetitive_reads(bt2, family_index, family_fastq): + """Reads whose seeds hit hundreds of places are re-seeded: -R 6 finds better alignments than -R 1 (gain pins current output).""" + r1 = align(bt2, family_index, family_fastq, ['-R', 1]) + r6 = align(bt2, family_index, family_fastq, ['-R', 6]) + assert total_score(r1) < total_score(r6) + + +def test_R_irrelevant_for_unique_reads(bt2, lambda_index, example_reads): + """Reads without repetitive seeds are never re-seeded: -R 1 and -R 6 give the same records.""" + fq = example_reads['reads_1'] + same_records(align(bt2, lambda_index, fq, ['-R', 1, '-u', 2000]), + align(bt2, lambda_index, fq, ['-R', 6, '-u', 2000]), 2000) + + +# --------------------------------------------------------------------------- +# Optional-field layout +# --------------------------------------------------------------------------- + +def test_optional_field_order(bt2, rep_index, genome_index, rep_fastq, tmp_path): + """Default optional fields and their order for unpaired reads (pins current output).""" + recs = one_per_read(align(bt2, rep_index, rep_fastq)) + assert recs['U'].tag_order == ['AS', 'XN', 'XM', 'XO', 'XG', 'NM', 'MD', 'YT'] + assert recs['S'].tag_order == ['AS', 'XS', 'XN', 'XM', 'XO', 'XG', 'NM', 'MD', 'YT'] + fq = write_reads(tmp_path, [('random', 'ACGT' * 25), ('short', 'A')]) + recs = one_per_read(align(bt2, genome_index, fq, ['-N', 1])) + assert recs['random'].is_unmapped + assert recs['random'].tag_order == ['YT'] + assert recs['short'].tag_order == ['YT', 'YF'] diff --git a/scripts/test/test_build_inspect.py b/scripts/test/test_build_inspect.py new file mode 100644 index 00000000..ab1401bd --- /dev/null +++ b/scripts/test/test_build_inspect.py @@ -0,0 +1,938 @@ +"""Tests for bowtie2-build and bowtie2-inspect. + +Most expected values are derived without bowtie2: the references are the +fixed random `genome` (with N runs, IUPAC codes, a gap and lowercase pasted +over known stretches), reads are cut from them at known places, and the +MANUAL's descriptions give what bowtie2-inspect prints and what the +--offrate and --ftabchars options change. + +The central invariant: the options that choose HOW the suffix array is +built (--bmax, --bmaxdivn, --dcv, --packed, --noauto, --threads, --seed) +must not change the index at all, and the options that change the index's +shape (--offrate, --ftabchars, --large-index) must not change alignments. +""" + +import gzip +import os +import re + +import pytest + +import bt2_helpers as h + + +SUFFIXES = ('1', '2', '3', '4', 'rev.1', 'rev.2') +READ_LEN = 60 + +# Every IUPAC ambiguity code plus X, in both cases. The MANUAL says +# bowtie2-inspect prints the reference "with all non-A/C/G/T characters +# converted to Ns". +IUPAC = 'RYKMSWBDHVNX' + + +def index_names(ext='bt2', suffixes=SUFFIXES): + """File-name endings of an index, e.g. '.rev.1.bt2'.""" + return sorted('.%s.%s' % (s, ext) for s in suffixes) + + +def files_written(prefix): + """Endings of every file in prefix's directory that starts with + '.', e.g. ['.1.bt2', '.2.bt2', ...].""" + d, base = os.path.split(prefix) + return sorted(f[len(base):] for f in os.listdir(d) if f.startswith(base + '.')) + + +def read_bytes(path): + with open(path, 'rb') as f: + return f.read() + + +def assert_same_files(prefix_a, prefix_b, ext='bt2', suffixes=SUFFIXES): + """Each index file of prefix_a is byte-identical to prefix_b's.""" + for s in suffixes: + a = '%s.%s.%s' % (prefix_a, s, ext) + b = '%s.%s.%s' % (prefix_b, s, ext) + assert read_bytes(a) == read_bytes(b), '%s differs from %s' % (a, b) + + +def inspected(seq): + """What bowtie2-inspect prints for a reference sequence, per the + MANUAL: upper case, and every non-A/C/G/T character becomes N.""" + return ''.join(c if c in 'ACGT' else 'N' for c in seq.upper()) + + +def fasta_text(records, width): + """FASTA text with `width` bases per line and no blank lines.""" + out = [] + for name, seq in records: + out.append('>%s\n' % name) + for i in range(0, len(seq), width): + out.append(seq[i:i + width] + '\n') + return ''.join(out) + + +def parse_fasta(text): + """Parse FASTA text into (name line, sequence) pairs.""" + records = [] + for line in text.splitlines(): + if line.startswith('>'): + records.append([line[1:], []]) + elif line.strip(): + records[-1][1].append(line.strip()) + return [(name, ''.join(parts)) for name, parts in records] + + +def assert_same_text(got, expected): + """got == expected. A mismatch is reported as the first differing line: + pytest's own diff of two multi-line strings that differ on every line + takes minutes for the ~10,000-line outputs compared here.""" + if got == expected: + return + g, e = got.splitlines(), expected.splitlines() + i = next((i for i, (a, b) in enumerate(zip(g, e)) if a != b), min(len(g), len(e))) + pytest.fail('texts differ first at line %d: got %r, expected %r (%d vs %d lines)' % ( + i + 1, g[i] if i < len(g) else None, e[i] if i < len(e) else None, len(g), len(e))) + + +def summary_fields(text): + """bowtie2-inspect -s output as a list of tab-separated field lists.""" + return [line.split('\t') for line in text.splitlines() if line] + + +def summary_value(text, key): + values = [f[1:] for f in summary_fields(text) if f[0] == key] + assert len(values) == 1, (key, text) + return values[0] + + +def sequence_lines(text): + """The 'Sequence-' rows of bowtie2-inspect -s, as (i, name, len).""" + rows = [] + for f in summary_fields(text): + if f[0].startswith('Sequence-'): + assert len(f) == 3, f + rows.append((int(f[0][len('Sequence-'):]), f[1], int(f[2]))) + return rows + + +def logged_binary(stderr): + """Basename of the binary a wrapper run with --verbose says it ran.""" + m = re.search(r'Command: (\S+)', stderr) + assert m, stderr + return os.path.basename(m.group(1)) + + +def md_tag(ref, mismatches): + """SAM MD:Z for an ungapped alignment against `ref` with substitutions + at the given 0-based positions.""" + out, run = [], 0 + for i, base in enumerate(ref): + if i in mismatches: + out.append('%d%s' % (run, base)) + run = 0 + else: + run += 1 + out.append(str(run)) + return ''.join(out) + + +def version(): + """The version the binaries were built with: BOWTIE2_VERSION next to + them, else in the repository.""" + for d in (h.BIN_DIR, h.ROOT): + path = os.path.join(d, 'BOWTIE2_VERSION') + if os.path.exists(path): + with open(path) as f: + return f.read().strip() + pytest.skip('BOWTIE2_VERSION not found') + + +def align_lines(bt2, index, args): + """Header (minus @PG, which holds the command line) and alignment + lines of a single-threaded bowtie2 run.""" + header, records, _ = bt2.align(['-p', '1', '-x', index] + list(args)) + return ([l for l in header if not l.startswith('@PG')], + [r.line for r in records]) + + +def arg_id(value): + """Test id for a parametrized list of command-line arguments.""" + if isinstance(value, list): + return ' '.join(value) or 'default' + return None + + +def splice(seq, start, text): + return seq[:start] + text + seq[start + len(text):] + + +# --------------------------------------------------------------------------- +# A reference with N runs, IUPAC codes, a gap and lowercase pasted over known +# stretches of `genome`. Pasting keeps every coordinate, so reads cut from the +# untouched stretches have the same positions as in `genome`. + +@pytest.fixture(scope='module') +def mixed_ref(genome_dict): + chr1 = splice(genome_dict['chr1'], 4000, 'RYKMSWBDHV') # IUPAC run + chr1 = splice(chr1, 4990, 'N' * 10) # trailing N run + chr2 = splice(genome_dict['chr2'], 1000, 'N' * 40) # internal N run + chr2 = splice(chr2, 2500, '--') # gap characters + chr3 = genome_dict['chr3'] + chr3 = 'N' * 10 + chr3[10:500] + chr3[500:800].lower() + chr3[800:] + return [('chr1', chr1), ('chr2', chr2), ('chr3', chr3)] + + +@pytest.fixture(scope='module') +def mixed_n(mixed_ref): + """Number of unambiguous (A/C/G/T) bases in mixed_ref.""" + return sum(1 for _, s in mixed_ref for c in s.upper() if c in 'ACGT') + + +@pytest.fixture(scope='module') +def mixed_fasta(mixed_ref, tmp_path_factory): + path = tmp_path_factory.mktemp('mixed') / 'mixed.fa' + return h.write_fasta(str(path), mixed_ref) + + +@pytest.fixture(scope='module') +def mixed_index(bt2, mixed_fasta, tmp_path_factory): + """mixed_ref built with default settings.""" + prefix = str(tmp_path_factory.mktemp('mixed_index') / 'mixed') + bt2.build_index(mixed_fasta, prefix) + return prefix + + +@pytest.fixture(scope='module') +def mixed_index_large(bt2, mixed_fasta, tmp_path_factory): + """mixed_ref built with --large-index.""" + prefix = str(tmp_path_factory.mktemp('mixed_index_large') / 'mixed') + bt2.build_index(mixed_fasta, prefix, extra=['--large-index']) + return prefix + + +# (name, chrom, 0-based start, strand, mismatch offsets along the reference) +READ_SPECS = [ + ('fw_chr1_100', 'chr1', 100, '+', ()), + ('rc_chr1_2500', 'chr1', 2500, '-', (30,)), + ('fw_chr1_4100_after_iupac', 'chr1', 4100, '+', (5, 50)), + ('fw_chr2_200', 'chr2', 200, '+', ()), + ('rc_chr2_1100_after_nrun', 'chr2', 1100, '-', ()), + ('fw_chr2_2900', 'chr2', 2900, '+', (20,)), + ('fw_chr3_20_after_leading_n', 'chr3', 20, '+', ()), + ('fw_chr3_550_lowercase', 'chr3', 550, '+', ()), + ('rc_chr3_1900', 'chr3', 1900, '-', (10,)), +] + + +@pytest.fixture(scope='module') +def mixed_reads(genome_dict, tmp_path_factory): + """FASTQ of READ_SPECS and the expected alignment of each read.""" + reads, expect = [], {} + for name, chrom, start, strand, mm in READ_SPECS: + ref = genome_dict[chrom][start:start + READ_LEN] + frag = h.mutate(ref, mm) + reads.append((name, frag if strand == '+' else h.revcomp(frag))) + expect[name] = dict(rname=chrom, pos=start + 1, + flag=0 if strand == '+' else h.REVERSE, nm=len(mm), + md=md_tag(ref, set(mm))) + path = tmp_path_factory.mktemp('mixed_reads') / 'reads.fq' + return h.write_fastq(str(path), reads), expect + + +@pytest.fixture(scope='module') +def mixed_pair(genome_dict, tmp_path_factory): + """One read pair from chr1: mate 1 forward at 1000, mate 2 the reverse + complement of 1240..1300, so the fragment is 300 bp.""" + chr1 = genome_dict['chr1'] + d = tmp_path_factory.mktemp('mixed_pair') + m1 = h.write_fastq(str(d / 'm1.fq'), [('pair', chr1[1000:1050])]) + m2 = h.write_fastq(str(d / 'm2.fq'), [('pair', h.revcomp(chr1[1240:1300]))]) + return m1, m2 + + +@pytest.fixture(scope='module') +def mixed_baseline(bt2, mixed_index, mixed_reads, mixed_pair): + """Inspect output and alignments of the default-built mixed index.""" + fq, _ = mixed_reads + m1, m2 = mixed_pair + return { + 'fasta': bt2.run('bowtie2-inspect', [mixed_index]).stdout, + 'names': bt2.run('bowtie2-inspect', ['-n', mixed_index]).stdout, + 'summary': bt2.run('bowtie2-inspect', ['-s', mixed_index]).stdout, + 'unpaired': align_lines(bt2, mixed_index, ['-U', fq]), + 'paired': align_lines(bt2, mixed_index, ['-1', m1, '-2', m2]), + } + + +def assert_matches_baseline(bt2, prefix, baseline, reads, pair): + """prefix inspects and aligns exactly like the default mixed index.""" + assert_same_text(bt2.run('bowtie2-inspect', [prefix]).stdout, baseline['fasta']) + assert bt2.run('bowtie2-inspect', ['-n', prefix]).stdout == baseline['names'] + assert bt2.run('bowtie2-inspect', ['-s', prefix]).stdout == baseline['summary'] + assert align_lines(bt2, prefix, ['-U', reads[0]]) == baseline['unpaired'] + assert align_lines(bt2, prefix, ['-1', pair[0], '-2', pair[1]]) == baseline['paired'] + + +# --------------------------------------------------------------------------- +# Files written + +def test_build_writes_six_small_index_files(mixed_index): + """A default build writes exactly .1-.4.bt2, .rev.1.bt2 and .rev.2.bt2.""" + assert files_written(mixed_index) == index_names('bt2') + + +def test_build_large_index_writes_bt2l_files(mixed_index_large): + """--large-index writes the same six files with the .bt2l extension.""" + assert files_written(mixed_index_large) == index_names('bt2l') + + +def test_large_index_inspects_and_aligns_like_small(bt2, mixed_index_large, + mixed_baseline, mixed_reads, mixed_pair): + """A .bt2l index of a reference inspects and aligns exactly like its .bt2.""" + assert_matches_baseline(bt2, mixed_index_large, mixed_baseline, mixed_reads, + mixed_pair) + + +@pytest.mark.parametrize('flag', ['-r', '--noref']) +def test_build_noref_skips_3_and_4(bt2, mixed_fasta, mixed_index, mixed_baseline, + tmp_path, flag): + """-r/--noref writes only .1/.2/.rev.1/.rev.2, identical to a full build's, still readable by -n and -s.""" + prefix = str(tmp_path / 'noref') + bt2.build_index(mixed_fasta, prefix, extra=[flag]) + kept = ('1', '2', 'rev.1', 'rev.2') + assert files_written(prefix) == index_names('bt2', kept) + assert_same_files(prefix, mixed_index, suffixes=kept) + assert bt2.run('bowtie2-inspect', ['-n', prefix]).stdout == mixed_baseline['names'] + assert bt2.run('bowtie2-inspect', ['-s', prefix]).stdout == mixed_baseline['summary'] + + +@pytest.mark.parametrize('flag', ['-3', '--justref']) +def test_build_justref_writes_only_3_and_4(bt2, mixed_fasta, mixed_index, + tmp_path, flag): + """-3/--justref writes only .3 and .4, identical to a full build's.""" + prefix = str(tmp_path / 'justref') + bt2.build_index(mixed_fasta, prefix, extra=[flag]) + assert files_written(prefix) == index_names('bt2', ('3', '4')) + assert_same_files(prefix, mixed_index, suffixes=('3', '4')) + + +def test_noref_plus_justref_is_the_full_index(bt2, mixed_fasta, mixed_index, + mixed_baseline, mixed_reads, mixed_pair, tmp_path): + """--noref then --justref into one basename reassembles the default index.""" + prefix = str(tmp_path / 'split') + bt2.build_index(mixed_fasta, prefix, extra=['--noref']) + bt2.build_index(mixed_fasta, prefix, extra=['--justref']) + assert files_written(prefix) == index_names('bt2') + assert_same_files(prefix, mixed_index) + assert_matches_baseline(bt2, prefix, mixed_baseline, mixed_reads, mixed_pair) + + +# --------------------------------------------------------------------------- +# Alignment against the mixed reference: expected values from READ_SPECS + +def test_mixed_index_aligns_reads_to_known_positions(bt2, mixed_index, mixed_reads): + """Reads beside N runs, IUPAC codes and lowercase align at their source with the planted mismatches.""" + # AS is -6 per mismatch: MX = 6 at Phred 40 ('I'). + fq, expect = mixed_reads + _, records, _ = bt2.align(['-p', '1', '-x', mixed_index, '-U', fq]) + assert sorted(r.qname for r in records) == sorted(expect) + for r in records: + e = expect[r.qname] + assert (r.rname, r.pos, r.flag, r.cigar) == (e['rname'], e['pos'], e['flag'], + '%dM' % READ_LEN), r + assert r.tags['NM'] == e['nm'] and r.tags['XM'] == e['nm'], r + assert r.tags['MD'] == e['md'], r + assert r.tags['AS'] == -6 * e['nm'], r + + +def test_mixed_index_aligns_pair(bt2, mixed_index, mixed_pair): + """The planted pair aligns concordantly at 1001/1241 with TLEN +-300.""" + _, records, _ = bt2.align(['-p', '1', '-x', mixed_index, '-1', mixed_pair[0], + '-2', mixed_pair[1]]) + assert len(records) == 2 + m1, m2 = sorted(records, key=lambda r: r.flag & h.SECOND_IN_PAIR) + assert (m1.rname, m1.pos, m1.tlen) == ('chr1', 1001, 300) + assert (m2.rname, m2.pos, m2.tlen) == ('chr1', 1241, -300) + assert m1.flag == h.PAIRED | h.PROPER_PAIR | h.MATE_REVERSE | h.FIRST_IN_PAIR + assert m2.flag == h.PAIRED | h.PROPER_PAIR | h.REVERSE | h.SECOND_IN_PAIR + + +# --------------------------------------------------------------------------- +# Index content invariance: SA construction settings change nothing + +STRATEGY_VARIANTS = [ + ['-f'], + ['-a'], + ['--noauto'], + ['-p'], + ['--packed'], + ['-a', '-p'], + ['-a', '--bmax', '64'], + ['--bmax', '500'], + ['-a', '--bmaxdivn', '16'], + ['--bmaxdivn', '1'], + ['-a', '--dcv', '16'], + ['-a', '--dcv', '4096'], + ['--dcv', '64'], + ['--threads', '4'], + ['--threads', '3', '-a', '--bmaxdivn', '9', '--dcv', '32'], + ['--seed', '7'], + ['--seed', '123456'], + ['-a', '--packed', '--bmax', '100', '--dcv', '128', '--threads', '2', + '--seed', '99'], +] + + +@pytest.mark.parametrize('extra', STRATEGY_VARIANTS, ids=arg_id) +def test_build_strategy_does_not_change_index(bt2, mixed_fasta, mixed_index, + mixed_baseline, mixed_reads, mixed_pair, tmp_path, extra): + """SA-construction settings, threads and seed give a byte-identical index that inspects and aligns the same.""" + prefix = str(tmp_path / 'variant') + bt2.build_index(mixed_fasta, prefix, extra=extra) + assert files_written(prefix) == index_names('bt2') + assert_same_files(prefix, mixed_index) + assert_matches_baseline(bt2, prefix, mixed_baseline, mixed_reads, mixed_pair) + + +@pytest.mark.parametrize('extra', [ + ['--large-index', '-a', '-p'], + ['--large-index', '--bmax', '200', '--dcv', '32'], + ['--large-index', '--threads', '3', '--seed', '5'], +], ids=arg_id) +def test_large_build_strategy_does_not_change_index(bt2, mixed_fasta, + mixed_index_large, tmp_path, extra): + """The same invariance for large indexes: byte-identical .bt2l files.""" + prefix = str(tmp_path / 'variant') + bt2.build_index(mixed_fasta, prefix, extra=extra) + assert files_written(prefix) == index_names('bt2l') + assert_same_files(prefix, mixed_index_large, ext='bt2l') + + +@pytest.mark.parametrize('extra,message', [ + ([], 'bmax according to bmaxDivN setting: {n4}'), + (['--threads', '4'], 'bmax according to bmaxDivN setting: {n16}'), + (['--threads', '4', '--bmaxdivn', '5'], 'bmax according to bmaxDivN setting: {n5}'), + (['--bmaxdivn', '8', '--bmax', '300'], 'bmax according to bmax setting: 300'), + (['--bmax', '300', '--bmaxdivn', '8'], 'bmax according to bmaxDivN setting: {n8}'), + (['-a', '--dcv', '64'], '--dcv 64'), + (['-a', '-p'], 'Strings: packed'), + (['-a'], 'Strings: unpacked'), +], ids=arg_id) +def test_build_reports_block_settings(bt2, mixed_fasta, mixed_n, tmp_path, extra, + message): + """The build log shows the block size from the MANUAL's --bmax/--bmaxdivn rules (pins the log wording).""" + # Default n / (4 * threads); an explicit --bmaxdivn is not multiplied by + # threads; of --bmax and --bmaxdivn the later one wins. n counts A/C/G/T only. + n = mixed_n + message = message.format(n4=n // 4, n16=n // 16, n5=n // 5, n8=n // 8) + prefix = str(tmp_path / 'idx') + proc = bt2.run('bowtie2-build', extra + [mixed_fasta, prefix]) + assert message in proc.stdout + + +# --------------------------------------------------------------------------- +# Index shape: --offrate and --ftabchars + +@pytest.mark.parametrize('extra,offrate', [ + (['-o', '0'], 0), (['--offrate', '3'], 3), (['-o', '5'], 5), (['-o', '10'], 10), +], ids=arg_id) +@pytest.mark.parametrize('large', [False, True], ids=['small', 'large']) +def test_offrate_sets_sa_sample_density(bt2, mixed_fasta, mixed_n, tmp_path, + extra, offrate, large): + """-o/--offrate k stores one SA offset per 2^k BWT rows and inspect -s reports 'SA-Sample 1 in 2^k'.""" + # n A/C/G/T bases give n+1 BWT rows; each .2 file is a 4-byte header plus + # ceil((n+1)/2^k) offsets of 4 bytes (8 in a large index). + prefix = str(tmp_path / 'idx') + ext, width = ('bt2l', 8) if large else ('bt2', 4) + bt2.build_index(mixed_fasta, prefix, extra=extra + (['--large-index'] if large else [])) + rows = mixed_n + 1 + samples = (rows + (1 << offrate) - 1) >> offrate + for s in ('2', 'rev.2'): + assert os.path.getsize('%s.%s.%s' % (prefix, s, ext)) == 4 + width * samples + summary = bt2.run('bowtie2-inspect', ['-s', prefix]).stdout + assert summary_value(summary, 'SA-Sample') == ['1 in %d' % (1 << offrate)] + + +@pytest.mark.parametrize('extra,chars', [ + (['-t', '1'], 1), (['--ftabchars', '5'], 5), (['-t', '8'], 8), +], ids=arg_id) +def test_ftabchars_sets_lookup_table_size(bt2, mixed_fasta, mixed_index, tmp_path, + extra, chars): + """-t/--ftabchars k: inspect -s reports k and .1 shrinks by the MANUAL's ftab size, 4^11 - 4^(k+1) bytes.""" + # The default is k = 10; allow a few bytes of other k-dependent bookkeeping. + prefix = str(tmp_path / 'idx') + bt2.build_index(mixed_fasta, prefix, extra=extra) + summary = bt2.run('bowtie2-inspect', ['-s', prefix]).stdout + assert summary_value(summary, 'FTab-Chars') == [str(chars)] + for s in ('1', 'rev.1'): + shrink = (os.path.getsize('%s.%s.bt2' % (mixed_index, s)) + - os.path.getsize('%s.%s.bt2' % (prefix, s))) + ftab = 4 ** 11 - 4 ** (chars + 1) + assert ftab <= shrink <= ftab + 128, (s, shrink, ftab) + # The reference itself (.3/.4) does not depend on the ftab. + assert_same_files(prefix, mixed_index, suffixes=('3', '4')) + + +def test_default_ftabchars_is_10(bt2, mixed_index): + """Without -t, inspect -s reports the MANUAL's default of 10 ftab chars.""" + summary = bt2.run('bowtie2-inspect', ['-s', mixed_index]).stdout + assert summary_value(summary, 'FTab-Chars') == ['10'] + + +@pytest.fixture(scope='module') +def lambda_reads(example_reads, tmp_path_factory): + """The first 200 pairs of the example paired reads and the first 100 + example long reads.""" + d = tmp_path_factory.mktemp('lambda_reads') + out = {} + for key, n in (('reads_1', 800), ('reads_2', 800), ('longreads', 400)): + with open(example_reads[key]) as f: + lines = [next(f) for _ in range(n)] + path = str(d / (key + '.fq')) + with open(path, 'w') as f: + f.writelines(lines) + out[key] = path + return out + + +@pytest.fixture(scope='module') +def lambda_shipped_alignments(bt2, lambda_index, lambda_reads): + return lambda_alignments(bt2, lambda_index, lambda_reads) + + +def lambda_alignments(bt2, index, reads): + return (align_lines(bt2, index, ['-1', reads['reads_1'], '-2', reads['reads_2']]), + align_lines(bt2, index, ['-U', reads['longreads']])) + + +@pytest.mark.parametrize('extra', [ + [], + ['-o', '0'], + ['--offrate', '9'], + ['-t', '4'], + ['--ftabchars', '7'], + ['-o', '2', '-t', '6'], + ['--large-index'], + ['--large-index', '-o', '1', '-t', '5'], +], ids=arg_id) +def test_index_shape_does_not_change_alignments(bt2, lambda_fasta, lambda_index, + lambda_reads, lambda_shipped_alignments, tmp_path, extra): + """Any --offrate/--ftabchars, small or large, aligns the example reads exactly like the shipped -o 7 index.""" + prefix = str(tmp_path / 'lambda') + bt2.build_index(lambda_fasta, prefix, extra=extra) + assert lambda_alignments(bt2, prefix, lambda_reads) == lambda_shipped_alignments + assert_same_text(bt2.run('bowtie2-inspect', [prefix]).stdout, + bt2.run('bowtie2-inspect', [lambda_index]).stdout) + + +def test_rebuilding_lambda_reproduces_shipped_index(bt2, lambda_fasta, lambda_index, + tmp_path): + """bowtie2-build -o 7 on the lambda FASTA reproduces the six shipped example/index files byte for byte.""" + prefix = str(tmp_path / 'lambda_virus') + bt2.build_index(lambda_fasta, prefix, extra=['-o', '7']) + assert_same_files(prefix, lambda_index) + + +# --------------------------------------------------------------------------- +# Reference input: formats, file lists, -c + +def write_text(path, text, compress=False): + data = text.encode() + if compress: + with gzip.open(path, 'wb') as f: + f.write(data) + else: + with open(path, 'wb') as f: + f.write(data) + return path + + +@pytest.mark.parametrize('layout', ['width13', 'one_line', 'crlf', 'gzip']) +def test_fasta_layout_does_not_change_index(bt2, mixed_ref, mixed_index, tmp_path, + layout): + """Line width, CRLF line ends and gzip compression of the reference FASTA give a byte-identical index.""" + if layout == 'width13': + path = write_text(str(tmp_path / 'ref.fa'), fasta_text(mixed_ref, 13)) + elif layout == 'one_line': + path = write_text(str(tmp_path / 'ref.fa'), fasta_text(mixed_ref, 10 ** 6)) + elif layout == 'crlf': + path = write_text(str(tmp_path / 'ref.fa'), + fasta_text(mixed_ref, 60).replace('\n', '\r\n')) + else: + path = write_text(str(tmp_path / 'ref.fa.gz'), fasta_text(mixed_ref, 60), + compress=True) + prefix = str(tmp_path / 'idx') + bt2.build_index(path, prefix) + assert_same_files(prefix, mixed_index) + + +def split_reference(mixed_ref, tmp_path, compress_last=False): + """Write each mixed_ref record to its own file; returns the paths.""" + paths = [] + for i, rec in enumerate(mixed_ref): + gz = compress_last and i == len(mixed_ref) - 1 + path = str(tmp_path / ('part%d.fa%s' % (i, '.gz' if gz else ''))) + paths.append(write_text(path, fasta_text([rec], 60), compress=gz)) + return paths + + +@pytest.mark.parametrize('compress_last', [False, True], ids=['plain', 'with_gz']) +def test_comma_separated_fasta_list(bt2, mixed_ref, mixed_index, tmp_path, + compress_last): + """A comma-separated FASTA list (plain or gzipped) indexes like one multi-FASTA file in that order.""" + paths = split_reference(mixed_ref, tmp_path, compress_last) + prefix = str(tmp_path / 'idx') + bt2.build_index(','.join(paths), prefix) + assert_same_files(prefix, mixed_index) + + +def test_space_separated_fasta_list(bt2, mixed_ref, mixed_index, tmp_path): + """FASTA files as separate arguments (undocumented wrapper alias) index like the comma list.""" + paths = split_reference(mixed_ref, tmp_path) + prefix = str(tmp_path / 'idx') + bt2.run('bowtie2-build', ['--quiet'] + paths + [prefix]) + assert_same_files(prefix, mixed_index) + + +def test_cmdline_sequences(bt2, genome_dict, tmp_path): + """-c indexes command-line sequences like a FASTA file of them named 0, 1, 2 (naming pins current output).""" + seqs = [genome_dict['chr1'][100:400], genome_dict['chr2'][50:250].lower(), + genome_dict['chr3'][1000:1180]] + prefix = str(tmp_path / 'cmd') + bt2.run('bowtie2-build', ['-q', '-c', ','.join(seqs), prefix]) + fasta = h.write_fasta(str(tmp_path / 'cmd.fa'), + [(str(i), s) for i, s in enumerate(seqs)]) + bt2.build_index(fasta, str(tmp_path / 'fa')) + assert_same_files(prefix, str(tmp_path / 'fa')) + out = bt2.run('bowtie2-inspect', [prefix]).stdout + assert parse_fasta(out) == [(str(i), s.upper()) for i, s in enumerate(seqs)] + # A read from the second sequence aligns to record '1' at its offset. + read = genome_dict['chr2'][100:160] + fq = h.write_fastq(str(tmp_path / 'r.fq'), [('r', read)]) + _, records, _ = bt2.align(['-p', '1', '-x', prefix, '-U', fq]) + assert [(r.rname, r.pos, r.flag) for r in records] == [('1', 51, 0)] + + +# --------------------------------------------------------------------------- +# Odd reference records + +@pytest.fixture(scope='module') +def odd_records(genome_dict): + """Records covering duplicate names, a name with a description, an + all-N sequence, every IUPAC code in both cases, a gap, and an empty + record (which bowtie2-build skips).""" + g = genome_dict['chr1'] + return [ + ('first', g[0:80]), + ('empty', ''), + ('dup', g[200:250]), + ('allN', 'N' * 12), + ('dup', g[300:340]), + ('with a description', g[400:470]), + ('codes', g[500:520] + IUPAC + IUPAC.lower() + '-' + g[520:550]), + ('last', g[600:633]), + ] + + +@pytest.fixture(scope='module') +def odd_index(bt2, odd_records, tmp_path_factory): + d = tmp_path_factory.mktemp('odd') + fasta = h.write_fasta(str(d / 'odd.fa'), odd_records) + proc = bt2.build_index(fasta, str(d / 'odd')) + return str(d / 'odd'), proc + + +def test_empty_record_is_skipped_with_warning(odd_index): + """An empty FASTA record is skipped with a warning (pins current behaviour; the MANUAL is silent).""" + _, proc = odd_index + assert 'Warning: Encountered empty reference sequence' in proc.stderr + + +def test_inspect_round_trips_odd_records(bt2, odd_records, odd_index): + """Default inspect prints each non-empty record under its full name, upper-cased, IUPAC/gap/N as N.""" + prefix, _ = odd_index + expected = [(name, inspected(seq)) for name, seq in odd_records if seq] + out = bt2.run('bowtie2-inspect', [prefix]).stdout + assert_same_text(out, fasta_text(expected, 60)) + + +def test_inspect_names_and_summary_of_odd_records(bt2, odd_records, odd_index): + """-n keeps input order and duplicate names; -s lengths count N, IUPAC and gap positions.""" + prefix, _ = odd_index + kept = [(name, seq) for name, seq in odd_records if seq] + names = bt2.run('bowtie2-inspect', ['-n', prefix]).stdout + assert names.splitlines() == [name for name, _ in kept] + summary = bt2.run('bowtie2-inspect', ['-s', prefix]).stdout + assert sequence_lines(summary) == [(i + 1, name, len(seq)) + for i, (name, seq) in enumerate(kept)] + + +# --------------------------------------------------------------------------- +# bowtie2-inspect + +def test_inspect_default_fasta(bt2, mixed_ref, mixed_index): + """Default inspect prints 60-column FASTA, upper-cased, with N runs, IUPAC codes and gaps as N.""" + out = bt2.run('bowtie2-inspect', [mixed_index]).stdout + expected = [(name, inspected(seq)) for name, seq in mixed_ref] + assert_same_text(out, fasta_text(expected, 60)) + + +@pytest.mark.parametrize('flag', ['-a', '--across']) +@pytest.mark.parametrize('width', [1, 7, 60, 61, 100000]) +def test_inspect_across(bt2, mixed_ref, mixed_index, flag, width): + """-a/--across N wraps the FASTA output every N bases.""" + out = bt2.run('bowtie2-inspect', [flag, width, mixed_index]).stdout + expected = [(name, inspected(seq)) for name, seq in mixed_ref] + assert_same_text(out, fasta_text(expected, width)) + + +@pytest.mark.parametrize('flag', ['-n', '--names']) +def test_inspect_names(bt2, genome, genome_index, flag): + """-n/--names prints the reference names, one per line, and nothing else.""" + out = bt2.run('bowtie2-inspect', [flag, genome_index]).stdout + assert out == ''.join(name + '\n' for name, _ in genome) + + +@pytest.mark.parametrize('flag', ['-s', '--summary']) +def test_inspect_summary(bt2, mixed_ref, mixed_fasta, tmp_path, flag): + """-s/--summary gives SA-Sample 1 in 2^offrate, FTab-Chars, and Sequence- rows.""" + prefix = str(tmp_path / 'idx') + bt2.build_index(mixed_fasta, prefix, extra=['-o', '3', '-t', '7']) + out = bt2.run('bowtie2-inspect', [flag, prefix]).stdout + assert summary_value(out, 'SA-Sample') == ['1 in 8'] + assert summary_value(out, 'FTab-Chars') == ['7'] + assert sequence_lines(out) == [(i + 1, name, len(seq)) + for i, (name, seq) in enumerate(mixed_ref)] + + +@pytest.mark.parametrize('flag', ['-o', '--output']) +def test_inspect_output_file(bt2, mixed_index, mixed_baseline, tmp_path, flag): + """-o/--output writes the FASTA to the file and nothing to stdout.""" + path = str(tmp_path / 'out.fa') + proc = bt2.run('bowtie2-inspect', [flag, path, mixed_index]) + assert proc.stdout == '' + with open(path) as f: + assert_same_text(f.read(), mixed_baseline['fasta']) + + +def test_inspect_output_file_with_across(bt2, mixed_ref, mixed_index, tmp_path): + """-o combined with -a writes the re-wrapped FASTA to the file.""" + path = str(tmp_path / 'out.fa') + bt2.run('bowtie2-inspect', ['-a', '25', '-o', path, mixed_index]) + with open(path) as f: + assert_same_text(f.read(), fasta_text([(n, inspected(s)) for n, s in mixed_ref], 25)) + + +def test_inspect_verbose(bt2, genome, genome_index): + """-v prints settings before the unchanged -n output (pins the first line's wording).""" + quiet = bt2.run('bowtie2-inspect', ['-n', genome_index]) + proc = bt2.run('bowtie2-inspect', ['-v', '-n', genome_index]) + assert proc.stdout.startswith('Input ebwt file: "%s"' % genome_index) + assert proc.stdout.endswith(quiet.stdout) + assert len(proc.stdout) > len(quiet.stdout) + + +def test_inspect_wrapper_verbose_logs_binary(bt2, genome, genome_index): + """Wrapper --verbose logs the inspect binary it runs and leaves the -n output in place (pins the log wording).""" + # Only the log line and the -n output are checked: the MANUAL's + # "-v/--verbose" says --verbose should also print the -v settings, which + # the current version does not do. + proc = bt2.run('bowtie2-inspect', ['--verbose', '-n', genome_index]) + assert logged_binary(proc.stderr) == 'bowtie2-inspect-s' + assert proc.stdout.endswith(''.join(name + '\n' for name, _ in genome)) + + +def test_inspect_large_index_detected(bt2, mixed_index_large, mixed_baseline): + """A .bt2l-only index is found and inspects (FASTA, -n, -s) exactly like the .bt2 index.""" + assert_same_text(bt2.run('bowtie2-inspect', [mixed_index_large]).stdout, mixed_baseline['fasta']) + assert bt2.run('bowtie2-inspect', ['-n', mixed_index_large]).stdout == mixed_baseline['names'] + assert bt2.run('bowtie2-inspect', ['-s', mixed_index_large]).stdout == mixed_baseline['summary'] + + +def test_inspect_large_index_option(bt2, mixed_fasta, tmp_path): + """With .bt2 (-o 3) and .bt2l (-o 6) under one basename, --large-index selects the .bt2l.""" + prefix = str(tmp_path / 'both') + bt2.build_index(mixed_fasta, prefix, extra=['-o', '3']) + bt2.build_index(mixed_fasta, prefix, extra=['-o', '6', '--large-index']) + small = bt2.run('bowtie2-inspect', ['--verbose', '-s', prefix]) + large = bt2.run('bowtie2-inspect', ['--verbose', '--large-index', '-s', prefix]) + assert summary_value(small.stdout, 'SA-Sample') == ['1 in 8'] + assert summary_value(large.stdout, 'SA-Sample') == ['1 in 64'] + assert logged_binary(small.stderr) == 'bowtie2-inspect-s' + assert logged_binary(large.stderr) == 'bowtie2-inspect-l' + + +@pytest.mark.parametrize('kind', ['debug', 'sanitized']) +@pytest.mark.parametrize('large', [False, True], ids=['small', 'large']) +def test_inspect_debug_sanitized(bt2, mixed_index, mixed_index_large, + mixed_baseline, kind, large): + """--debug/--sanitized run the matching inspect binary, whose output equals the release binary's.""" + binary = 'bowtie2-inspect-%s-%s' % ('l' if large else 's', kind) + if not bt2.has_binary(binary): + pytest.skip('%s not built' % binary) + prefix = mixed_index_large if large else mixed_index + extra = ['--' + kind] + (['--large-index'] if large else []) + proc = bt2.run('bowtie2-inspect', extra + ['--verbose', prefix]) + assert logged_binary(proc.stderr) == binary + assert_same_text(proc.stdout, mixed_baseline['fasta']) + for flag, key in (('-n', 'names'), ('-s', 'summary')): + assert bt2.run('bowtie2-inspect', extra + [flag, prefix]).stdout == mixed_baseline[key] + + +@pytest.mark.parametrize('flag', ['-n', '-s']) +def test_inspect_finds_index_through_bowtie2_indexes(bt2, mixed_index, + mixed_baseline, tmp_path, flag): + """A basename not in the current directory is found through $BOWTIE2_INDEXES.""" + env = dict(os.environ) + env['BOWTIE2_INDEXES'] = os.path.dirname(mixed_index) + proc = bt2.run('bowtie2-inspect', [flag, os.path.basename(mixed_index)], + cwd=str(tmp_path), env=env) + assert proc.stdout == mixed_baseline['names' if flag == '-n' else 'summary'] + + +def test_inspect_missing_index(bt2, tmp_path): + """Inspecting a basename with no index fails with bowtie2's message.""" + missing = str(tmp_path / 'nothing') + proc = bt2.run('bowtie2-inspect', [missing], check=False) + assert proc.returncode != 0 + assert 'Could not locate a Bowtie index corresponding to basename "%s"' % missing \ + in proc.stderr + + +@pytest.mark.parametrize('flag', ['-h', '--help']) +def test_inspect_help(bt2, flag): + """-h/--help prints usage listing every documented option and exits 0.""" + proc = bt2.run('bowtie2-inspect', [flag]) + assert proc.stdout.startswith('Bowtie 2 version %s' % version()) + assert 'Usage: bowtie2-inspect [options]* ' in proc.stdout + for opt in ('--large-index', '--debug', '--sanitized', '--verbose', + '-a/--across ', '-n/--names', '-s/--summary', '-o/--output', + '-v/--verbose', '-h/--help'): + assert opt in proc.stdout, opt + + +def test_inspect_version(bt2): + """--version prints the bowtie2-inspect version and exits 0.""" + proc = bt2.run('bowtie2-inspect', ['--version']) + lines = proc.stdout.splitlines() + assert re.search(r'bowtie2-inspect-s version %s$' % re.escape(version()), lines[0]) + assert '64-bit' in lines + + +# --------------------------------------------------------------------------- +# The shipped lambda index + +def test_inspect_shipped_lambda_fasta(bt2, lambda_fasta, lambda_index): + """Inspecting the shipped lambda index gives back example/reference/lambda_virus.fa.""" + with open(lambda_fasta) as f: + expected = parse_fasta(f.read()) + out = bt2.run('bowtie2-inspect', [lambda_index]).stdout + assert parse_fasta(out) == [(n, inspected(s)) for n, s in expected] + assert all(len(line) <= 60 for line in out.splitlines()[1:]) + + +def test_inspect_shipped_lambda_names_and_summary(bt2, lambda_fasta, lambda_index): + """Shipped lambda index: -n gives the name line; -s gives length 48502, SA-Sample 1 in 128, 10 ftab chars.""" + with open(lambda_fasta) as f: + [(name, seq)] = parse_fasta(f.read()) + assert len(seq) == 48502 + assert bt2.run('bowtie2-inspect', ['-n', lambda_index]).stdout == name + '\n' + summary = bt2.run('bowtie2-inspect', ['-s', lambda_index]).stdout + assert summary_value(summary, 'SA-Sample') == ['1 in 128'] + assert summary_value(summary, 'FTab-Chars') == ['10'] + assert sequence_lines(summary) == [(1, name, len(seq))] + + +# --------------------------------------------------------------------------- +# bowtie2-build wrapper options, help, version, quiet, errors + +@pytest.mark.parametrize('extra,binary', [ + ([], 'bowtie2-build-s'), + (['--large-index'], 'bowtie2-build-l'), + (['--debug'], 'bowtie2-build-s-debug'), + (['--sanitized'], 'bowtie2-build-s-sanitized'), + (['--debug', '--large-index'], 'bowtie2-build-l-debug'), + (['--sanitized', '--large-index'], 'bowtie2-build-l-sanitized'), +], ids=arg_id) +def test_build_wrapper_binary_selection(bt2, mixed_fasta, mixed_index, + mixed_index_large, tmp_path, extra, binary): + """--large-index/--debug/--sanitized pick the logged binary, and each writes the release index bytes.""" + if not bt2.has_binary(binary): + pytest.skip('%s not built' % binary) + prefix = str(tmp_path / 'idx') + proc = bt2.build_index(mixed_fasta, prefix, extra=['--verbose'] + extra) + assert logged_binary(proc.stderr) == binary + if '--large-index' in extra: + assert files_written(prefix) == index_names('bt2l') + assert_same_files(prefix, mixed_index_large, ext='bt2l') + else: + assert files_written(prefix) == index_names('bt2') + assert_same_files(prefix, mixed_index) + + +@pytest.mark.parametrize('flag', ['-q', '--quiet']) +def test_build_quiet(bt2, mixed_fasta, mixed_index, tmp_path, flag): + """-q/--quiet silences a clean build; without it the log names the input; the index is the same.""" + quiet = str(tmp_path / 'quiet') + proc = bt2.run('bowtie2-build', [flag, mixed_fasta, quiet]) + assert proc.stdout == '' and proc.stderr == '' + loud = str(tmp_path / 'loud') + proc = bt2.run('bowtie2-build', [mixed_fasta, loud]) + assert ' %s\n' % mixed_fasta in proc.stdout + assert_same_files(quiet, mixed_index) + assert_same_files(loud, mixed_index) + + +BUILD_OPTIONS_IN_HELP = ('-f', '-c', '--large-index', '--debug', '--sanitized', + '--verbose', '-a/--noauto', '-p/--packed', '--bmax ', '--bmaxdivn ', + '--dcv ', '--nodc', '-r/--noref', '-3/--justref', '-o/--offrate ', + '-t/--ftabchars ', '--threads ', '--seed ', '-q/--quiet', + '--help', '--version') + + +@pytest.mark.parametrize('flag', ['-h', '--help']) +def test_build_help(bt2, tmp_path, flag): + """-h/--help prints usage listing every documented option, exits 0 and writes nothing.""" + proc = bt2.run('bowtie2-build', [flag], cwd=str(tmp_path)) + assert proc.stdout.startswith('Bowtie 2 version %s' % version()) + assert 'Usage: bowtie2-build [options]* ' in proc.stdout + for opt in BUILD_OPTIONS_IN_HELP: + assert opt in proc.stdout, opt + assert os.listdir(str(tmp_path)) == [] + + +def test_build_version(bt2, tmp_path): + """--version prints the bowtie2-build version, exits 0, writes nothing.""" + proc = bt2.run('bowtie2-build', ['--version'], cwd=str(tmp_path)) + lines = proc.stdout.splitlines() + assert re.search(r'bowtie2-build-s version %s$' % re.escape(version()), lines[0]) + assert '64-bit' in lines + assert os.listdir(str(tmp_path)) == [] + + +@pytest.mark.parametrize('args,message', [ + (['-q'], 'No input sequence or sequence file specified!'), + (['-q', '{fa}'], 'No output file specified!'), + (['-q', '{missing}', '{prefix}'], 'Error: could not open'), + (['-q', '{empty}', '{prefix}'], 'Warning: All fasta inputs were empty'), + (['-q', '--bmax', '0', '{fa}', '{prefix}'], '--bmax arg must be at least 1'), + (['-q', '--bmaxdivn', '0', '{fa}', '{prefix}'], '--bmaxdivn arg must be at least 1'), + (['-q', '--dcv', '2', '{fa}', '{prefix}'], '--dcv arg must be at least 3'), + (['-q', '-o', '-1', '{fa}', '{prefix}'], '-o/--offRate arg must be at least 0'), + (['-q', '-t', '0', '{fa}', '{prefix}'], '-t/--ftabChars arg must be at least 1'), + (['-q', '-t', '17', '{fa}', '{prefix}'], '-t/--ftabChars arg must not exceed 16'), + (['-q', '--seed', '-1', '{fa}', '{prefix}'], '--seed arg must be at least 0'), + (['-q', '--no-such-option', '{fa}', '{prefix}'], 'Usage: bowtie2-build'), +], ids=arg_id) +def test_build_errors(bt2, mixed_fasta, tmp_path, args, message): + """Bad invocations exit non-zero with bowtie2-build's message and leave no index files.""" + work = tmp_path / 'work' + work.mkdir() + empty = str(tmp_path / 'empty.fa') + open(empty, 'w').close() + subs = dict(fa=mixed_fasta, missing=str(tmp_path / 'missing.fa'), empty=empty, + prefix=str(work / 'idx')) + proc = bt2.run('bowtie2-build', [a.format(**subs) for a in args], check=False) + assert proc.returncode != 0 + assert message in proc.stderr + assert os.listdir(str(work)) == [] diff --git a/scripts/test/test_input.py b/scripts/test/test_input.py new file mode 100644 index 00000000..b163bab5 --- /dev/null +++ b/scripts/test/test_input.py @@ -0,0 +1,995 @@ +"""Input formats and read preprocessing. + +Covers the read input formats (-q, --tab5, --tab6, --12, --qseq, -f, -r, -F, +-c, --interleaved, -b with --align-paired-reads and --preserve-tags), which +reads are aligned and how they are trimmed (-s, -u, -5, -3, --trim-to), +quality encodings (--phred33, --phred64, --solexa-quals and their aliases), +comma-separated and repeated read files, compressed files and standard input, +the QNAME options --sam-no-qname-trunc and --sam-append-comment, +--qc-filter, and reads that are shorter than a seed, empty or all N. + +Reads are cut from the fixed random `genome` fixture at known positions and +strands, so FLAG, POS, CIGAR, SEQ, QUAL, AS and MD of every record are derived +here rather than taken from bowtie2. Behaviour of the current version that +contradicts the MANUAL is not tested here. +""" + +import bz2 +import gzip +import math +import os +import shutil +import struct +import subprocess +import zlib + +import pytest + +import bt2_helpers as h + + +# Unpaired reads: (name, chromosome, 0-based start, length, strand). +UNPAIRED = [ + ('u1', 'chr1', 120, 50, '+'), + ('u2', 'chr2', 900, 45, '-'), + ('u3', 'chr3', 1500, 60, '+'), + ('u4', 'chr1', 3300, 40, '-'), +] + +# Pairs: (name, chromosome, fragment start, fragment length, mate 1 length, +# mate 2 length). Mate 1 is the forward start of the fragment and mate 2 the +# reverse complement of its end, the default --fr orientation. +PAIRS = [ + ('p1', 'chr1', 1000, 250, 50, 50), + ('p2', 'chr2', 1500, 300, 40, 45), +] + +# Fields 2-8 of the QSEQ records written here, and the QNAME suffix bowtie2 +# builds from them: fields 1-7 joined with '_', then '/' and field 8. The +# MANUAL does not describe QSEQ read names, so this pins the current output +# (read_qseq.cpp). +QSEQ_FIELDS = ['7', '1', '11', '101', '202', '0', '1'] +QSEQ_SUFFIX = '_7_1_11_101_202_0/1' + + +# ---------------------------------------------------------------- helpers + +def run_align(bt2, index, args, input=None, check=True): + """Align with one thread; returns (records, CompletedProcess).""" + _, records, proc = bt2.align(['-x', index, '-p', '1'] + list(args), + input=input, check=check) + return records, proc + + +def run_bytes(bt2, args, data): + """Run the bowtie2 wrapper with binary stdin; returns (records, proc).""" + proc = subprocess.run([bt2.bowtie2] + [str(a) for a in args], input=data, + capture_output=True, timeout=300) + assert proc.returncode == 0, proc.stderr.decode() + return h.parse_sam(proc.stdout.decode())[1], proc + + +def sam_body(proc): + """SAM records of a run without its header lines.""" + return [line for line in proc.stdout.splitlines() if not line.startswith('@')] + + +def read_seq(gd, spec): + name, chrom, start, length, strand = spec + seq = gd[chrom][start:start + length] + return seq if strand == '+' else h.revcomp(seq) + + +def varied_qual(length, shift=0): + """Phred+33 qualities between 25 and 40 that differ between neighbours, + so a reversed or shifted QUAL string is detectable.""" + return ''.join(chr(33 + 25 + (i * 7 + shift) % 16) for i in range(length)) + + +def unpaired_reads(gd): + """(name, seq, qual) of the UNPAIRED reads.""" + return [(s[0], read_seq(gd, s), varied_qual(s[3], k)) + for k, s in enumerate(UNPAIRED)] + + +def pair_seqs(gd, pair): + name, chrom, start, frag, l1, l2 = pair + g = gd[chrom] + return g[start:start + l1], h.revcomp(g[start + frag - l2:start + frag]) + + +def assert_placed(rec, gd, spec, qual, trim5=0, trim3=0): + """`rec` is the alignment of read `spec`, with read quality string `qual`, + after trim5 bases were removed from its 5' end and trim3 from its 3' end.""" + name, chrom, start, length, strand = spec + n = length - trim5 - trim3 + kept = qual[trim5:length - trim3] + if strand == '+': + ref_start, flag, exp_qual = start + trim5, 0, kept + else: + ref_start, flag, exp_qual = start + trim3, h.REVERSE, kept[::-1] + assert rec.flag == flag, rec + assert rec.rname == chrom, rec + assert rec.pos == ref_start + 1, rec + assert rec.cigar == '%dM' % n, rec + assert rec.seq == gd[chrom][ref_start:ref_start + n], rec + assert rec.qual == exp_qual, rec + assert rec.tags['AS'] == 0, rec + assert rec.tags['NM'] == 0, rec + assert rec.tags['MD'] == str(n), rec + + +def assert_pair(mate1, mate2, gd, pair, qual1=None, qual2=None): + """mate1/mate2 are a concordant --fr alignment of `pair`.""" + name, chrom, start, frag, l1, l2 = pair + g = gd[chrom] + pos2 = start + frag - l2 + assert mate1.flag == 99, mate1 + assert mate2.flag == 147, mate2 + assert (mate1.rname, mate2.rname) == (chrom, chrom) + assert mate1.pos == start + 1, mate1 + assert mate2.pos == pos2 + 1, mate2 + assert (mate1.cigar, mate2.cigar) == ('%dM' % l1, '%dM' % l2) + assert mate1.seq == g[start:start + l1] + assert mate2.seq == g[pos2:start + frag] + assert (mate1.pnext, mate2.pnext) == (mate2.pos, mate1.pos) + assert (mate1.tlen, mate2.tlen) == (frag, -frag) + assert mate1.tags['AS'] == 0 and mate2.tags['AS'] == 0 + if qual1 is not None: + assert mate1.qual == qual1 + if qual2 is not None: + assert mate2.qual == qual2[::-1] + + +def qseq_line(name, seq, qual, filt='1'): + return '\t'.join([name] + QSEQ_FIELDS + [seq, qual, filt]) + '\n' + + +def write_text(path, text): + with open(str(path), 'w') as f: + f.write(text) + return str(path) + + +def write_input(fmt, tmp_path, reads): + """Write (name, seq, qual) reads in format `fmt`. + + Returns (bowtie2 arguments, expected QNAMEs, whether QUAL is taken from + the input). Formats without qualities get Phred 40 ('I') everywhere.""" + names = [r[0] for r in reads] + index_names = [str(i) for i in range(len(reads))] + if fmt == 'fastq': + path = write_text(tmp_path / 'r.fq', ''.join('@%s desc %d\n%s\n+\n%s\n' + % (n, i, s, q) for i, (n, s, q) in enumerate(reads))) + return ['-q', '-U', path], names, True + if fmt == 'fasta': + path = str(tmp_path / 'r.fa') + h.write_fasta(path, [('%s desc %d' % (n, i), s) + for i, (n, s, q) in enumerate(reads)], width=25) + return ['-f', '-U', path], names, False + if fmt in ('tab5', 'tab6'): + path = write_text(tmp_path / ('r.' + fmt), + ''.join('%s\t%s\t%s\n' % r for r in reads)) + return ['--' + fmt, path], names, True + if fmt == 'qseq': + path = write_text(tmp_path / 'r_qseq.txt', + ''.join(qseq_line(n, s, q) for n, s, q in reads)) + return ['--qseq', '-U', path], [n + QSEQ_SUFFIX for n in names], True + if fmt == 'raw': + path = write_text(tmp_path / 'r.raw', ''.join(s + '\n' for n, s, q in reads)) + return ['-r', '-U', path], index_names, False + if fmt == 'cmdline': + return ['-c', '-U', ','.join(s for n, s, q in reads)], index_names, False + if fmt == 'bam': + path = str(tmp_path / 'r.bam') + write_bam(path, [(n, 0x4, s, q, []) for n, s, q in reads]) + return ['-b', path], names, True + raise ValueError(fmt) + + +FORMATS = ['fastq', 'fasta', 'tab5', 'tab6', 'qseq', 'raw', 'cmdline', 'bam'] + + +# ---------------------------------------------------- minimal BAM writer + +BAM_EOF = bytes.fromhex('1f8b08040000000000ff0600424302001b0003000000000000000000') +BAM_SEQ_CODES = '=ACMGRSVTWYHKDBN' +BAM_INT_FORMATS = {'c': 'b', 'C': 'B', 's': 'h', 'S': 'H', 'i': 'i', 'I': 'I', 'f': 'f'} + + +def bgzf_block(data): + """One BGZF block: a gzip member with the BC extra subfield.""" + comp = zlib.compressobj(6, zlib.DEFLATED, -15) + cdata = comp.compress(data) + comp.flush() + header = struct.pack('<4BI2BH2BHH', 0x1f, 0x8b, 8, 4, 0, 0, 0xff, 6, + ord('B'), ord('C'), 2, len(cdata) + 25) + return header + cdata + struct.pack('> (4 * (1 - i % 2))) & 0xf] + for i in range(l_seq)) + qoff = seq_off + (l_seq + 1) // 2 + return seq, ''.join(chr(q + 33) for q in data[qoff:qoff + l_seq]) + + +# ------------------------------------------------------- input formats + +@pytest.mark.parametrize('fmt', FORMATS) +def test_format_parses_reads(bt2, genome_dict, genome_index, tmp_path, fmt): + """Each input format yields the reads at their known positions, strands, QNAMEs and QUALs. + + The QNAMEs of -r and -c reads (the 0-based read index) and of QSEQ reads + (QSEQ_SUFFIX) pin the current output; the MANUAL only says that -r and + -c reads have no names.""" + reads = unpaired_reads(genome_dict) + args, names, has_qual = write_input(fmt, tmp_path, reads) + recs, _ = run_align(bt2, genome_index, args) + assert [r.qname for r in recs] == names + for rec, spec, (n, s, q) in zip(recs, UNPAIRED, reads): + assert_placed(rec, genome_dict, spec, q if has_qual else 'I' * len(s)) + + +def test_q_is_the_default(bt2, genome_dict, genome_index, tmp_path): + """-q (FASTQ) gives the same SAM as giving no format option.""" + args, _, _ = write_input('fastq', tmp_path, unpaired_reads(genome_dict)) + _, with_q = run_align(bt2, genome_index, args) + _, without_q = run_align(bt2, genome_index, args[1:]) + assert len(sam_body(with_q)) == len(UNPAIRED) + assert sam_body(with_q) == sam_body(without_q) + + +@pytest.mark.parametrize('fmt', ['fasta', 'raw', 'cmdline']) +def test_formats_without_qualities_score_as_ignore_quals(bt2, genome_dict, + genome_index, tmp_path, fmt): + """-f, -r and -c score a mismatch at the full MX penalty, as --ignore-quals does.""" + ref = genome_dict['chr2'][400:450] + read = h.mutate(ref, [17]) + args, _, _ = write_input(fmt, tmp_path, [('m1', read, None)]) + recs, plain = run_align(bt2, genome_index, args) + _, ignore = run_align(bt2, genome_index, args + ['--ignore-quals']) + assert sam_body(plain) == sam_body(ignore) + rec, = recs + assert (rec.pos, rec.qual) == (401, 'I' * 50) + assert rec.tags['AS'] == -6 + assert rec.tags['XM'] == 1 and rec.tags['NM'] == 1 + assert rec.tags['MD'] == '17%s32' % ref[17] + + +def test_fasta_continuous_kmers(bt2, genome_dict, genome_index, tmp_path): + """-F , aligns every k-mer at offsets 0, i, 2i... of each FASTA record, named _. + + seqB is 36 bases long, so its last k-mer (offset 16) ends at the record's + last base. Only the undocumented "-F 20,4" spelling works in the current + version; the MANUAL's "-F k:20,i:4" yields no reads and is not tested here.""" + seq_a = genome_dict['chr2'][200:230] + seq_b = genome_dict['chr3'][100:136] + path = str(tmp_path / 'long.fa') + h.write_fasta(path, [('seqA some description', seq_a), ('seqB', seq_b)], width=11) + recs, _ = run_align(bt2, genome_index, ['-F', '20,4', '-U', path]) + expected = [] + for name, chrom, start, length in (('seqA', 'chr2', 200, 30), ('seqB', 'chr3', 100, 36)): + for off in range(0, length - 20 + 1, 4): + expected.append(('%s_%d' % (name, off), chrom, start + off)) + assert [(r.qname, r.rname, r.pos - 1) for r in recs] == expected + for rec in recs: + assert rec.flag == 0 and rec.cigar == '20M' + assert rec.seq == genome_dict[rec.rname][rec.pos - 1:rec.pos + 19] + assert rec.qual == 'I' * 20 + assert rec.tags['AS'] == 0 + + +def test_cmdline_paired(bt2, genome_dict, genome_index): + """-c with -1/-2 takes mates as comma-separated sequences and names pairs by index.""" + m1s, m2s = zip(*[pair_seqs(genome_dict, p) for p in PAIRS]) + recs, _ = run_align(bt2, genome_index, ['-c', '-1', ','.join(m1s), '-2', ','.join(m2s)]) + assert [r.qname for r in recs] == ['0', '0', '1', '1'] + for k, pair in enumerate(PAIRS): + assert_pair(recs[2 * k], recs[2 * k + 1], genome_dict, pair) + + +def test_tab5_mixes_paired_and_unpaired(bt2, genome_dict, genome_index, tmp_path): + """--tab5 treats 5-field lines as pairs sharing one name and 3-field lines as unpaired reads.""" + (m1a, m2a), (m1b, m2b) = [pair_seqs(genome_dict, p) for p in PAIRS] + u1 = read_seq(genome_dict, UNPAIRED[0]) + q1, q2, q3, q4, qu = [varied_qual(len(s), k) for k, s in enumerate((m1a, m2a, m1b, m2b, u1))] + path = write_text(tmp_path / 'r.tab5', 'p1\t%s\t%s\t%s\t%s\nu1\t%s\t%s\np2\t%s\t%s\t%s\t%s\n' + % (m1a, q1, m2a, q2, u1, qu, m1b, q3, m2b, q4)) + recs, _ = run_align(bt2, genome_index, ['--tab5', path]) + assert [r.qname for r in recs] == ['p1', 'p1', 'u1', 'p2', 'p2'] + assert_pair(recs[0], recs[1], genome_dict, PAIRS[0], q1, q2) + assert_placed(recs[2], genome_dict, UNPAIRED[0], qu) + assert_pair(recs[3], recs[4], genome_dict, PAIRS[1], q3, q4) + + +def test_tab6_mate_names(bt2, genome_dict, genome_index, tmp_path): + """--tab6 gives each mate its own name; unpaired 3-field lines still work.""" + m1, m2 = pair_seqs(genome_dict, PAIRS[0]) + u1 = read_seq(genome_dict, UNPAIRED[0]) + path = write_text(tmp_path / 'r.tab6', 'left\t%s\t%s\tright\t%s\t%s\nu1\t%s\t%s\n' + % (m1, 'I' * len(m1), m2, 'I' * len(m2), u1, 'I' * len(u1))) + recs, _ = run_align(bt2, genome_index, ['--tab6', path]) + assert [r.qname for r in recs] == ['left', 'right', 'u1'] + assert_pair(recs[0], recs[1], genome_dict, PAIRS[0]) + assert_placed(recs[2], genome_dict, UNPAIRED[0], 'I' * len(u1)) + + +def test_12_is_tab5(bt2, genome_dict, genome_index, tmp_path): + """The undocumented --12 option reads the --tab5 format.""" + m1, m2 = pair_seqs(genome_dict, PAIRS[0]) + u1 = read_seq(genome_dict, UNPAIRED[0]) + path = write_text(tmp_path / 'r.tab5', 'p1\t%s\t%s\t%s\t%s\nu1\t%s\t%s\n' + % (m1, 'I' * len(m1), m2, 'I' * len(m2), u1, 'I' * len(u1))) + _, tab5 = run_align(bt2, genome_index, ['--tab5', path]) + _, onetwo = run_align(bt2, genome_index, ['--12', path]) + assert len(sam_body(tab5)) == 3 + assert sam_body(onetwo) == sam_body(tab5) + + +@pytest.mark.parametrize('fmt', ['fastq', 'fasta']) +def test_interleaved(bt2, genome_dict, genome_index, tmp_path, fmt): + """--interleaved takes consecutive records as mate 1 and mate 2; /1 and /2 are dropped from QNAME.""" + text = [] + for pair in PAIRS: + for mate, seq in zip((1, 2), pair_seqs(genome_dict, pair)): + if fmt == 'fastq': + text.append('@%s/%d\n%s\n+\n%s\n' % (pair[0], mate, seq, 'I' * len(seq))) + else: + text.append('>%s/%d\n%s\n' % (pair[0], mate, seq)) + path = write_text(tmp_path / ('r.' + fmt), ''.join(text)) + args = ['--interleaved', path] + (['-f'] if fmt == 'fasta' else []) + recs, _ = run_align(bt2, genome_index, args) + assert [r.qname for r in recs] == ['p1', 'p1', 'p2', 'p2'] + for k, pair in enumerate(PAIRS): + assert_pair(recs[2 * k], recs[2 * k + 1], genome_dict, pair) + + +def test_interleaved_rejects_other_formats(bt2, genome_dict, genome_index, tmp_path): + """--interleaved with -r is an error.""" + path = write_text(tmp_path / 'r.raw', 'ACGTACGTACGTACGTACGTAC\n') + _, proc = run_align(bt2, genome_index, ['-r', '--interleaved', path], check=False) + assert proc.returncode != 0 + assert '--interleaved only works in combination with FASTA (-f) and FASTQ (-q) formats' in proc.stderr + + +@pytest.mark.parametrize('fmt,flag,message', [ + ('fastq', '-f', 'does not look like a FASTA file'), + ('fasta', '-q', 'does not look like a FASTQ file'), +]) +def test_wrong_format_is_an_error(bt2, genome_dict, genome_index, tmp_path, fmt, flag, message): + """A FASTQ file read with -f, or a FASTA file read with -q, is reported as the wrong format.""" + args, _, _ = write_input(fmt, tmp_path, unpaired_reads(genome_dict)) + _, proc = run_align(bt2, genome_index, [flag] + args[1:], check=False) + assert proc.returncode != 0 + assert message in proc.stderr + + +# ------------------------------------------------------------------ QSEQ + +def test_qc_filter(bt2, genome_dict, genome_index, tmp_path): + """--qc-filter removes exactly the QSEQ reads of one filter value (unaligned, YF:Z:QC, SEQ/QUAL kept) and aligns the rest. + + Which value is removed is left open on purpose. The MANUAL says the reads + whose filter field is 1 ("non-zero"); the current version removes the + reads marked 0, following Illumina's convention that 1 means "passed + filter", so the suite stays green whichever of the two is corrected.""" + reads = unpaired_reads(genome_dict) + filters = ['1', '0', '0', '1'] + path = write_text(tmp_path / 'r_qseq.txt', + ''.join(qseq_line(n, s, q, f) for (n, s, q), f in zip(reads, filters))) + names = [n + QSEQ_SUFFIX for n, s, q in reads] + recs, _ = run_align(bt2, genome_index, ['--qseq', '-U', path]) + assert [r.qname for r in recs] == names + for rec, spec, (n, s, q) in zip(recs, UNPAIRED, reads): + assert_placed(rec, genome_dict, spec, q) + recs, _ = run_align(bt2, genome_index, ['--qseq', '--qc-filter', '-U', path]) + assert [r.qname for r in recs] == names + removed = [f for rec, f in zip(recs, filters) if rec.is_unmapped] + assert removed in (['1', '1'], ['0', '0']) + for rec, spec, (n, s, q), f in zip(recs, UNPAIRED, reads, filters): + if f == removed[0]: + assert (rec.flag, rec.rname, rec.pos, rec.cigar) == (h.UNMAPPED, '*', 0, '*') + assert rec.tags['YF'] == 'QC' + assert 'AS' not in rec.tags + assert rec.seq == s and rec.qual == q + else: + assert_placed(rec, genome_dict, spec, q) + + +def test_qc_filter_has_no_effect_on_fastq(bt2, genome_dict, genome_index, tmp_path): + """--qc-filter changes nothing for FASTQ input, which has no filter field.""" + args, _, _ = write_input('fastq', tmp_path, unpaired_reads(genome_dict)) + _, plain = run_align(bt2, genome_index, args) + _, filtered = run_align(bt2, genome_index, args + ['--qc-filter']) + assert len(sam_body(plain)) == len(UNPAIRED) + assert sam_body(filtered) == sam_body(plain) + + +def test_qseq_bad_filter_value(bt2, genome_dict, genome_index, tmp_path): + """A QSEQ filter field other than 0 or 1 is an error.""" + n, s, q = unpaired_reads(genome_dict)[0] + path = write_text(tmp_path / 'r_qseq.txt', qseq_line(n, s, q, '2')) + _, proc = run_align(bt2, genome_index, ['--qseq', '-U', path], check=False) + assert proc.returncode != 0 + assert 'for qseq filter flag' in proc.stderr + + +@pytest.mark.parametrize('fmt', ['fastq', 'fasta', 'qseq']) +def test_dot_is_read_as_n(bt2, genome_dict, genome_index, tmp_path, fmt): + """A '.' in a FASTQ, FASTA or QSEQ sequence is an N: SEQ shows N and AS is the N penalty (-1).""" + ref = genome_dict['chr3'][800:840] + read = ref[:12] + '.' + ref[13:] + args, _, _ = write_input(fmt, tmp_path, [('dot', read, 'I' * 40)]) + rec, = run_align(bt2, genome_index, args)[0] + assert rec.pos == 801 and rec.flag == 0 + assert rec.seq == ref[:12] + 'N' + ref[13:] + assert rec.tags['AS'] == -1 + assert rec.tags['NM'] == 1 + assert rec.tags['MD'] == '12%s27' % ref[12] + + +# ------------------------------------------------------------------- BAM + +def bam_test_records(gd): + """Records for the BAM tests: one pair (flags 77/141), two unpaired + unaligned reads (flag 4) and one record that is already aligned (flag 0).""" + m1, m2 = pair_seqs(gd, PAIRS[0]) + (n1, s1, q1), (n2, s2, q2) = unpaired_reads(gd)[:2] + qm1, qm2 = varied_qual(len(m1), 3), varied_qual(len(m2), 5) + return [ + ('p1', 77, m1, qm1, [('QZ', 'Z', 'mate one'), ('Qi', 'i', -7)]), + ('p1', 141, m2, qm2, [('QC', 'C', 200)]), + (n1, 4, s1, q1, [('QZ', 'Z', 'a b:c'), ('Qc', 'c', -100), ('QC', 'C', 250), + ('Qs', 's', -30000), ('QS', 'S', 60000), ('Qi', 'i', -2000000000), + ('QI', 'I', 4000000000), ('Qf', 'f', -0.25), ('Qb', 'B', ('c', [-1, 2])), + ('QB', 'B', ('S', [7, 65535])), ('Qg', 'B', ('f', [0.5, 2.0]))]), + (n2, 4, s2, q2, []), + ('aligned', 0, s1, q1, []), + ] + + +def test_bam_unpaired_by_default(bt2, genome_dict, genome_index, tmp_path): + """-b aligns only the unpaired records, taking SEQ and QUAL (Phred, no offset) from the BAM. + + That paired records are skipped is documented; that the already-aligned + (flag 0) record is skipped pins the current output.""" + recs_in = bam_test_records(genome_dict) + path = write_bam(str(tmp_path / 'r.bam'), recs_in) + recs, _ = run_align(bt2, genome_index, ['-b', path]) + assert [r.qname for r in recs] == ['u1', 'u2'] + assert_placed(recs[0], genome_dict, UNPAIRED[0], recs_in[2][3]) + assert_placed(recs[1], genome_dict, UNPAIRED[1], recs_in[3][3]) + assert not any(t.startswith('Q') for r in recs for t in r.tags) + + +def test_bam_align_paired_reads(bt2, genome_dict, genome_index, tmp_path): + """-b --align-paired-reads aligns the paired records as pairs and skips unpaired ones.""" + recs_in = bam_test_records(genome_dict) + path = write_bam(str(tmp_path / 'r.bam'), recs_in) + recs, _ = run_align(bt2, genome_index, ['-b', path, '--align-paired-reads']) + assert [r.qname for r in recs] == ['p1', 'p1'] + assert_pair(recs[0], recs[1], genome_dict, PAIRS[0], recs_in[0][3], recs_in[1][3]) + + +EXPECTED_PRESERVED = ['QZ:Z:a b:c', 'Qc:i:-100', 'QC:i:250', 'Qs:i:-30000', + 'QS:i:60000', 'Qi:i:-2000000000', 'QI:i:4000000000', 'Qf:f:{-0.25}', + 'Qb:B:c,-1,2', 'QB:B:S,7,65535', 'Qg:B:f,{0.5},{2.0}'] + + +def assert_tags_preserved(line, expected): + """The SAM line ends with `expected` tags; '{x}' marks a float compared numerically.""" + fields = line.split('\t')[-len(expected):] + for got, want in zip(fields, expected): + if '{' not in want: + assert got == want, line + continue + prefix = want.split('{')[0] + assert got.startswith(prefix), line + got_vals = [float(v) for v in got[len(prefix):].split(',')] + want_vals = [float(v.strip('{}')) for v in want[len(prefix):].split(',')] + assert got_vals == want_vals, line + + +def test_bam_preserve_tags_unpaired(bt2, genome_dict, genome_index, tmp_path): + """--preserve-tags appends each BAM tag, integer types as i, after bowtie2's own fields.""" + recs_in = bam_test_records(genome_dict) + recs_in.append(('unal', 4, 'ACGT' * 10, 'I' * 40, [('QZ', 'Z', 'kept')])) + path = write_bam(str(tmp_path / 'r.bam'), recs_in) + recs, _ = run_align(bt2, genome_index, ['-b', path, '--preserve-tags']) + assert [r.qname for r in recs] == ['u1', 'u2', 'unal'] + assert_placed(recs[0], genome_dict, UNPAIRED[0], recs_in[2][3]) + assert recs[0].tag_order.index('YT') < recs[0].tag_order.index('QZ') + assert_tags_preserved(recs[0].line, EXPECTED_PRESERVED) + assert not any(t.startswith('Q') for t in recs[1].tags) + assert recs[2].is_unmapped + assert recs[2].line.endswith('\tQZ:Z:kept') + + +def test_bam_preserve_tags_paired(bt2, genome_dict, genome_index, tmp_path): + """--preserve-tags with --align-paired-reads puts each mate's own BAM tags on that mate.""" + recs_in = bam_test_records(genome_dict) + path = write_bam(str(tmp_path / 'r.bam'), recs_in) + recs, _ = run_align(bt2, genome_index, ['-b', path, '--align-paired-reads', '--preserve-tags']) + assert_pair(recs[0], recs[1], genome_dict, PAIRS[0]) + assert recs[0].line.endswith('\tYT:Z:CP\tQZ:Z:mate one\tQi:i:-7') + assert recs[1].line.endswith('\tYT:Z:CP\tQC:i:200') + + +@pytest.mark.parametrize('option', ['--preserve-tags', '--align-paired-reads']) +def test_bam_options_require_bam_input(bt2, genome_dict, genome_index, tmp_path, option): + """--preserve-tags and --align-paired-reads are errors without -b.""" + args, _, _ = write_input('fastq', tmp_path, unpaired_reads(genome_dict)) + _, proc = run_align(bt2, genome_index, args + [option], check=False) + assert proc.returncode != 0 + assert 'can only be used when aligning BAM reads' in proc.stderr + + +def test_bam_shipped_example(bt2, lambda_index, example_reads): + """-b reads the shipped example BAM: QNAME, SEQ and QUAL match the file's records, in order. + + example/reads/combined_reads.bam spans many BGZF blocks; -s/-u count + unpaired records, or pairs with --align-paired-reads.""" + headers, data = read_bam_headers(example_reads['bam']) + unpaired = [r for r in headers if r[1] & 0x4 and not r[1] & 0x1] + firsts = [r for r in headers if r[1] & 0x40] + seconds = [r for r in headers if r[1] & 0x80] + assert len(firsts) == len(seconds) >= 5 and len(unpaired) >= 13 + + def check(rec, bam_rec): + seq, qual = bam_seq_qual(data, bam_rec[2], bam_rec[3]) + if rec.is_reverse: + seq, qual = h.revcomp(seq), qual[::-1] + assert (rec.qname, rec.seq, rec.qual) == (bam_rec[0], seq, qual) + + recs, _ = run_align(bt2, lambda_index, ['-b', example_reads['bam'], '-s', '5', '-u', '8']) + assert len(recs) == 8 + for rec, bam_rec in zip(recs, unpaired[5:13]): + check(rec, bam_rec) + recs, _ = run_align(bt2, lambda_index, ['-b', example_reads['bam'], '--align-paired-reads', '-u', '5']) + assert len(recs) == 10 + for k in range(5): + assert recs[2 * k].flag & h.FIRST_IN_PAIR and recs[2 * k + 1].flag & h.SECOND_IN_PAIR + check(recs[2 * k], firsts[k]) + check(recs[2 * k + 1], seconds[k]) + + +# ------------------------------------------------- read selection: -s / -u + +@pytest.mark.parametrize('args,expected', [ + ([], ['u1', 'u2', 'u3', 'u4']), + (['-s', '1'], ['u2', 'u3', 'u4']), + (['-s', '3'], ['u4']), + (['-s', '4'], []), + (['-s', '10'], []), + (['-u', '2'], ['u1', 'u2']), + (['-u', '9'], ['u1', 'u2', 'u3', 'u4']), + (['-s', '1', '-u', '2'], ['u2', 'u3']), + (['--skip', '1', '--upto', '2'], ['u2', 'u3']), + (['-s', '1', '--qupto', '2'], ['u2', 'u3']), +]) +def test_skip_upto(bt2, genome_dict, genome_index, tmp_path, args, expected): + """-s/--skip skips the first reads and -u/--upto/--qupto stops after that many more, counting across files.""" + reads = unpaired_reads(genome_dict) + a = h.write_fastq(str(tmp_path / 'a.fq'), reads[:2]) + b = h.write_fastq(str(tmp_path / 'b.fq.gz'), reads[2:], compress='gz') + recs, _ = run_align(bt2, genome_index, ['-U', a + ',' + b] + args) + assert [r.qname for r in recs] == expected + for rec in recs: + k = int(rec.qname[1]) - 1 + assert_placed(rec, genome_dict, UNPAIRED[k], reads[k][2]) + + +@pytest.mark.parametrize('args,expected', [ + (['-s', '1'], ['p2', 'p2']), + (['-u', '1'], ['p1', 'p1']), +]) +def test_skip_upto_count_pairs(bt2, genome_dict, genome_index, tmp_path, args, expected): + """-s and -u count pairs, not mates.""" + lines = [] + for pair in PAIRS: + m1, m2 = pair_seqs(genome_dict, pair) + lines.append('%s\t%s\t%s\t%s\t%s\n' % (pair[0], m1, 'I' * len(m1), m2, 'I' * len(m2))) + path = write_text(tmp_path / 'r.tab5', ''.join(lines)) + recs, _ = run_align(bt2, genome_index, ['--tab5', path] + args) + assert [r.qname for r in recs] == expected + pair = PAIRS[0] if expected[0] == 'p1' else PAIRS[1] + assert_pair(recs[0], recs[1], genome_dict, pair) + + +# ----------------------------------------------------------- trimming + +@pytest.mark.parametrize('fmt', FORMATS) +def test_trim5_trim3(bt2, genome_dict, genome_index, tmp_path, fmt): + """-5 3 -3 5 removes 3 bases from the read's 5' end and 5 from its 3' end, on either strand.""" + reads = unpaired_reads(genome_dict) + args, names, has_qual = write_input(fmt, tmp_path, reads) + recs, _ = run_align(bt2, genome_index, args + ['-5', '3', '-3', '5']) + assert [r.qname for r in recs] == names + for rec, spec, (n, s, q) in zip(recs, UNPAIRED, reads): + assert_placed(rec, genome_dict, spec, q if has_qual else 'I' * len(s), trim5=3, trim3=5) + + +def test_trim_long_options(bt2, genome_dict, genome_index, tmp_path): + """--trim5 and --trim3 are the same as -5 and -3.""" + args, _, _ = write_input('fastq', tmp_path, unpaired_reads(genome_dict)) + _, short = run_align(bt2, genome_index, args + ['-5', '4', '-3', '2']) + _, long_ = run_align(bt2, genome_index, args + ['--trim5', '4', '--trim3', '2']) + assert sam_body(long_) == sam_body(short) + assert len(sam_body(short)) == len(UNPAIRED) + assert all('\t%dM\t' % (spec[3] - 6) in line for spec, line in zip(UNPAIRED, sam_body(short))) + + +@pytest.mark.parametrize('fmt', ['fastq', 'bam']) +@pytest.mark.parametrize('spec,end', [('3:42', 3), ('5:42', 5), ('42', 3)]) +def test_trim_to(bt2, genome_dict, genome_index, tmp_path, fmt, spec, end): + """--trim-to [3:|5:]42 cuts reads to 42 bases from the given end (default 3'); a 40-base read is unchanged.""" + reads = unpaired_reads(genome_dict) + args, _, _ = write_input(fmt, tmp_path, reads) + recs, _ = run_align(bt2, genome_index, args + ['--trim-to', spec]) + assert len(recs) == len(UNPAIRED) + for rec, read_spec, (n, s, q) in zip(recs, UNPAIRED, reads): + cut = max(0, len(s) - 42) + trim5, trim3 = (0, cut) if end == 3 else (cut, 0) + assert_placed(rec, genome_dict, read_spec, q, trim5=trim5, trim3=trim3) + + +def test_trim_to_default_end_is_3(bt2, genome_dict, genome_index, tmp_path): + """--trim-to is the same as --trim-to 3:.""" + args, _, _ = write_input('fastq', tmp_path, unpaired_reads(genome_dict)) + _, bare = run_align(bt2, genome_index, args + ['--trim-to', '33']) + _, three = run_align(bt2, genome_index, args + ['--trim-to', '3:33']) + assert len(sam_body(three)) == len(UNPAIRED) + assert all('\t33M\t' in line for line in sam_body(three)) + assert sam_body(bare) == sam_body(three) + + +def test_trim_to_bad_end(bt2, genome_dict, genome_index, tmp_path): + """--trim-to with an end other than 3 or 5 is an error.""" + args, _, _ = write_input('fastq', tmp_path, unpaired_reads(genome_dict)) + _, proc = run_align(bt2, genome_index, args + ['--trim-to', '4:30'], check=False) + assert proc.returncode != 0 + assert 'trim position must be either 3 or 5' in proc.stderr + + +def test_trimming_whole_read_leaves_empty_read(bt2, genome_dict, genome_index, tmp_path): + """Trimming more bases than a read has leaves an empty, unaligned read (SEQ/QUAL '*', YF:Z:LN).""" + n, s, q = unpaired_reads(genome_dict)[0] + path = h.write_fastq(str(tmp_path / 'r.fq'), [(n, s, q)]) + rec, = run_align(bt2, genome_index, ['-U', path, '-5', '30', '-3', '30'])[0] + assert rec.qname == n and rec.flag == h.UNMAPPED + assert (rec.seq, rec.qual) == ('*', '*') + assert rec.tags['YF'] == 'LN' + + +# --------------------------------------------------- quality encodings + +def low_quality_mismatch_read(gd, qlow): + """A 50-base forward read from chr3:500 with one mismatch at offset 25, + where the base quality is `qlow`; every other base is Phred 40.""" + ref = gd['chr3'][500:550] + phred = [40] * 50 + phred[25] = qlow + return ref, h.mutate(ref, [25]), phred + + +@pytest.mark.parametrize('fmt', ['fastq', 'tab5', 'qseq']) +@pytest.mark.parametrize('qlow', [2, 20, 30]) +def test_phred33_phred64_agree(bt2, genome_dict, genome_index, tmp_path, fmt, qlow): + """A read in Phred+33 and in Phred+64 (with each alias) gives the same QUAL (Phred+33) and AS. + + AS is minus the quality-scaled mismatch penalty at the low-quality base.""" + ref, read, phred = low_quality_mismatch_read(genome_dict, qlow) + q33 = ''.join(chr(33 + p) for p in phred) + q64 = ''.join(chr(64 + p) for p in phred) + (tmp_path / '33').mkdir() + (tmp_path / '64').mkdir() + args33, _, _ = write_input(fmt, tmp_path / '33', [('lowq', read, q33)]) + args64, _, _ = write_input(fmt, tmp_path / '64', [('lowq', read, q64)]) + runs = [args33, args33 + ['--phred33'], args33 + ['--phred33-quals'], + args64 + ['--phred64'], args64 + ['--phred64-quals'], args64 + ['--solexa1.3-quals']] + bodies = [sam_body(run_align(bt2, genome_index, args)[1]) for args in runs] + assert all(b == bodies[0] for b in bodies) + rec = h.SamRecord(bodies[0][0]) + assert rec.pos == 501 and rec.flag == 0 + assert rec.qual == q33 + assert rec.tags['AS'] == -h.mismatch_penalty(chr(33 + qlow)) + assert rec.tags['XM'] == 1 and rec.tags['NM'] == 1 + assert rec.tags['MD'] == '25%s24' % ref[25] + + +@pytest.mark.parametrize('solexa', [-5, 0, 5, 9, 15, 40]) +def test_solexa_quals(bt2, genome_dict, genome_index, tmp_path, solexa): + """--solexa-quals converts Solexa+64 qualities to Phred, which sets QUAL and the mismatch penalty. + + Q = 10 log10(10^(S/10) + 1), rounded to the nearest integer.""" + ref, read, _ = low_quality_mismatch_read(genome_dict, 40) + qual = 'h' * 25 + chr(64 + solexa) + 'h' * 24 + phred = int(math.floor(10 * math.log10(10 ** (solexa / 10.0) + 1) + 0.5)) + path = h.write_fastq(str(tmp_path / 'r.fq'), [('sol', read, qual)]) + rec, = run_align(bt2, genome_index, ['-U', path, '--solexa-quals'])[0] + assert rec.pos == 501 + assert rec.qual == 'I' * 25 + chr(33 + phred) + 'I' * 24 + assert rec.tags['AS'] == -h.mismatch_penalty(chr(33 + phred)) + + +def test_phred64_rejects_phred33_input(bt2, genome_dict, genome_index, tmp_path): + """--phred64 on qualities below ASCII 64 is an error.""" + n, s, q = unpaired_reads(genome_dict)[0] + path = h.write_fastq(str(tmp_path / 'r.fq'), [(n, s, '5' * len(s))]) + _, proc = run_align(bt2, genome_index, ['-U', path, '--phred64'], check=False) + assert proc.returncode != 0 + assert 'expected 64-based Phred qual' in proc.stderr + + +# --------------------------------------- read files, compression, stdin + +def zstd_input_supported(bt2): + """True when the aligner was built WITH_ZSTD (shown in its --version + compiler options) and the zstd program is available to compress input.""" + if h.which('zstd') is None: + return False + proc = subprocess.run([os.path.join(bt2.bin_dir, 'bowtie2-align-s'), '--version'], + capture_output=True, text=True) + return '-DWITH_ZSTD' in proc.stdout + + +@pytest.mark.parametrize('kind', ['plain', 'gz', 'bz2', 'gz-no-extension', 'zst']) +def test_compressed_reads(bt2, genome_dict, genome_index, tmp_path, kind): + """Plain, gzip, bzip2 and zstd FASTQ read files align identically. + + gzip detection without a .gz extension pins current behaviour (the aligner + checks the gzip magic bytes).""" + reads = unpaired_reads(genome_dict) + if kind == 'bz2' and h.which('bzip2') is None: + pytest.skip('the wrapper decompresses .bz2 with the bzip2 program, not found') + if kind == 'zst' and not zstd_input_supported(bt2): + pytest.skip('bowtie2 built without WITH_ZSTD, or zstd program not found') + if kind == 'plain': + path = h.write_fastq(str(tmp_path / 'r.fq'), reads) + elif kind == 'gz': + path = h.write_fastq(str(tmp_path / 'r.fq.gz'), reads, compress='gz') + elif kind == 'bz2': + path = h.write_fastq(str(tmp_path / 'r.fq.bz2'), reads, compress='bz2') + elif kind == 'gz-no-extension': + path = h.write_fastq(str(tmp_path / 'r.fq'), reads, compress='gz') + else: + plain = h.write_fastq(str(tmp_path / 'r.fq'), reads) + path = str(tmp_path / 'r.fq.zst') + subprocess.run(['zstd', '-q', '-f', '-o', path, plain], check=True) + recs, _ = run_align(bt2, genome_index, ['-U', path]) + assert [r.qname for r in recs] == [r[0] for r in reads] + for rec, spec, (n, s, q) in zip(recs, UNPAIRED, reads): + assert_placed(rec, genome_dict, spec, q) + + +@pytest.mark.parametrize('layout', [ + ['-U', '{a},{b}', '-U', '{c}'], + ['-U', '{a}', '-U', '{b},{c}'], + ['-U', '{a}', '-U', '{b}', '-U', '{c}'], +]) +def test_comma_lists_and_repeated_U(bt2, genome_dict, genome_index, tmp_path, layout): + """Comma-separated -U lists and repeated -U read every file, in the order given.""" + reads = unpaired_reads(genome_dict) + paths = {'a': h.write_fastq(str(tmp_path / 'a.fq'), reads[:2]), + 'b': h.write_fastq(str(tmp_path / 'b.fq.gz'), reads[2:3], compress='gz'), + 'c': h.write_fastq(str(tmp_path / 'c.fq'), reads[3:])} + args = [a.format(**paths) for a in layout] + recs, _ = run_align(bt2, genome_index, args) + assert [r.qname for r in recs] == ['u1', 'u2', 'u3', 'u4'] + for rec, spec, (n, s, q) in zip(recs, UNPAIRED, reads): + assert_placed(rec, genome_dict, spec, q) + + +def test_repeated_U_with_bzip2(bt2, genome_dict, genome_index, tmp_path): + """A bzip2 file given in its own -U is read along with a plain one.""" + if h.which('bzip2') is None: + pytest.skip('the wrapper decompresses .bz2 with the bzip2 program, not found') + reads = unpaired_reads(genome_dict) + a = h.write_fastq(str(tmp_path / 'a.fq'), reads[:2]) + b = h.write_fastq(str(tmp_path / 'b.fq.bz2'), reads[2:], compress='bz2') + recs, _ = run_align(bt2, genome_index, ['-U', a, '-U', b]) + assert [r.qname for r in recs] == ['u1', 'u2', 'u3', 'u4'] + + +def test_compressed_mates(bt2, genome_dict, genome_index, tmp_path): + """-1/-2 read gzip and bzip2 mate files.""" + if h.which('bzip2') is None: + pytest.skip('the wrapper decompresses .bz2 with the bzip2 program, not found') + seqs = [pair_seqs(genome_dict, p) for p in PAIRS] + m1 = h.write_fastq(str(tmp_path / 'm_1.fq.gz'), [(p[0], s[0]) for p, s in zip(PAIRS, seqs)], compress='gz') + m2 = h.write_fastq(str(tmp_path / 'm_2.fq.bz2'), [(p[0], s[1]) for p, s in zip(PAIRS, seqs)], compress='bz2') + recs, _ = run_align(bt2, genome_index, ['-1', m1, '-2', m2]) + assert [r.qname for r in recs] == ['p1', 'p1', 'p2', 'p2'] + for k, pair in enumerate(PAIRS): + assert_pair(recs[2 * k], recs[2 * k + 1], genome_dict, pair) + + +@pytest.mark.parametrize('fmt', ['fastq', 'fastq-gz', 'fasta', 'tab5', 'bam']) +def test_reads_from_stdin(bt2, genome_dict, genome_index, tmp_path, fmt): + """'-' reads from standard input (FASTQ, gzipped FASTQ, FASTA, --tab5, BAM).""" + reads = unpaired_reads(genome_dict) + if fmt in ('fastq', 'fastq-gz'): + data = h.fastq_text(reads).encode() + if fmt == 'fastq-gz': + data = gzip.compress(data) + args = ['-U', '-'] + elif fmt == 'fasta': + data = ''.join('>%s\n%s\n' % (n, s) for n, s, q in reads).encode() + args = ['-f', '-U', '-'] + elif fmt == 'tab5': + data = ''.join('%s\t%s\t%s\n' % r for r in reads).encode() + args = ['--tab5', '-'] + else: + path = write_bam(str(tmp_path / 'r.bam'), [(n, 4, s, q, []) for n, s, q in reads]) + with open(path, 'rb') as f: + data = f.read() + args = ['-b', '-'] + recs, _ = run_bytes(bt2, ['-x', genome_index, '-p', '1'] + args, data) + assert [r.qname for r in recs] == [r[0] for r in reads] + for rec, spec, (n, s, q) in zip(recs, UNPAIRED, reads): + assert_placed(rec, genome_dict, spec, 'I' * len(s) if fmt == 'fasta' else q) + + +# ------------------------------------------------------ QNAME options + +def test_qname_truncated_by_default(bt2, genome_dict, genome_index, tmp_path): + """By default QNAME stops at the first whitespace and a long name is cut short; --sam-no-qname-trunc keeps both whole. + + The 300-character name may come out with 254 or 255 characters: the + current version cuts it to 255, one more than the SAM specification's + QNAME limit of 254, so either length passes here.""" + long_name = 'L' + 'x' * 299 + reads = unpaired_reads(genome_dict)[:2] + path = write_text(tmp_path / 'r.fq', '@u1 has a comment\n%s\n+\n%s\n@%s\n%s\n+\n%s\n' + % (reads[0][1], reads[0][2], long_name, reads[1][1], reads[1][2])) + recs, _ = run_align(bt2, genome_index, ['-U', path]) + assert recs[0].qname == 'u1' + assert long_name.startswith(recs[1].qname) and len(recs[1].qname) in (254, 255) + assert_placed(recs[1], genome_dict, UNPAIRED[1], reads[1][2]) + recs, _ = run_align(bt2, genome_index, ['-U', path, '--sam-no-qname-trunc']) + assert [r.qname for r in recs] == ['u1 has a comment', long_name] + assert_placed(recs[0], genome_dict, UNPAIRED[0], reads[0][2]) + assert_placed(recs[1], genome_dict, UNPAIRED[1], reads[1][2]) + + +@pytest.mark.parametrize('fmt', ['fastq', 'fasta']) +def test_sam_append_comment(bt2, genome_dict, genome_index, tmp_path, fmt): + """--sam-append-comment appends the text after the first space of the read name to the record. + + The BC:Z: prefix added to an Illumina-style comment (1:N:0:) + pins current output; the MANUAL does not mention it.""" + reads = unpaired_reads(genome_dict)[:2] + comments = ['QA:Z:hello\tQB:i:5', '1:N:0:ACGTAC'] + if fmt == 'fastq': + text = ''.join('@%s %s\n%s\n+\n%s\n' % (n, c, s, q) for (n, s, q), c in zip(reads, comments)) + args = ['-U', write_text(tmp_path / 'r.fq', text)] + else: + text = ''.join('>%s %s\n%s\n' % (n, c, s) for (n, s, q), c in zip(reads, comments)) + args = ['-f', '-U', write_text(tmp_path / 'r.fa', text)] + recs, _ = run_align(bt2, genome_index, args + ['--sam-append-comment']) + assert [r.qname for r in recs] == ['u1', 'u2'] + assert recs[0].line.endswith('\tYT:Z:UU\tQA:Z:hello\tQB:i:5') + assert recs[0].tags['QB'] == 5 + assert recs[1].line.endswith('\tYT:Z:UU\tBC:Z:1:N:0:ACGTAC') + for rec, spec, (n, s, q) in zip(recs, UNPAIRED, reads): + assert_placed(rec, genome_dict, spec, q if fmt == 'fastq' else 'I' * len(s)) + _, plain = run_align(bt2, genome_index, args) + assert 'QA:Z:hello' not in plain.stdout and 'BC:Z:' not in plain.stdout + + +def test_sam_append_comment_needs_fasta_or_fastq(bt2, genome_dict, genome_index, tmp_path): + """--sam-append-comment with --tab5 input is an error.""" + args, _, _ = write_input('tab5', tmp_path, unpaired_reads(genome_dict)) + _, proc = run_align(bt2, genome_index, args + ['--sam-append-comment'], check=False) + assert proc.returncode != 0 + assert '--sam-append-comment only works with FASTA (-f) and FASTQ (-q) formats' in proc.stderr + + +# ----------------------------------------------------- edge-case reads + +def test_read_shorter_than_seed(bt2, genome_dict, genome_index, tmp_path): + """Reads of 15 and 20 bases, shorter than the 22-base default seed, align end to end where they were cut.""" + specs = [('s15', 'chr1', 3000, 15, '+'), ('s20', 'chr2', 2500, 20, '-')] + reads = [(s[0], read_seq(genome_dict, s), varied_qual(s[3])) for s in specs] + path = h.write_fastq(str(tmp_path / 'r.fq'), reads) + recs, _ = run_align(bt2, genome_index, ['-U', path]) + assert [r.qname for r in recs] == ['s15', 's20'] + for rec, spec, (n, s, q) in zip(recs, specs, reads): + assert_placed(rec, genome_dict, spec, q) + + +def test_empty_and_all_n_reads(bt2, genome_dict, genome_index, tmp_path): + """An empty read gets an unaligned record with YF:Z:LN and an all-N read one with YF:Z:NS; later reads align. + + A 1-base read is left out: the current version also filters it with + YF:Z:LN under the default -N 0, which the MANUAL's definition of LN + (length <= -N) does not allow, so that case is not tested here.""" + n, s, q = unpaired_reads(genome_dict)[0] + path = h.write_fastq(str(tmp_path / 'r.fq'), + [('empty', '', ''), ('alln', 'N' * 40, 'I' * 40), (n, s, q)]) + recs, _ = run_align(bt2, genome_index, ['-U', path]) + assert [r.qname for r in recs] == ['empty', 'alln', n] + for rec, seq, yf in zip(recs[:2], ['*', 'N' * 40], ['LN', 'NS']): + assert rec.flag == h.UNMAPPED and rec.rname == '*' and rec.pos == 0 + assert rec.seq == seq + assert rec.tags['YF'] == yf + assert 'AS' not in rec.tags + assert recs[0].qual == '*' + assert_placed(recs[2], genome_dict, UNPAIRED[0], q) + + +def test_lowercase_and_ambiguous_bases(bt2, genome_dict, genome_index, tmp_path): + """Lowercase read bases are read as uppercase and IUPAC codes other than ACGT as N. + + The MANUAL says IUPAC codes become N for reference characters; for read + characters, and for lowercase, this pins the current output.""" + ref = genome_dict['chr1'][4000:4040] + read = ref[:10].lower() + ref[10:20] + 'R' + ref[21:] + path = h.write_fastq(str(tmp_path / 'r.fq'), [('lc', read)]) + rec, = run_align(bt2, genome_index, ['-U', path])[0] + assert rec.pos == 4001 and rec.flag == 0 + assert rec.seq == ref[:20] + 'N' + ref[21:] + assert rec.tags['AS'] == -1 + assert rec.tags['MD'] == '20%s19' % ref[20] diff --git a/scripts/test/test_output.py b/scripts/test/test_output.py new file mode 100644 index 00000000..f071be95 --- /dev/null +++ b/scripts/test/test_output.py @@ -0,0 +1,1358 @@ +"""bowtie2 output options, performance options, the Perl wrapper and errors. + +Covers -S, -t, --un/--al (plain and compressed), --quiet, the metrics options, +the SAM options (--no-unal, --no-hd, --no-sq, --rg-id, --rg, --omit-sec-seq, +--xeq, --sam-opt-config), header contents, the unpaired alignment summary, +-p/--reorder/--mm, OMP_NUM_THREADS, --version/-h, error exits, the wrapper-only +options (--debug, --sanitized, --verbose, --keep, --temp-directory, +--no-named-pipes, --log-file, --large-index, --ref-string, --bam) and index +auto-detection. Reads are cut from the fixed random `genome` (and from two +small references built here) at known places, so positions, strands, CIGARs, +MD strings and scores are known before bowtie2 runs. +""" + +import bz2 +import gzip +import os +import random +import re +import shutil +import stat +import subprocess +import sys +import threading +import time + +import pytest + +import bt2_helpers as h + + +# ---------------------------------------------------------------- helpers -- + +def version_string(bt2): + """The version bowtie2 was built with, read from BOWTIE2_VERSION.""" + for d in (bt2.bin_dir, h.ROOT): + path = os.path.join(d, 'BOWTIE2_VERSION') + if os.path.exists(path): + with open(path) as f: + return f.read().strip() + pytest.skip('BOWTIE2_VERSION not found') + + +def body(text): + """SAM text without header lines.""" + return [line for line in text.splitlines() if line and not line.startswith('@')] + + +def headers(text): + return [line for line in text.splitlines() if line.startswith('@')] + + +def opt_fields(line): + """Optional fields of a SAM line as (tag, type, value) triples. Values stay + strings because some bowtie2 debug fields print NA in an i-typed field.""" + return [tuple(f.split(':', 2)) for f in line.split('\t')[11:] if f] + + +def opt_dict(line): + return dict((t, v) for t, _, v in opt_fields(line)) + + +def fastq_records(text): + """Split FASTQ text into raw 4-line records, keeping every line verbatim.""" + lines = text.splitlines() + assert len(lines) % 4 == 0, text + return [tuple(lines[i:i + 4]) for i in range(0, len(lines), 4)] + + +def fasta_records(text): + lines = text.splitlines() + assert len(lines) % 2 == 0, text + return [tuple(lines[i:i + 2]) for i in range(0, len(lines), 2)] + + +def qname(read_name): + """SAM QNAME of a read: its name up to the first whitespace.""" + return read_name.split()[0] + + +def read_output(path, kind): + """Decompress a file written by --un/--al and friends.""" + if kind == 'gz': + with gzip.open(path, 'rt') as f: + return f.read() + if kind == 'bz2': + with bz2.open(path, 'rt') as f: + return f.read() + if kind in ('zst', 'lz4'): + tool = {'zst': 'zstd', 'lz4': 'lz4'}[kind] + return subprocess.run([tool, '-dc', path], capture_output=True, text=True, + check=True).stdout + with open(path) as f: + return f.read() + + +def align_bin(bt2, name='bowtie2-align-s'): + return os.path.join(bt2.bin_dir, name) + + +def env_without_omp(**extra): + env = dict(os.environ) + env.pop('OMP_NUM_THREADS', None) + env.update(extra) + return env + + +def metrics_rows(text): + """(header columns, data rows) of bowtie2 metrics output in `text`.""" + cols, rows = None, [] + for line in text.splitlines(): + if line.startswith('Time\tRead\tBase'): + cols = line.rstrip('\t').split('\t') + elif cols is not None and re.match(r'^\d+\t\d+\t', line): + rows.append(line.rstrip('\t').split('\t')) + return cols, rows + + +def pct(num, den): + return '%.2f%%' % (100.0 * num / den if den else 0.0) + + +def unescape_passthrough(line): + return re.sub(r'%([0-9A-Fa-f]{2})', lambda m: chr(int(m.group(1), 16)), line) + + +# ------------------------------------------------------------- read sets -- + +def varied_qual(n): + """A quality string cycling through Phred 2..40.""" + return ''.join(chr(33 + 2 + (i * 7) % 39) for i in range(n)) + + +@pytest.fixture(scope='module') +def unp(genome_dict, tmp_path_factory): + """Unpaired reads with known placements, written to one FASTQ. + + Returns a dict with 'reads' (name, seq, qual) as written, 'fq' (path), + 'aligned' {qname: expected placement} and 'unaligned' (qnames).""" + c1, c2, c3 = genome_dict['chr1'], genome_dict['chr2'], genome_dict['chr3'] + rng = random.Random(31337) + reads = [ + ('fw_exact first comment', c1[100:150], 'I' * 50), + ('rc_exact', h.revcomp(c2[500:560]), varied_qual(60)), + ('fw_mm', h.mutate(c1[2000:2060], [7, 50]), 'I' * 60), + ('rc_mm', h.revcomp(h.mutate(c3[1000:1060], [7, 50])), 'I' * 60), + ('unal_1 second comment', h.random_dna(50, rng), varied_qual(50)), + ('unal_2', h.random_dna(55, rng), 'I' * 55), + ('filt_ns', 'ACGT' * 5 + 'N' * 30, 'I' * 50), + ] + aligned = { + 'fw_exact': dict(rname='chr1', pos=101, rev=False, length=50, md='50', nmm=0), + 'rc_exact': dict(rname='chr2', pos=501, rev=True, length=60, md='60', nmm=0), + 'fw_mm': dict(rname='chr1', pos=2001, rev=False, length=60, + md='7%s42%s9' % (c1[2007], c1[2050]), nmm=2), + 'rc_mm': dict(rname='chr3', pos=1001, rev=True, length=60, + md='7%s42%s9' % (c3[1007], c3[1050]), nmm=2), + } + path = str(tmp_path_factory.mktemp('unp') / 'unp.fq') + h.write_fastq(path, reads) + return dict(reads=reads, fq=path, aligned=aligned, + unaligned=['unal_1', 'unal_2', 'filt_ns']) + + +@pytest.fixture(scope='module') +def pair(genome_dict, tmp_path_factory): + """One concordant --fr pair: mate 1 forward with one mismatch (AS -6), + mate 2 reverse with two mismatches (AS -12), fragment 1001-1300 on chr1.""" + c1 = genome_dict['chr1'] + d = tmp_path_factory.mktemp('pair') + m1 = h.write_fastq(str(d / 'p_1.fq'), [('pr', h.mutate(c1[1000:1050], [25]))]) + m2 = h.write_fastq(str(d / 'p_2.fq'), + [('pr', h.revcomp(h.mutate(c1[1250:1300], [10, 40])))]) + return dict(m1=m1, m2=m2) + + +@pytest.fixture(scope='module') +def rep(bt2, tmp_path_factory): + """A one-chromosome reference with an 80-bp unit repeated three times and a + 50-bp segment followed by a copy with six mismatches in its middle (seed + 0-21 stays exact), plus reads. + + The FASTA name carries a description, which bowtie2 drops from @SQ SN.""" + rng = random.Random(42) + unit = h.random_dna(80, rng) + seg = h.random_dna(50, rng) + bg = [h.random_dna(300, rng) for _ in range(6)] + parts = [bg[0], unit, bg[1], unit, bg[2], unit, bg[3], seg, bg[4], + h.mutate(seg, [24, 27, 30, 33, 36, 39]), bg[5]] + seq = ''.join(parts) + offsets = [sum(len(p) for p in parts[:i]) for i in range(len(parts))] + unit_starts = [offsets[1], offsets[3], offsets[5]] + d = tmp_path_factory.mktemp('rep') + fasta = h.write_fasta(str(d / 'rep.fa'), [('rep extra description', seq)]) + prefix = str(d / 'rep') + bt2.build_index(fasta, prefix, extra=['--seed', '0']) + reads = [ + ('rep_fw', unit[10:60]), + ('rep_rc', h.revcomp(unit[20:70])), + ('uni_fw', bg[0][100:150]), + ('uni_rc', h.revcomp(bg[3][50:100])), + ('near', seg), + ('unal', h.random_dna(50, random.Random(99))), + ] + fq = h.write_fastq(str(d / 'rep.fq'), reads) + return dict(index=prefix, fq=fq, seq=seq, reads=reads, + rep_fw_pos=sorted(s + 10 + 1 for s in unit_starts), + rep_rc_pos=sorted(s + 20 + 1 for s in unit_starts), + near_pos=offsets[7] + 1) + + +@pytest.fixture(scope='module') +def many(genome, tmp_path_factory): + """1200 reads (1100 cut from the genome, some with a mismatch, half reverse + complemented; 100 random) in shuffled order, for the threading tests. At + this size -p 4 output without --reorder came out of input order in each of + 30 runs pinned to 1, 2 and 4 cores, so test_reorder notices a --reorder + that does nothing (very likely, though not certainly).""" + rng = random.Random(4242) + reads = [] + for i in range(1100): + name, seq = genome[rng.randrange(len(genome))] + length = rng.choice([40, 50, 75, 100]) + start = rng.randrange(len(seq) - length) + s = seq[start:start + length] + if rng.random() < 0.5: + s = h.mutate(s, [rng.randrange(2, 8)]) + if rng.random() < 0.5: + s = h.revcomp(s) + reads.append(('m%04d' % i, s)) + for i in range(100): + reads.append(('n%03d' % i, h.random_dna(60, rng))) + rng.shuffle(reads) + path = str(tmp_path_factory.mktemp('many') / 'many.fq') + h.write_fastq(path, reads) + return dict(fq=path, names=[r[0] for r in reads]) + + +def unp_args(genome_index, unp): + return ['-x', genome_index, '-U', unp['fq'], '-p', '1'] + + +# ------------------------------------------------------------ -S / -t -- + +@pytest.mark.parametrize('opt', ['-S', '--output']) +def test_S_writes_sam_to_file(bt2, genome_index, unp, tmp_path, opt): + """-S/--output sends the SAM to the file and nothing to stdout.""" + ref = bt2.run('bowtie2', unp_args(genome_index, unp)).stdout + out = str(tmp_path / 'out.sam') + proc = bt2.run('bowtie2', unp_args(genome_index, unp) + [opt, out]) + assert proc.stdout == '' + with open(out) as f: + text = f.read() + assert body(text) == body(ref) + assert [l for l in headers(text) if not l.startswith('@PG')] == \ + [l for l in headers(ref) if not l.startswith('@PG')] + assert len(body(text)) == len(unp['reads']) + + +def test_time_prints_phase_times(bt2, genome_index, unp): + """-t/--time adds hh:mm:ss timings for index loading and the whole run; + the labels pin the current output.""" + plain = bt2.run('bowtie2', unp_args(genome_index, unp)) + for opt in ('-t', '--time'): + timed = bt2.run('bowtie2', unp_args(genome_index, unp) + [opt]) + lines = timed.stderr.splitlines() + assert any(re.match(r'^Time loading .+: \d\d:\d\d:\d\d$', l) for l in lines), timed.stderr + assert any(re.match(r'^Overall time: \d\d:\d\d:\d\d$', l) for l in lines), timed.stderr + assert body(timed.stdout) == body(plain.stdout) + assert not re.search(r'\d\d:\d\d:\d\d', plain.stderr) + + +# ---------------------------------------------------------- --un / --al -- + +COMPRESSION = [ + ('', None, None), + ('-gz', 'gz', 'gzip'), + ('-bz2', 'bz2', 'bzip2'), + ('-zst', 'zst', 'zstd'), + ('-lz4', 'lz4', 'lz4'), +] + + +@pytest.mark.parametrize('suffix,kind,tool', COMPRESSION, + ids=['plain', 'gz', 'bz2', 'zst', 'lz4']) +def test_un_al_split_reads_by_alignment(bt2, genome_index, unp, tmp_path, suffix, kind, tool): + """--un/--al (and their compressed forms) receive the unaligned/aligned + reads exactly as they were in the input, and the SAM is unchanged.""" + if tool is not None and shutil.which(tool) is None: + pytest.skip('%s not installed' % tool) + un, al, sam = str(tmp_path / 'un.out'), str(tmp_path / 'al.out'), str(tmp_path / 'o.sam') + bt2.run('bowtie2', unp_args(genome_index, unp) + ['--un' + suffix, un, + '--al' + suffix, al, '-S', sam]) + records = fastq_records(h.fastq_text(unp['reads'])) + by_name = dict((qname(r[0][1:]), r) for r in records) + got_un = fastq_records(read_output(un, kind)) + got_al = fastq_records(read_output(al, kind)) + assert sorted(got_un) == sorted(by_name[n] for n in unp['unaligned']) + assert sorted(got_al) == sorted(by_name[n] for n in unp['aligned']) + with open(sam) as f: + assert body(f.read()) == body(bt2.run('bowtie2', unp_args(genome_index, unp)).stdout) + + +@pytest.mark.parametrize('variant', ['trim', 'phred64']) +def test_un_al_keep_original_record(bt2, genome_index, unp, tmp_path, variant): + """--un/--al write the untrimmed read with its original quality encoding.""" + reads, extra = unp['reads'], [] + if variant == 'trim': + extra = ['-5', '3', '-3', '10'] + else: + reads = [(n, s, ''.join(chr(ord(c) + 31) for c in q)) for n, s, q in reads] + extra = ['--phred64'] + fq = h.write_fastq(str(tmp_path / 'in.fq'), reads) + un, al = str(tmp_path / 'un.fq'), str(tmp_path / 'al.fq') + _, recs, _ = bt2.align(['-x', genome_index, '-U', fq, '-p', '1', '--un', un, '--al', al] + extra) + assert sorted(r.qname for r in recs if r.is_unmapped) == sorted(unp['unaligned']) + records = dict((qname(r[0][1:]), r) for r in fastq_records(h.fastq_text(reads))) + with open(un) as f: + assert sorted(fastq_records(f.read())) == sorted(records[n] for n in unp['unaligned']) + with open(al) as f: + assert sorted(fastq_records(f.read())) == sorted(records[n] for n in unp['aligned']) + + +def test_un_al_fasta_input(bt2, genome_index, unp, tmp_path): + """With -f input, --un/--al write the FASTA records as they were input.""" + fa = str(tmp_path / 'in.fa') + with open(fa, 'w') as f: + for name, seq, _ in unp['reads']: + f.write('>%s\n%s\n' % (name, seq)) + un, al = str(tmp_path / 'un.fa'), str(tmp_path / 'al.fa') + bt2.run('bowtie2', ['-x', genome_index, '-f', '-U', fa, '-p', '1', '--un', un, + '--al', al, '-S', str(tmp_path / 'o.sam')]) + with open(fa) as f: + records = dict((qname(r[0][1:]), r) for r in fasta_records(f.read())) + with open(un) as f: + assert sorted(fasta_records(f.read())) == sorted(records[n] for n in unp['unaligned']) + with open(al) as f: + assert sorted(fasta_records(f.read())) == sorted(records[n] for n in unp['aligned']) + + +def test_un_al_directory_path(bt2, genome_index, unp, tmp_path): + """--un/--al given a directory write un-seqs/al-seqs inside it (wrapper + behaviour; the file names pin the current output).""" + d = tmp_path / 'outdir' + d.mkdir() + bt2.run('bowtie2', unp_args(genome_index, unp) + ['--un', str(d), '--al', str(d), + '-S', str(tmp_path / 'o.sam')]) + assert sorted(os.listdir(str(d))) == ['al-seqs', 'un-seqs'] + with open(str(d / 'un-seqs')) as f: + assert sorted(qname(r[0][1:]) for r in fastq_records(f.read())) == sorted(unp['unaligned']) + with open(str(d / 'al-seqs')) as f: + assert sorted(qname(r[0][1:]) for r in fastq_records(f.read())) == sorted(unp['aligned']) + + +# -------------------------------------------------------------- --quiet -- + +def test_quiet_suppresses_summary(bt2, genome_index, unp): + """--quiet leaves stderr empty on success and does not change the SAM.""" + plain = bt2.run('bowtie2', unp_args(genome_index, unp)) + quiet = bt2.run('bowtie2', unp_args(genome_index, unp) + ['--quiet']) + assert 'overall alignment rate' in plain.stderr + assert quiet.stderr == '' + assert body(quiet.stdout) == body(plain.stdout) + + +# ------------------------------------------------------------- metrics -- + +def check_metrics(text, unp): + cols, rows = metrics_rows(text) + assert cols is not None, text + assert rows, text + final = dict(zip(cols, rows[-1])) + assert int(final['Read']) == len(unp['reads']) + assert int(final['Base']) == sum(len(r[1]) for r in unp['reads']) + assert int(final['UnfilteredRead']) == len(unp['reads']) - 1 # filt_ns + + +def test_met_file(bt2, genome_index, unp, tmp_path): + """--met-file writes a metrics table whose final row counts the reads and + bases; nothing goes to stderr and the SAM is unchanged.""" + met = str(tmp_path / 'met.txt') + proc = bt2.run('bowtie2', unp_args(genome_index, unp) + ['--met-file', met]) + with open(met) as f: + check_metrics(f.read(), unp) + assert 'Time\tRead' not in proc.stderr + assert body(proc.stdout) == body(bt2.run('bowtie2', unp_args(genome_index, unp)).stdout) + + +def test_met_stderr_and_file_together(bt2, genome_index, unp, tmp_path): + """--met-stderr prints the metrics table to stderr, also when --met-file + is given (the two are not mutually exclusive).""" + proc = bt2.run('bowtie2', unp_args(genome_index, unp) + ['--met-stderr']) + check_metrics(proc.stderr, unp) + met = str(tmp_path / 'met.txt') + proc = bt2.run('bowtie2', unp_args(genome_index, unp) + ['--met-stderr', '--met-file', met]) + check_metrics(proc.stderr, unp) + with open(met) as f: + check_metrics(f.read(), unp) + + +def feed_copies(proc, data, stop, sent): + """Write `data` to proc's stdin again and again until `stop` is set, then + close it; sent[0] counts the copies written in full.""" + try: + while not stop.is_set(): + proc.stdin.write(data) + proc.stdin.flush() + sent[0] += 1 + proc.stdin.close() + except (OSError, ValueError): + pass # the process died or was killed; the test reports it + + +def test_met_interval(bt2, genome_index, many, tmp_path): + """--met sets the reporting period (default 1 s): fed reads for long + enough that --met 1 and the default each print two interval rows before + the final one, --met 100, fed the same way, prints only the final row, + which counts every read.""" + # bowtie2 checks the clock only between reads, so an interval row needs + # reads to be aligned at least s after the search threads start. A + # fixed sleep before sending the reads would depend on how fast bowtie2 + # starts on a loaded machine; instead each process is fed copies of the + # reads until the --met 1 and default processes have both printed two + # interval rows (a second or two of aligning), with a generous deadline. + # Two rows are printed at different clock seconds, so the --met 100 + # process, fed alongside, has also been aligning across a second boundary. + with open(many['fq'], 'rb') as f: + data = f.read() + ivals = ('1', 'default', '100') + errs = dict((i, str(tmp_path / ('met_%s.txt' % i))) for i in ivals) + procs, sent, feeders = {}, {}, [] + stop = threading.Event() + + def rows(ival): + with open(errs[ival]) as f: + return metrics_rows(f.read())[1] + + try: + for ival in ivals: + met = [] if ival == 'default' else ['--met', ival] + with open(errs[ival], 'wb') as err: + procs[ival] = subprocess.Popen([align_bin(bt2), '--wrapper', 'basic-0', '-x', + genome_index, '-U', '-', '-p', '1', '--met-stderr', '--quiet', '-S', + os.devnull] + met, + stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=err, + env=env_without_omp()) + sent[ival] = [0] + t = threading.Thread(target=feed_copies, args=(procs[ival], data, stop, sent[ival])) + t.daemon = True + t.start() + feeders.append(t) + deadline = time.time() + 60 + while not all(len(rows(i)) >= 2 for i in ('1', 'default')): + for ival, p in procs.items(): + assert p.poll() is None, 'bowtie2-align-s --met %s exited early' % ival + assert time.time() < deadline, dict((i, len(rows(i))) for i in ivals) + time.sleep(0.05) + stop.set() + for t in feeders: + t.join(120) + for ival, p in procs.items(): + assert p.wait(timeout=120) == 0, rows(ival) + finally: + stop.set() + for p in procs.values(): + if p.poll() is None: + p.kill() + p.wait() + final = dict((i, rows(i)) for i in ivals) + assert len(final['100']) == 1 + assert len(final['1']) >= 3 + assert len(final['default']) >= 3 + for ival in ivals: + assert sent[ival][0] >= 1 + assert int(final[ival][-1][1]) == sent[ival][0] * len(many['names']) + + +# ---------------------------------------------------------- SAM options -- + +def test_default_records_match_design(bt2, genome_index, unp): + """Baseline for the SAM option tests: each designed read lands where it + was cut, with the documented default optional fields.""" + _, recs, _ = bt2.align(unp_args(genome_index, unp)) + assert [r.qname for r in recs] == [qname(r[0]) for r in unp['reads']] + for r in recs: + exp = unp['aligned'].get(r.qname) + if exp is None: + assert r.flag == h.UNMAPPED and r.rname == '*' and r.pos == 0 + assert sorted(r.tag_order) == (['YF', 'YT'] if r.qname == 'filt_ns' else ['YT']) + continue + assert (r.rname, r.pos, r.is_reverse) == (exp['rname'], exp['pos'], exp['rev']) + assert r.cigar == '%dM' % exp['length'] + assert r.tags['MD'] == exp['md'] + assert r.tags['AS'] == -6 * exp['nmm'] + assert r.tags['XM'] == r.tags['NM'] == exp['nmm'] + assert sorted(r.tag_order) == ['AS', 'MD', 'NM', 'XG', 'XM', 'XN', 'XO', 'YT'] + assert recs[-1].tags['YF'] == 'NS' + + +@pytest.mark.parametrize('mode', ['wrapper', 'binary']) +def test_no_unal(bt2, genome_index, unp, tmp_path, mode): + """--no-unal drops every unaligned record and leaves the aligned ones as + they were (the wrapper filters itself; the binary has its own path).""" + ref = body(bt2.run('bowtie2', unp_args(genome_index, unp)).stdout) + if mode == 'wrapper': + out = str(tmp_path / 'o.sam') + bt2.run('bowtie2', unp_args(genome_index, unp) + ['--no-unal', '-S', out]) + with open(out) as f: + got = body(f.read()) + else: + got = body(bt2.run(align_bin(bt2), unp_args(genome_index, unp) + ['--no-unal']).stdout) + assert got == [l for l in ref if not int(l.split('\t')[1]) & h.UNMAPPED] + assert sorted(l.split('\t')[0] for l in got) == sorted(unp['aligned']) + + +def test_no_unal_still_writes_un(bt2, genome_index, unp, tmp_path): + """--no-unal hides unaligned reads from the SAM but --un still gets them.""" + un = str(tmp_path / 'un.fq') + proc = bt2.run('bowtie2', unp_args(genome_index, unp) + ['--no-unal', '--un', un]) + assert sorted(l.split('\t')[0] for l in body(proc.stdout)) == sorted(unp['aligned']) + with open(un) as f: + assert sorted(qname(r[0][1:]) for r in fastq_records(f.read())) == sorted(unp['unaligned']) + + +NO_HD = ['--no-hd', '--no-head', '--no-HD', '--sam-no-hd', '--sam-no-head', '--sam-nohead', + '--sam-noHD'] +NO_SQ = ['--no-sq', '--no-SQ', '--sam-no-sq', '--sam-nosq', '--sam-noSQ'] + + +@pytest.mark.parametrize('opt', NO_HD) +def test_no_hd(bt2, genome_index, unp, opt): + """--no-hd (and its aliases) removes every header line and nothing else.""" + ref = bt2.run('bowtie2', unp_args(genome_index, unp)).stdout + out = bt2.run('bowtie2', unp_args(genome_index, unp) + [opt]).stdout + assert headers(out) == [] + assert body(out) == body(ref) + + +@pytest.mark.parametrize('opt', NO_SQ) +def test_no_sq(bt2, genome_index, unp, opt): + """--no-sq (and its aliases) removes the @SQ lines but keeps @HD and @PG.""" + ref = bt2.run('bowtie2', unp_args(genome_index, unp)).stdout + out = bt2.run('bowtie2', unp_args(genome_index, unp) + [opt]).stdout + assert [l.split('\t')[0] for l in headers(out)] == ['@HD', '@PG'] + assert headers(out)[0] == headers(ref)[0] + assert body(out) == body(ref) + + +def test_rg_id_and_rg(bt2, genome_index, unp): + """--rg-id makes an @RG line (ID first, then each --rg in order) and tags + every record with RG:Z:; @RG sitting between @SQ and @PG and RG:Z + coming last pin the current output.""" + header, recs, _ = bt2.align(unp_args(genome_index, unp) + ['--rg-id', 'grp1', + '--rg', 'SM:sample', '--rg', 'PL:ILLUMINA', '--rg', 'DS:two words']) + assert [l.split('\t')[0] for l in header] == ['@HD', '@SQ', '@SQ', '@SQ', '@RG', '@PG'] + assert header[4] == '@RG\tID:grp1\tSM:sample\tPL:ILLUMINA\tDS:two words' + assert len(recs) == len(unp['reads']) + for r in recs: + assert r.tags['RG'] == 'grp1', r + assert r.tag_order[-1] == 'RG' + + +def test_rg_without_rg_id(bt2, genome_index, unp): + """--rg without --rg-id prints no @RG line and no RG:Z field.""" + ref = bt2.run('bowtie2', unp_args(genome_index, unp)).stdout + out = bt2.run('bowtie2', unp_args(genome_index, unp) + ['--rg', 'SM:x']).stdout + assert not any(l.startswith('@RG') for l in headers(out)) + assert body(out) == body(ref) + + +@pytest.mark.parametrize('args', [ + ['--rg', 'ID:grp1', '--rg', 'SM:s'], + ['--sam-rg-id', 'grp1', '--sam-rg', 'SM:s'], + ['--rg-id', 'grp1', '--sam-RG', 'SM:s'], + ['--rg-id', 'grp1', '--RG', 'SM:s'], +], ids=['rg-ID-form', 'sam-rg', 'sam-RG', 'RG']) +def test_rg_equivalent_spellings(bt2, genome_index, unp, args): + """--rg ID: and the --sam-rg*/--RG aliases equal --rg-id/--rg.""" + ref = bt2.run('bowtie2', unp_args(genome_index, unp) + ['--rg-id', 'grp1', '--rg', 'SM:s']).stdout + out = bt2.run('bowtie2', unp_args(genome_index, unp) + args).stdout + assert [l for l in headers(out) if not l.startswith('@PG')] == \ + [l for l in headers(ref) if not l.startswith('@PG')] + assert body(out) == body(ref) + + +def test_no_hd_keeps_rg_field(bt2, genome_index, unp): + """--no-hd with --rg-id drops the @RG line but keeps RG:Z on records.""" + out = bt2.run('bowtie2', unp_args(genome_index, unp) + ['--no-hd', '--rg-id', 'g']).stdout + assert headers(out) == [] + assert all(opt_dict(l)['RG'] == 'g' for l in body(out)) + + +@pytest.mark.parametrize('opt', ['--omit-sec-seq', '--sam-omit-sec-seq']) +def test_omit_sec_seq(bt2, rep, opt): + """--omit-sec-seq puts * in SEQ and QUAL of secondary (-k) records only.""" + args = ['-x', rep['index'], '-U', rep['fq'], '-p', '1', '-k', '3'] + _, full, _ = bt2.align(args) + _, omit, _ = bt2.align(args + [opt]) + reads = dict(rep['reads']) + for name, positions, rev in (('rep_fw', rep['rep_fw_pos'], False), + ('rep_rc', rep['rep_rc_pos'], True)): + recs = h.by_name(full)[name] + assert sorted(r.pos for r in recs) == positions + assert [r.is_secondary for r in recs] == [False, True, True] + assert all(r.is_reverse == rev for r in recs) + seq = h.revcomp(reads[name]) if rev else reads[name] + assert all(r.seq == seq and r.qual == 'I' * 50 for r in recs) + orecs = h.by_name(omit)[name] + assert orecs[0].seq == seq and orecs[0].qual == 'I' * 50 + assert [(r.seq, r.qual) for r in orecs[1:]] == [('*', '*'), ('*', '*')] + assert [(r.flag, r.pos, r.cigar) for r in orecs] == [(r.flag, r.pos, r.cigar) for r in recs] + # Reads without secondary alignments are untouched. + assert [r.line for r in full if r.qname not in ('rep_fw', 'rep_rc')] == \ + [r.line for r in omit if r.qname not in ('rep_fw', 'rep_rc')] + + +def xeq_reads(genome_dict): + """Reads with mismatches and gaps placed so the left-aligned CIGAR is + unique; returns (name, seq, rname, pos, xeq_cigar, m_cigar).""" + c1, c3 = genome_dict['chr1'], genome_dict['chr3'] + out = [ + ('x_fw', h.mutate(c1[2000:2060], [7, 50]), 'chr1', 2001, '7=1X42=1X9=', '60M'), + ('x_rc', h.revcomp(h.mutate(c3[1000:1060], [7, 50])), 'chr3', 1001, '7=1X42=1X9=', '60M'), + ] + # 3-bp deletion at read offset a: unique when ref[a-1] != ref[a+2]. + st = 4000 + a = next(a for a in range(28, 40) if c1[st + a - 1] != c1[st + a + 2]) + out.append(('x_del', c1[st:st + a] + c1[st + a + 3:st + 63], 'chr1', st + 1, + '%d=3D%d=' % (a, 60 - a), '%dM3D%dM' % (a, 60 - a))) + # 2-bp insertion before ref offset b: unique when ins[-1] != ref[b-1] and + # ins[0] != ref[b]. + st = 1500 + b = 30 + ins = next(x + y for x in 'ACGT' for y in 'ACGT' + if y != c1[st + b - 1] and x != c1[st + b] and x != y) + out.append(('x_ins', c1[st:st + b] + ins + c1[st + b:st + 58], 'chr1', st + 1, + '%d=2I%d=' % (b, 28), '%dM2I%dM' % (b, 28))) + return out + + +def test_xeq(bt2, genome_index, genome_dict, tmp_path): + """--xeq writes =/X instead of M at the placed mismatches; everything else + in the record is unchanged.""" + design = xeq_reads(genome_dict) + fq = h.write_fastq(str(tmp_path / 'x.fq'), [(d[0], d[1]) for d in design]) + args = ['-x', genome_index, '-U', fq, '-p', '1'] + _, plain, _ = bt2.align(args) + _, xeq, _ = bt2.align(args + ['--xeq']) + assert len(plain) == len(xeq) == len(design) + for d, p, x in zip(design, plain, xeq): + name, _, rname, pos, xcig, mcig = d + assert (x.qname, x.rname, x.pos) == (name, rname, pos) + assert x.cigar == xcig + assert p.cigar == mcig + assert x.line.split('\t')[:5] + x.line.split('\t')[6:] == \ + p.line.split('\t')[:5] + p.line.split('\t')[6:] + + +# ------------------------------------------------------- --sam-opt-config -- + +@pytest.fixture(scope='module') +def datasets(genome_index, unp, pair, rep): + return { + 'unp': ['-x', genome_index, '-U', unp['fq'], '-p', '1'], + 'pair': ['-x', genome_index, '-1', pair['m1'], '-2', pair['m2'], '-p', '1'], + 'rep': ['-x', rep['index'], '-U', rep['fq'], '-p', '1'], + } + + +_default_cache = {} + + +def default_lines(bt2, args): + key = tuple(args) + if key not in _default_cache: + _default_cache[key] = body(bt2.run('bowtie2', args).stdout) + return _default_cache[key] + + +def same_core(a, b): + """SAM lines a and b agree in the 11 mandatory fields.""" + assert a.split('\t')[:11] == b.split('\t')[:11] + + +@pytest.mark.parametrize('config,removed,data', [ + ('-xn', ['XN'], 'unp'), ('-xm', ['XM'], 'unp'), ('-xo', ['XO'], 'unp'), + ('-xg', ['XG'], 'unp'), ('-nm', ['NM'], 'unp'), ('-md', ['MD'], 'unp'), + ('-yt', ['YT'], 'unp'), ('-yf', ['YF'], 'unp'), ('-ys', ['YS'], 'pair'), + ('-xs', ['XS'], 'rep'), ('-MD,-XM', ['MD', 'XM'], 'unp'), +]) +def test_sam_opt_config_turns_off(bt2, datasets, config, removed, data): + """--sam-opt-config - removes that default field and nothing else.""" + ref = default_lines(bt2, datasets[data]) + out = body(bt2.run('bowtie2', datasets[data] + ['--sam-opt-config', config]).stdout) + assert len(out) == len(ref) + assert any(t in removed for l in ref for t, _, _ in opt_fields(l)), 'tag never printed' + for a, b in zip(ref, out): + same_core(a, b) + assert opt_fields(b) == [f for f in opt_fields(a) if f[0] not in removed] + + +def check_yn(lines): + for l in lines: + f = opt_dict(l) + length = len(l.split('\t')[9]) + assert int(f['YN']) == int(-0.6 + -0.6 * length), l + assert f['Yn'] == '0', l + + +def check_present(*tags): + def check(lines): + for l in lines: + f = opt_fields(l) + for t in tags: + assert [x for x in f if x[0] == t], (t, l) + return check + + +def check_ints(*tags): + def check(lines): + for l in lines: + f = opt_dict(l) + for t in tags: + assert re.match(r'^-?\d+$', f[t]), (t, l) + return check + + +def aligned(lines): + return [l for l in lines if not int(l.split('\t')[1]) & h.UNMAPPED] + + +def check_ym(lines): + for l in aligned(lines): + assert opt_dict(l)['YM'] == '0', l + + +def check_zu(lines): + for l in aligned(lines): + f = opt_dict(l) + assert f['ZU'] == f['AS'] and 'Zu' in f, l + for l in lines: + if int(l.split('\t')[1]) & h.UNMAPPED: + assert 'ZU' not in opt_dict(l), l + + +def check_zt(lines): + for l in aligned(lines): + f = opt_dict(l) + assert f['ZT'].split(',')[0] == f['AS'], l + + +def check_xss(lines): + near = [l for l in lines if l.split('\t')[0] == 'near'] + assert len(near) == 1 + f = opt_dict(near[0]) + assert f['AS'] == '0' + assert f['Xs'] == '-36' # the copy with six Q40 mismatches, below --score-min + + +def mates(lines): + m1 = [l for l in lines if int(l.split('\t')[1]) & h.FIRST_IN_PAIR] + m2 = [l for l in lines if int(l.split('\t')[1]) & h.SECOND_IN_PAIR] + assert len(m1) == len(m2) == 1 + return m1[0], m2[0] + + +def check_mc(lines): + m1, m2 = mates(lines) + assert opt_dict(m1)['MC'] == m2.split('\t')[5] == '50M' + assert opt_dict(m2)['MC'] == m1.split('\t')[5] == '50M' + + +def check_mq(lines): + m1, m2 = mates(lines) + assert opt_dict(m1)['MQ'] == m2.split('\t')[4] + assert opt_dict(m2)['MQ'] == m1.split('\t')[4] + + +def check_yp(lines): + for l in mates(lines): + assert opt_dict(l)['YP'] == '0' + + +def check_zp(lines): + m1, m2 = mates(lines) + assert (opt_dict(m1)['AS'], opt_dict(m2)['AS']) == ('-6', '-12') + for l in (m1, m2): + assert opt_dict(l)['ZP'] == '-18' # best concordant score: sum of mates' AS + + +def check_on_all(*tags): + """Each tag is on every record (aligned or not) with an integer value.""" + def check(lines): + check_present(*tags)(lines) + check_ints(*tags)(lines) + return check + + +ADD_CASES = [ + ('yn', ['YN', 'Yn'], 'unp', check_yn), + ('zs', ['ZS'], 'unp', check_on_all('ZS')), + ('ym', ['YM'], 'unp', check_ym), + ('xt', ['XT'], 'unp', check_on_all('XT')), + ('xd', ['XD', 'Xd'], 'unp', check_on_all('XD', 'Xd')), + ('xu', ['XU', 'Xu'], 'unp', check_on_all('XU', 'Xu')), + ('ye', ['YE', 'Ye'], 'unp', check_on_all('YE', 'Ye')), + ('yl', ['YL', 'Yl'], 'unp', check_on_all('YL', 'Yl')), + ('yu', ['YU', 'Yu'], 'unp', check_on_all('YU', 'Yu')), + ('xp', ['XP'], 'unp', check_present('XP')), + ('yr', ['YR'], 'unp', check_on_all('YR')), + ('zb', ['ZB'], 'unp', check_on_all('ZB')), + ('zr', ['ZR'], 'unp', check_present('ZR')), + ('zf', ['ZF', 'Zf'], 'unp', check_on_all('ZF', 'Zf')), + ('zm', ['ZM'], 'unp', check_present('ZM')), + ('zi', ['ZI'], 'unp', check_on_all('ZI')), + ('zu', ['ZU', 'Zu'], 'unp', check_zu), + ('zt', ['ZT'], 'unp', check_zt), + ('xss', ['Xs', 'Ys'], 'rep', check_xss), + ('mc', ['MC'], 'pair', check_mc), + ('mq', ['MQ'], 'pair', check_mq), + ('yp', ['YP'], 'pair', check_yp), + ('zp', ['ZP', 'Zp'], 'pair', check_zp), + ('YN,Zs', ['YN', 'Yn', 'ZS'], 'unp', check_present('YN', 'Yn', 'ZS')), +] + + +@pytest.mark.parametrize('config,added,data,check', ADD_CASES, ids=[c[0] for c in ADD_CASES]) +def test_sam_opt_config_turns_on(bt2, datasets, config, added, data, check): + """--sam-opt-config adds that off-by-default field (values checked + where they follow from the design) and changes nothing else.""" + ref = default_lines(bt2, datasets[data]) + out = body(bt2.run('bowtie2', datasets[data] + ['--sam-opt-config', config]).stdout) + assert len(out) == len(ref) + assert not any(t in added for l in ref for t, _, _ in opt_fields(l)) + for a, b in zip(ref, out): + same_core(a, b) + assert [f for f in opt_fields(b) if f[0] not in added] == opt_fields(a) + check(out) + + +@pytest.mark.parametrize('config', ['x0', 'x1', 'yi', '-x0,-x1,-yi']) +def test_sam_opt_config_accepted_noop_tags(bt2, datasets, config): + """X0, X1 and YI are accepted but print nothing with the default MAPQ + model (pins the current output).""" + proc = bt2.run('bowtie2', datasets['unp'] + ['--sam-opt-config', config]) + assert 'not a valid SAM Optional flag' not in proc.stderr + assert body(proc.stdout) == default_lines(bt2, datasets['unp']) + + +def test_sam_opt_config_case_insensitive(bt2, datasets): + """Tag names in --sam-opt-config are case-insensitive.""" + upper = bt2.run('bowtie2', datasets['unp'] + ['--sam-opt-config', '-MD,ZS']).stdout + lower = bt2.run('bowtie2', datasets['unp'] + ['--sam-opt-config', '-md,zs']).stdout + assert body(upper) == body(lower) + assert all('MD' not in opt_dict(l) and 'ZS' in opt_dict(l) for l in body(upper)) + + +def test_sam_opt_config_invalid_tag(bt2, datasets): + """An unknown tag in --sam-opt-config is reported on stderr.""" + proc = bt2.run('bowtie2', datasets['unp'] + ['--sam-opt-config', 'qq'], check=False) + assert 'Error: qq is not a valid SAM Optional flag.' in proc.stderr + + +def test_sam_opt_config_xr_passthrough(bt2, datasets, unp): + """--sam-opt-config xr follows each record with the input record, %-escaped + (the line the wrapper reads for --un/--al).""" + out = bt2.run('bowtie2', datasets['unp'] + ['--sam-opt-config', 'xr', '--no-hd']).stdout + lines = out.splitlines() + assert len(lines) == 2 * len(unp['reads']) + for i, (name, seq, qual) in enumerate(unp['reads']): + assert lines[2 * i].split('\t')[0] == qname(name) + assert unescape_passthrough(lines[2 * i + 1]) == '@%s\n%s\n+\n%s\n' % (name, seq, qual) + + +# --------------------------------------------------------------- header -- + +def test_header_lines(bt2, genome, genome_index, unp): + """@HD, one @SQ per reference (name, length, index order) and @PG with + the program, version and command line; the @HD line (VN:1.5, SO, GO) + pins the current output.""" + header, _, _ = bt2.align(unp_args(genome_index, unp)) + assert header[0] == '@HD\tVN:1.5\tSO:unsorted\tGO:query' + assert header[1:4] == ['@SQ\tSN:%s\tLN:%d' % (n, len(s)) for n, s in genome] + assert len(header) == 5 and header[4].startswith('@PG\t') + pg = dict(f.split(':', 1) for f in header[4].split('\t')[1:]) + assert sorted(pg) == ['CL', 'ID', 'PN', 'VN'] + assert pg['ID'] == 'bowtie2' and pg['PN'] == 'bowtie2' + assert pg['VN'] == version_string(bt2) + cl = pg['CL'] + assert cl.startswith('"') and cl.endswith('"') + assert 'bowtie2-align-s' in cl + assert '-x %s' % genome_index in cl and unp['fq'] in cl and '-p 1' in cl + + +def test_sq_name_truncated_at_whitespace(bt2, rep): + """@SQ SN and RNAME use the FASTA name up to its first whitespace.""" + header, recs, _ = bt2.align(['-x', rep['index'], '-U', rep['fq'], '-p', '1']) + assert [l for l in header if l.startswith('@SQ')] == ['@SQ\tSN:rep\tLN:%d' % len(rep['seq'])] + assert set(r.rname for r in recs if not r.is_unmapped) == {'rep'} + near = h.by_name(recs)['near'][0] + assert near.pos == rep['near_pos'] + + +# -------------------------------------------------------------- summary -- + +def test_unpaired_summary(bt2, rep): + """The unpaired summary counts reads aligned 0, 1 and >1 times as the + MANUAL lays it out.""" + proc = bt2.run('bowtie2', ['-x', rep['index'], '-U', rep['fq'], '-p', '1']) + n, n0, n1, nm = 6, 1, 3, 2 # unal; uni_fw, uni_rc, near; rep_fw, rep_rc + expected = ( + '%d reads; of these:\n' + ' %d (%s) were unpaired; of these:\n' + ' %d (%s) aligned 0 times\n' + ' %d (%s) aligned exactly 1 time\n' + ' %d (%s) aligned >1 times\n' + '%s overall alignment rate\n') % (n, n, pct(n, n), n0, pct(n0, n), n1, pct(n1, n), + nm, pct(nm, n), pct(n1 + nm, n)) + assert proc.stderr == expected + + +def test_summary_zero_reads(bt2, genome_index, tmp_path): + """An empty read file gives a header-only SAM and a zero-read summary.""" + empty = tmp_path / 'empty.fq' + empty.write_text('') + proc = bt2.run('bowtie2', ['-x', genome_index, '-U', str(empty), '-p', '1']) + assert body(proc.stdout) == [] + assert proc.stderr.splitlines()[-2:] == ['0 reads', '0.00% overall alignment rate'] + + +# ---------------------------------------------------------- performance -- + +def test_threads_same_records(bt2, genome_index, many): + """-p 2, -p 4 and --threads 3 report the same records as -p 1 (as a + multiset); -p 1 keeps input order.""" + args = ['-x', genome_index, '-U', many['fq']] + ref = body(bt2.run('bowtie2', args + ['-p', '1']).stdout) + assert [l.split('\t')[0] for l in ref] == many['names'] + for p in (['-p', '2'], ['-p', '4'], ['--threads', '3']): + out = body(bt2.run('bowtie2', args + p).stdout) + assert sorted(out) == sorted(ref), p + + +def test_reorder(bt2, genome_index, many): + """--reorder with -p 4 gives exactly the -p 1 output, in input order; with + -p 1 it changes nothing.""" + args = ['-x', genome_index, '-U', many['fq']] + ref = body(bt2.run('bowtie2', args + ['-p', '1']).stdout) + assert body(bt2.run('bowtie2', args + ['-p', '4', '--reorder']).stdout) == ref + assert body(bt2.run('bowtie2', args + ['-p', '1', '--reorder']).stdout) == ref + + +def test_mm(bt2, genome_index, unp): + """--mm (memory-mapped index) gives the same alignments.""" + ref = bt2.run('bowtie2', unp_args(genome_index, unp)).stdout + assert body(bt2.run('bowtie2', unp_args(genome_index, unp) + ['--mm']).stdout) == body(ref) + + +def thread_counts(bt2, genome_index, runs, settled, timeout=30): + """Number of threads of bowtie2-align-s, waiting on stdin, for each + (extra args, OMP_NUM_THREADS) in runs, read from /proc. + + A process starts its threads one after another, so a count read too + early is too low, and a process held up on a loaded machine can sit at + such a count for a while. The counts are therefore taken once every + count has held for 0.5 s AND settled(counts) is true, and only if that + never happens are the last counts returned after `timeout` s. A slow + machine then delays the reading instead of failing the test, and a real + failure still fails, after the timeout.""" + procs = [] + try: + for args, omp in runs: + env = env_without_omp() if omp is None else env_without_omp(OMP_NUM_THREADS=omp) + procs.append(subprocess.Popen([align_bin(bt2), '--wrapper', 'basic-0', '-x', + genome_index, '-U', '-', '-S', os.devnull] + args, stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env)) + last = [None] * len(procs) + since = [time.time()] * len(procs) + deadline = time.time() + timeout + while True: + now = time.time() + for i, p in enumerate(procs): + assert p.poll() is None, 'bowtie2-align-s %s exited early' % (runs[i],) + n = len(os.listdir('/proc/%d/task' % p.pid)) + if n != last[i]: + last[i], since[i] = n, now + if all(now - s >= 0.5 for s in since) and settled(last): + return last + if now > deadline: + return last + time.sleep(0.01) + finally: + for p in procs: + p.stdin.close() + for p in procs: + p.wait(timeout=60) + + +@pytest.mark.skipif(not sys.platform.startswith('linux'), reason='counts threads via /proc') +def test_omp_num_threads_sets_default_thread_count(bt2, genome_index): + """OMP_NUM_THREADS sets the number of search threads when -p is absent, + and -p overrides it.""" + def settled(counts): + p1, p3, omp3, omp3_p1 = counts + return p3 - p1 == 2 and omp3 == p3 and omp3_p1 == p1 + + p1, p3, omp3, omp3_p1 = thread_counts(bt2, genome_index, [ + (['-p', '1'], None), (['-p', '3'], None), ([], '3'), (['-p', '1'], '3')], settled) + assert p3 - p1 == 2 + assert omp3 == p3 + assert omp3_p1 == p1 + + +def test_omp_num_threads_same_records(bt2, genome_index, many): + """With OMP_NUM_THREADS=3 and no -p the records equal those of -p 1.""" + args = ['-x', genome_index, '-U', many['fq']] + ref = body(bt2.run('bowtie2', args + ['-p', '1']).stdout) + out = body(bt2.run('bowtie2', args, env=env_without_omp(OMP_NUM_THREADS='3')).stdout) + assert sorted(out) == sorted(ref) + + +def test_omp_num_threads_not_a_number(bt2, genome_index, unp): + """A non-numeric OMP_NUM_THREADS is an error (no -p given).""" + proc = bt2.run('bowtie2', ['-x', genome_index, '-U', unp['fq']], check=False, + env=env_without_omp(OMP_NUM_THREADS='abc')) + assert proc.returncode != 0 + assert 'Error' in proc.stderr + assert body(proc.stdout) == [] + + +# ----------------------------------------------------- --version / --help -- + +def test_version(bt2): + """--version names the aligner binary and the version, and exits 0. + + The binary is bowtie2-align-s, or bowtie2-align-s-v256 where the CPU + dispatch selects the AVX2 build.""" + proc = bt2.run('bowtie2', ['--version']) + first = proc.stdout.splitlines()[0] + assert re.search(r'bowtie2-align-s(-v256)? version %s$' % re.escape(version_string(bt2)), first), first + + +@pytest.mark.parametrize('opt', ['-h', '--help']) +def test_help(bt2, opt): + """-h/--help prints the usage, listing the output and performance options, + to stdout and exits 0.""" + proc = bt2.run('bowtie2', [opt]) + out = proc.stdout + assert 'Usage:' in out + assert 'bowtie2 [options]* -x ' in out + for o in ('-t/--time', '--un ', '--al ', '--quiet', '--met-file ', + '--met-stderr', '--met ', '--no-unal', '--no-sq', '--rg-id ', + '--rg ', '--omit-sec-seq', '--xeq', '--sam-opt-config', '-p/--threads', + '--reorder', '--mm', '--version', '-h/--help'): + assert o in out, o + assert out == bt2.run('bowtie2', ['-h']).stdout + + +# --------------------------------------------------------------- errors -- + +def test_missing_index(bt2, unp, tmp_path): + """A -x that names no index fails with the wrapper's message.""" + prefix = str(tmp_path / 'nothing_here') + proc = bt2.run('bowtie2', ['-x', prefix, '-U', unp['fq']], check=False) + assert proc.returncode != 0 + assert '"%s" does not exist or is not a Bowtie 2 index' % prefix in proc.stderr + assert proc.stdout == '' + + +def test_missing_index_binary(bt2, unp, tmp_path): + """bowtie2-align-s itself also fails on a missing index.""" + prefix = str(tmp_path / 'nothing_here') + proc = bt2.run(align_bin(bt2), ['-x', prefix, '-U', unp['fq']], check=False) + assert proc.returncode == 1 + assert 'Could not locate a Bowtie index corresponding to basename "%s"' % prefix in proc.stderr + assert body(proc.stdout) == [] + + +def test_missing_read_file(bt2, genome_index, tmp_path): + """A read file that does not exist is an error when it is the only one.""" + missing = str(tmp_path / 'missing.fq') + proc = bt2.run('bowtie2', ['-x', genome_index, '-U', missing], check=False) + assert proc.returncode == 1 + assert 'Could not open read file "%s"' % missing in proc.stderr + assert 'No input read files were valid' in proc.stderr + assert body(proc.stdout) == [] + + +def test_missing_read_file_in_list_is_skipped(bt2, genome_index, unp, tmp_path): + """A missing file in a list is skipped with a warning and the others are + aligned (pins the current behaviour).""" + missing = str(tmp_path / 'missing.fq') + proc = bt2.run('bowtie2', ['-x', genome_index, '-U', missing + ',' + unp['fq'], '-p', '1']) + assert 'Could not open read file "%s"' % missing in proc.stderr + assert body(proc.stdout) == body(bt2.run('bowtie2', unp_args(genome_index, unp)).stdout) + + +def test_unknown_option(bt2, genome_index, unp): + """An unknown option prints the usage and exits 1.""" + proc = bt2.run('bowtie2', unp_args(genome_index, unp) + ['--no-such-option-xyz'], check=False) + assert proc.returncode == 1 + assert 'Usage:' in proc.stderr + assert 'bowtie2-align exited with value 1' in proc.stderr + assert proc.stdout == '' + + +BAD_VALUES = [ + (['-k', '0'], '-k arg must be at least 1'), + (['-k', '-1'], '-k arg must be at least 1'), + (['-p', '0'], '-p/--threads arg must be at least 1'), + (['--threads', '0'], '-p/--threads arg must be at least 1'), + (['--met', '0'], '--metrics arg must be at least 1'), + (['-u', '0'], '-u/--qupto arg must be at least 1'), + (['-s', '-1'], '-s arg must be positive'), + (['-3', '-1'], '-3/--trim3 arg must be at least 0'), + (['-5', '-1'], '-5/--trim5 arg must be at least 0'), + (['--trim-to', '4:10'], '--trim-to: trim position must be either 3 or 5'), + (['-L', '0'], '-L argument must be within the interval [1,32]; was 0'), + (['-L', '33'], '-L argument must be within the interval [1,32]; was 33'), + (['-N', '2'], '-N argument must be within the interval [0,1]; was 2'), + (['-i', 'X,1,2'], "Bad function type 'X'"), + (['--score-min', 'X,1,2'], "Bad function type 'X'"), + (['--score-min', 'foo'], "Bad function type 'foo'"), + (['--local', '--score-min', 'L,-1,0'], 'the match penalty is greater than 0 (2) but the ' + '--score-min function can be less than or equal to zero'), + (['--mp', '2,6'], 'Maximum mismatch penalty (2) is less than minimum penalty (6'), + (['--gbar', '0'], '--gbar must be no less than 1'), + (['--dpad', '-1'], '--dpad must be no less than 0'), + (['--seed', '-1'], '--seed arg must be at least 0'), + (['-I', '-1'], '-I arg must be positive'), + (['-X', '0'], '-X arg must be at least 1'), + (['--soft-clipped-unmapped-tlen'], '--soft-clipped-unmapped-tlen can only be set for local alignments'), + (['--preserve-tags'], 'can only be used when aligning BAM reads'), + (['--align-paired-reads'], '--align-paired-reads can only be used when aligning BAM reads'), +] + + +@pytest.mark.parametrize('args,message', BAD_VALUES, ids=[' '.join(a) for a, _ in BAD_VALUES]) +def test_bad_option_value(bt2, genome_index, unp, args, message): + """Invalid option values exit 1 with bowtie2's message and no records.""" + proc = bt2.run('bowtie2', unp_args(genome_index, unp) + args, check=False) + assert proc.returncode == 1 + assert message in proc.stderr + assert body(proc.stdout) == [] + + +def test_no_index_option(bt2, unp): + """Leaving out -x is an error.""" + proc = bt2.run('bowtie2', ['-U', unp['fq']], check=False) + assert proc.returncode == 1 + assert 'No index, query, or output file specified!' in proc.stderr + + +def test_no_reads_option(bt2, genome_index): + """Leaving out every read input is an error.""" + proc = bt2.run('bowtie2', ['-x', genome_index], check=False) + assert proc.returncode == 1 + assert 'Must specify at least one read input with -U/-1/-2' in proc.stderr + + +# -------------------------------------------------------------- wrapper -- + +def test_debug_binary(bt2, genome_index, unp): + """--debug runs bowtie2-align-s-debug (which announces debug mode) and + gives the same alignments.""" + if not bt2.has_binary('bowtie2-align-s-debug'): + pytest.skip('bowtie2-align-s-debug not built') + ref = bt2.run('bowtie2', unp_args(genome_index, unp)) + proc = bt2.run('bowtie2', unp_args(genome_index, unp) + ['--debug']) + assert 'Running in debug mode' in proc.stderr + assert 'Running in debug mode' not in ref.stderr + assert body(proc.stdout) == body(ref.stdout) + + +def test_sanitized_binary(bt2, genome_index, unp, tmp_path): + """--sanitized runs bowtie2-align-s-sanitized and gives the same alignments.""" + if not bt2.has_binary('bowtie2-align-s-sanitized'): + pytest.skip('bowtie2-align-s-sanitized not built') + ref = bt2.run('bowtie2', unp_args(genome_index, unp)) + out = str(tmp_path / 'o.sam') + proc = bt2.run('bowtie2', unp_args(genome_index, unp) + ['--sanitized', '--verbose', '-S', out]) + assert 'bowtie2-align-s-sanitized"' in proc.stderr + with open(out) as f: + assert body(f.read()) == body(ref.stdout) + + +def test_verbose(bt2, genome_index, unp, tmp_path): + """--verbose makes the wrapper print (INFO): lines, including the binary + command, without changing the alignments written with -S.""" + ref = bt2.run('bowtie2', unp_args(genome_index, unp)) + out = str(tmp_path / 'o.sam') + proc = bt2.run('bowtie2', unp_args(genome_index, unp) + ['--verbose', '-S', out]) + info = [l for l in proc.stderr.splitlines() if l.startswith('(INFO):')] + assert any('bowtie2-align-s" --wrapper basic-0' in l for l in info), proc.stderr + assert not any(l.startswith('(INFO):') for l in ref.stderr.splitlines()) + with open(out) as f: + assert body(f.read()) == body(ref.stdout) + + +def bz2_reads(unp, tmp_path): + """unp's reads as .bz2, which the wrapper decompresses with the bzip2 + tool into a named pipe or a temporary file; skips without bzip2.""" + if shutil.which('bzip2') is None: + pytest.skip('bzip2 not installed') + path = str(tmp_path / 'reads.fq.bz2') + h.write_fastq(path, unp['reads'], compress='bz2') + return path + + +@pytest.mark.skipif(not hasattr(os, 'mkfifo'), reason='needs named pipes') +def test_keep_named_pipe_in_temp_directory(bt2, genome_index, unp, tmp_path): + """For .bz2 input the wrapper decompresses through a named pipe in + --temp-directory; --keep leaves it there.""" + tmp = tmp_path / 'tmp' + tmp.mkdir() + fq = bz2_reads(unp, tmp_path) + proc = bt2.run('bowtie2', ['-x', genome_index, '-U', fq, '-p', '1', '--temp-directory', + str(tmp), '--keep']) + assert body(proc.stdout) == body(bt2.run('bowtie2', unp_args(genome_index, unp)).stdout) + left = os.listdir(str(tmp)) + assert len(left) == 1 and left[0].endswith('.unps'), left + assert stat.S_ISFIFO(os.stat(str(tmp / left[0])).st_mode) + + +def test_temp_directory_cleaned_without_keep(bt2, genome_index, unp, tmp_path): + """Without --keep the wrapper removes its temporary files.""" + tmp = tmp_path / 'tmp' + tmp.mkdir() + fq = bz2_reads(unp, tmp_path) + for extra in ([], ['--no-named-pipes']): + proc = bt2.run('bowtie2', ['-x', genome_index, '-U', fq, '-p', '1', '--temp-directory', + str(tmp)] + extra) + assert body(proc.stdout) == body(bt2.run('bowtie2', unp_args(genome_index, unp)).stdout) + assert os.listdir(str(tmp)) == [] + + +def test_no_named_pipes_keep(bt2, genome_index, unp, tmp_path): + """--no-named-pipes decompresses .bz2 input to a regular file, which + --keep leaves holding the decompressed reads.""" + tmp = tmp_path / 'tmp' + tmp.mkdir() + fq = bz2_reads(unp, tmp_path) + proc = bt2.run('bowtie2', ['-x', genome_index, '-U', fq, '-p', '1', '--temp-directory', + str(tmp), '--keep', '--no-named-pipes']) + assert body(proc.stdout) == body(bt2.run('bowtie2', unp_args(genome_index, unp)).stdout) + left = os.listdir(str(tmp)) + assert len(left) == 1 and left[0].endswith('.unps'), left + path = str(tmp / left[0]) + assert stat.S_ISREG(os.stat(path).st_mode) + with open(path) as f: + assert f.read() == h.fastq_text(unp['reads']) + + +def test_log_file(bt2, genome_index, unp, tmp_path): + """--log-file sends what would go to stderr (the summary) to the file.""" + log = str(tmp_path / 'log.txt') + ref = bt2.run('bowtie2', unp_args(genome_index, unp)) + proc = bt2.run('bowtie2', unp_args(genome_index, unp) + ['--log-file', log]) + assert proc.stderr == '' + with open(log) as f: + assert f.read() == ref.stderr + assert 'overall alignment rate' in ref.stderr + assert body(proc.stdout) == body(ref.stdout) + + +@pytest.fixture(scope='module') +def dual_index(bt2, genome_index_large, tmp_path_factory): + """A prefix holding a small index of a 1500-bp 'other' reference and the + large index of `genome`, so the @SQ lines show which one was used.""" + d = tmp_path_factory.mktemp('dual') + src_dir, src_base = os.path.split(genome_index_large) + for name in os.listdir(src_dir): + if name.startswith(src_base + '.') and name.endswith('.bt2l'): + shutil.copy(os.path.join(src_dir, name), str(d / ('dual' + name[len(src_base):]))) + other = h.write_fasta(str(d / 'other.fa'), [('other', h.random_dna(1500, random.Random(3)))]) + prefix = str(d / 'dual') + bt2.build_index(other, prefix, extra=['--seed', '0']) + assert os.path.exists(prefix + '.1.bt2') and os.path.exists(prefix + '.1.bt2l') + return prefix + + +def sq_names(text): + return [l.split('\t')[1][3:] for l in headers(text) if l.startswith('@SQ')] + + +def test_large_index_forces_bt2l(bt2, genome, dual_index, unp): + """With .bt2 and .bt2l at one prefix the wrapper uses the .bt2 unless + --large-index is given, which selects the .bt2l and bowtie2-align-l.""" + args = ['-x', dual_index, '-U', unp['fq'], '-p', '1'] + small = bt2.run('bowtie2', args) + assert sq_names(small.stdout) == ['other'] + large = bt2.run('bowtie2', args + ['--large-index']) + assert sq_names(large.stdout) == [n for n, _ in genome] + _, recs, _ = bt2.align(args + ['--large-index']) + for r in recs: + exp = unp['aligned'].get(r.qname) + if exp: + assert (r.rname, r.pos, r.is_reverse) == (exp['rname'], exp['pos'], exp['rev']) + + +def test_index_autodetect_bt2l(bt2, genome_index, genome_index_large, unp, tmp_path): + """A prefix with only a .bt2l index is used through bowtie2-align-l + without --large-index, with the same alignments as the .bt2 index.""" + out = str(tmp_path / 'o.sam') + proc = bt2.run('bowtie2', ['-x', genome_index_large, '-U', unp['fq'], '-p', '1', '--verbose', + '-S', out]) + assert 'bowtie2-align-l" --wrapper basic-0' in proc.stderr + with open(out) as f: + assert body(f.read()) == body(bt2.run('bowtie2', unp_args(genome_index, unp)).stdout) + small = bt2.run('bowtie2', unp_args(genome_index, unp) + ['--verbose', '-S', out]) + assert 'bowtie2-align-s" --wrapper basic-0' in small.stderr + + +def test_large_index_missing_bt2l(bt2, genome_index, unp): + """--large-index with only a .bt2 index fails.""" + proc = bt2.run('bowtie2', unp_args(genome_index, unp) + ['--large-index'], check=False) + assert proc.returncode != 0 + assert 'Cannot find the large index %s.1.bt2l' % genome_index in proc.stderr + + +def test_bowtie2_indexes_env(bt2, genome_index, unp, tmp_path): + """-x is looked up in $BOWTIE2_INDEXES when it is not in the + current directory.""" + d, name = os.path.split(genome_index) + env = env_without_omp(BOWTIE2_INDEXES=d) + proc = bt2.run('bowtie2', ['-x', name, '-U', unp['fq'], '-p', '1'], cwd=str(tmp_path), env=env) + assert body(proc.stdout) == body(bt2.run('bowtie2', unp_args(genome_index, unp)).stdout) + + +def test_ref_string(bt2, genome_dict, tmp_path): + """--ref-string aligns against a reference given on the command line + (named '1'); the wrapper needs bowtie2-build on PATH for it.""" + c1 = genome_dict['chr1'] + ref = c1[1000:1600] + fq = h.write_fastq(str(tmp_path / 'r.fq'), [('a', ref[100:150]), ('b', h.revcomp(ref[300:350]))]) + tmp = tmp_path / 'tmp' + tmp.mkdir() + env = env_without_omp(PATH=bt2.bin_dir + os.pathsep + os.environ.get('PATH', '')) + out = str(tmp_path / 'o.sam') + bt2.run('bowtie2', ['--ref-string', ref, '-U', fq, '-p', '1', '-S', out, + '--temp-directory', str(tmp)], env=env) + with open(out) as f: + header, recs = h.parse_sam(f.read()) + assert [l for l in header if l.startswith('@SQ')] == ['@SQ\tSN:1\tLN:600'] + assert [(r.qname, r.rname, r.pos, r.is_reverse) for r in recs] == \ + [('a', '1', 101, False), ('b', '1', 301, True)] + assert os.listdir(str(tmp)) == [] + + +def test_bam_output(bt2, genome_index, unp, tmp_path): + """--bam writes BAM (through samtools) holding the same records.""" + if shutil.which('samtools') is None: + pytest.skip('samtools not installed') + out = str(tmp_path / 'o.bam') + bt2.run('bowtie2', unp_args(genome_index, unp) + ['--bam', '-S', out]) + with open(out, 'rb') as f: + assert f.read(2) == b'\x1f\x8b' + view = subprocess.run(['samtools', 'view', out], capture_output=True, text=True, check=True) + assert body(view.stdout) == body(bt2.run('bowtie2', unp_args(genome_index, unp)).stdout) diff --git a/scripts/test/test_paired.py b/scripts/test/test_paired.py new file mode 100644 index 00000000..b53971a9 --- /dev/null +++ b/scripts/test/test_paired.py @@ -0,0 +1,1056 @@ +"""Paired-end alignment tests for bowtie2. + +Every pair is cut from the fixed random `genome` (conftest.py) at a known +position, orientation and fragment length, so the expected FLAG, POS, RNEXT, +PNEXT, TLEN and YT:Z of both mates are derived here before bowtie2 runs. The +same applies to the classification of overlapping, containing and dovetailing +mates (the MANUAL's own examples), the fragment-length bounds -I/-X, mixed +and discordant reporting, the reads written by --un-conc/--al-conc, and the +counts in the paired alignment summary. + +Run from the repository root after `make`: + + python3 -m pytest scripts/test/test_paired.py +""" + +import bz2 +import gzip +import os +import random +import shutil +import struct +import subprocess +import tempfile +import zlib + +import pytest + +import bt2_helpers as h +from bt2_helpers import (PAIRED, PROPER_PAIR, UNMAPPED, MATE_UNMAPPED, REVERSE, + MATE_REVERSE, FIRST_IN_PAIR, SECOND_IN_PAIR) + + +# --------------------------------------------------------------------------- +# Helpers (candidates for bt2_helpers.py) +# --------------------------------------------------------------------------- + +class Fragment(object): + """Mates cut from both ends of ref[start:start + length]. + + `orient` is the library orientation ('fr', 'rf' or 'ff', as in the + MANUAL's --fr/--rf/--ff) and `strand` the strand the fragment came from + ('+' Watson, '-' Crick). For a Watson fragment mate 1 is upstream: with + 'fr' it reads forward and mate 2 is the reverse complement of the + downstream end; 'rf' swaps the strands of the two mates; 'ff' reads both + forward. A Crick fragment is the reverse complement of the Watson one, cut + the same way. `pos`, `rev` and `tlen` hold the expected SAM POS (1-based), + reverse-strand bit and TLEN of mate 1 and mate 2 when the pair aligns. + """ + + def __init__(self, ref, start, length, len1, len2=None, orient='fr', + strand='+', chrom='chr1'): + len2 = len1 if len2 is None else len2 + frag = ref[start:start + length] + flip = strand == '-' + if flip: + frag = h.revcomp(frag) + left, right = frag[:len1], frag[length - len2:] + if orient == 'fr': + self.mate1, self.mate2 = left, h.revcomp(right) + elif orient == 'rf': + self.mate1, self.mate2 = h.revcomp(left), right + elif orient == 'ff': + self.mate1, self.mate2 = left, right + else: + raise ValueError(orient) + self.chrom = chrom + self.length = length + self.rev = ((orient == 'rf') != flip, (orient == 'fr') != flip) + if flip: + self.pos = (start + length - len1 + 1, start + 1) + self.tlen = (-length, length) + else: + self.pos = (start + 1, start + length - len2 + 1) + self.tlen = (length, -length) + + def pair(self, name): + return (name, self.mate1, self.mate2) + + +def pair_flags(rev1, rev2, proper): + """Expected FLAGs of mate 1 and mate 2 when both mates align.""" + base = PAIRED | (PROPER_PAIR if proper else 0) + f1 = base | FIRST_IN_PAIR | (REVERSE if rev1 else 0) | (MATE_REVERSE if rev2 else 0) + f2 = base | SECOND_IN_PAIR | (REVERSE if rev2 else 0) | (MATE_REVERSE if rev1 else 0) + return f1, f2 + + +def junk(length, seed): + """Random sequence that aligns nowhere in the 10 kbp test genome.""" + return h.random_dna(length, random.Random(seed)) + + +def mismatching(seq): + """A sequence differing from `seq` at every position (soft-clip filler).""" + return h.mutate(seq, range(len(seq))) + + +def write_pair_files(directory, pairs): + """Write (name, mate1, mate2[, qual1, qual2]) pairs as two FASTQ files.""" + r1, r2 = [], [] + for p in pairs: + q1 = p[3] if len(p) > 3 else None + q2 = p[4] if len(p) > 4 else None + r1.append((p[0], p[1], q1)) + r2.append((p[0], p[2], q2)) + m1 = h.write_fastq(os.path.join(directory, 'reads_1.fq'), r1) + m2 = h.write_fastq(os.path.join(directory, 'reads_2.fq'), r2) + return m1, m2 + + +def new_dir(tmp_path): + return tempfile.mkdtemp(dir=str(tmp_path)) + + +def align_pairs(bt2, index, tmp_path, pairs, args=(), check=True): + """Align pairs given with -1/-2 at -p 1; returns (records, proc).""" + m1, m2 = write_pair_files(new_dir(tmp_path), pairs) + _, records, proc = bt2.align(['-x', index, '-p', '1', '-1', m1, '-2', m2] + + list(args), check=check) + return records, proc + + +def mates_by_name(records): + """qname -> [mate 1 record, mate 2 record], primary records only.""" + out = {} + for r in h.primary(records): + assert r.flag & PAIRED, r + slot = 0 if r.flag & FIRST_IN_PAIR else 1 + assert out.setdefault(r.qname, [None, None])[slot] is None, r + out[r.qname][slot] = r + for name, (r1, r2) in out.items(): + assert r1 is not None and r2 is not None, name + return out + + +def check_aligned_pair(r1, r2, frag, proper, yt): + """Both mates aligned where `frag` says, with mate fields filled in.""" + f1, f2 = pair_flags(frag.rev[0], frag.rev[1], proper) + assert (r1.flag, r2.flag) == (f1, f2) + assert (r1.rname, r2.rname) == (frag.chrom, frag.chrom) + assert (r1.pos, r2.pos) == frag.pos + assert (r1.rnext, r2.rnext) == ('=', '=') + assert (r1.pnext, r2.pnext) == (frag.pos[1], frag.pos[0]) + assert (r1.tags['YT'], r2.tags['YT']) == (yt, yt) + + +def summary_block(stderr): + """The alignment summary lines of stderr, from 'N reads' to the rate.""" + lines = stderr.splitlines() + start = [i for i, l in enumerate(lines) if l.endswith(' reads; of these:')] + end = [i for i, l in enumerate(lines) if l.endswith(' overall alignment rate')] + assert start and end, stderr + return lines[start[-1]:end[-1] + 1] + + +def paired_summary(npairs, conc, disc, mates, discordant=True, mixed=True): + """Expected paired alignment summary, in the MANUAL's format. + + conc = pairs aligning concordantly (0 times, exactly 1 time, >1 times); + disc = pairs aligning discordantly; mates = the mates of the remaining + pairs aligning (0 times, exactly 1 time, >1 times). + """ + def pct(n, d): + return '%.2f%%' % (100.0 * n / d if d else 0.0) + lines = [ + '%d reads; of these:' % npairs, + ' %d (%s) were paired; of these:' % (npairs, pct(npairs, npairs)), + ' %d (%s) aligned concordantly 0 times' % (conc[0], pct(conc[0], npairs)), + ' %d (%s) aligned concordantly exactly 1 time' % (conc[1], pct(conc[1], npairs)), + ' %d (%s) aligned concordantly >1 times' % (conc[2], pct(conc[2], npairs))] + if discordant: + lines += [ + ' ----', + ' %d pairs aligned concordantly 0 times; of these:' % conc[0], + ' %d (%s) aligned discordantly 1 time' % (disc, pct(disc, conc[0]))] + if mixed: + rest = conc[0] - disc + lines += [ + ' ----', + ' %d pairs aligned 0 times concordantly or discordantly; of these:' % rest, + ' %d mates make up the pairs; of these:' % (2 * rest), + ' %d (%s) aligned 0 times' % (mates[0], pct(mates[0], 2 * rest)), + ' %d (%s) aligned exactly 1 time' % (mates[1], pct(mates[1], 2 * rest)), + ' %d (%s) aligned >1 times' % (mates[2], pct(mates[2], 2 * rest))] + unpaired = mates[1] + mates[2] if mixed else 0 + aligned = 2 * (conc[1] + conc[2]) + 2 * disc + unpaired + lines.append('%s overall alignment rate' % pct(aligned, 2 * npairs)) + return lines + + +def _bgzf_block(data): + comp = zlib.compressobj(6, zlib.DEFLATED, -15) + cdata = comp.compress(data) + comp.flush() + header = struct.pack('<4BI2BH', 31, 139, 8, 4, 0, 0, 255, 6) + b'BC' + \ + struct.pack('/--un-conc- write compressed per-mate files (out.fq. -> out.fq.1., out.fq.2.) holding the right reads.""" + tool = {'gz': 'gzip', 'bz2': 'bzip2', 'zst': 'zstd', 'lz4': 'lz4'}[comp] + if not shutil.which(tool): + pytest.skip('%s not installed (the wrapper compresses through it)' % tool) + d = new_dir(tmp_path) + m1, m2, text, classes = conc_split_input(genome_dict, d) + out = os.path.join(d, 'out') + os.mkdir(out) + run_with_read_files(bt2, genome_index, d, m1, m2, + ['--%s-conc-%s' % (kind, comp), os.path.join(out, 'out.fq.' + comp)]) + files = ['out.fq.1.' + comp, 'out.fq.2.' + comp] + assert sorted(os.listdir(out)) == files + for mate, fn in zip((1, 2), files): + path = os.path.join(out, fn) + with open(path, 'rb') as f: + assert f.read(len(MAGIC[comp])) == MAGIC[comp] + got = fastq_records(read_compressed(path, comp)) + assert sorted(got) == sorted(text[mate][i] for i in classes[kind]) + + +def test_conc_compressed_percent_pattern(bt2, genome_dict, genome_index, tmp_path): + """The % pattern also applies to compressed output: un_%.fq.gz -> un_1.fq.gz, un_2.fq.gz.""" + if not shutil.which('gzip'): + pytest.skip('gzip not installed (the wrapper compresses through it)') + d = new_dir(tmp_path) + m1, m2, text, classes = conc_split_input(genome_dict, d) + out = os.path.join(d, 'out') + os.mkdir(out) + run_with_read_files(bt2, genome_index, d, m1, m2, + ['--un-conc-gz', os.path.join(out, 'un_%.fq.gz')]) + assert sorted(os.listdir(out)) == ['un_1.fq.gz', 'un_2.fq.gz'] + for mate in (1, 2): + got = fastq_records(read_compressed(os.path.join(out, 'un_%d.fq.gz' % mate), 'gz')) + assert sorted(got) == sorted(text[mate][i] for i in classes['un']) + + +def test_al_conc_writes_repeat_pair_once(bt2, genome_dict, repeat_index, tmp_path): + """With -k 3 a pair with two concordant placements has secondary records in SAM but appears once in --al-conc.""" + c2 = genome_dict['chr2'] + frag = Fragment(c2, 1020, 330, 50, chrom='chr2') + d = new_dir(tmp_path) + m1, m2 = write_pair_files(d, [frag.pair('rep')]) + records, _ = run_with_read_files(bt2, repeat_index, d, m1, m2, + ['-k', '3', '--al-conc', os.path.join(d, 'al.fq')]) + assert len(records) == 4 + assert sum(1 for r in records if r.is_secondary) == 2 + for mate, seq in ((1, frag.mate1), (2, frag.mate2)): + got = h.read_fastq(os.path.join(d, 'al.%d.fq' % mate)) + assert got == [('rep', seq, 'I' * len(seq))] + + +def test_un_mates(bt2, genome_dict, genome_index, tmp_path): + """--un-mates (wrapper option, not in the MANUAL) writes each unaligned mate of a pair to its mate's file; pins the current wrapper behaviour.""" + d = new_dir(tmp_path) + m1, m2, text, _ = conc_split_input(genome_dict, d) + out = os.path.join(d, 'out') + os.mkdir(out) + run_with_read_files(bt2, genome_index, d, m1, m2, ['--un-mates', os.path.join(out, 'um.fq')]) + assert sorted(os.listdir(out)) == ['um.1.fq', 'um.2.fq'] + # Unaligned: mate 2 of m1only (index 1) and both mates of none (index 2). + with open(os.path.join(out, 'um.1.fq')) as f: + assert fastq_records(f.read()) == [text[1][2]] + with open(os.path.join(out, 'um.2.fq')) as f: + assert sorted(fastq_records(f.read())) == sorted([text[2][1], text[2][2]]) + + +# --------------------------------------------------------------------------- +# Paired alignment summary +# --------------------------------------------------------------------------- + +def summary_pairs(genome_dict, with_discordant=True): + """Pairs of known class against `repeat_index`: 3 concordant once, 2 + concordant twice (inside the duplicated region), 2 discordant (other + chromosome; FF orientation), and 3 that are neither: unique mate + junk, + junk + junk, repeat mate + unique mate on another chromosome.""" + c1, c2, c3 = genome_dict['chr1'], genome_dict['chr2'], genome_dict['chr3'] + pairs = [ + Fragment(c1, 100, 300, 50).pair('conc1a'), + Fragment(c1, 1100, 250, 50).pair('conc1b'), + Fragment(c3, 500, 250, 50).pair('conc1c'), + Fragment(c2, 1020, 330, 50).pair('conc2a'), + Fragment(c2, 1050, 250, 50, strand='-').pair('conc2b'), + ] + if with_discordant: + pairs += [ + ('disc_chr', c1[2000:2050], h.revcomp(c3[1500:1550])), + Fragment(c1, 3000, 250, 50, orient='ff').pair('disc_ff'), + ] + pairs += [ + ('one_mate', c1[4000:4050], junk(50, 6)), + ('no_mate', junk(50, 7), junk(50, 8)), + ('rep_mate', c2[1100:1150], h.revcomp(c3[100:150])), + ] + return pairs + + +@pytest.mark.parametrize('args,with_disc,expect', [ + ([], True, dict(conc=(5, 3, 2), disc=2, mates=(3, 2, 1))), + (['--no-discordant'], True, dict(conc=(5, 3, 2), disc=0, mates=(3, 6, 1), discordant=False)), + (['--no-mixed'], False, dict(conc=(3, 3, 2), disc=0, mates=None, mixed=False)), + (['--no-mixed', '--no-discordant'], True, + dict(conc=(5, 3, 2), disc=0, mates=None, mixed=False, discordant=False)), +]) +def test_paired_summary(bt2, genome_dict, repeat_index, tmp_path, args, with_disc, expect): + """The paired alignment summary counts concordant 0/1/>1, discordant, and mates 0/1/>1 times as the input was built; --no-discordant/--no-mixed drop their sections. + + The MANUAL shows only the full layout, so the dropped sections pin the + current output. --no-mixed is run without the discordant pairs, which the + current version wrongly suppresses. + """ + pairs = summary_pairs(genome_dict, with_disc) + records, proc = align_pairs(bt2, repeat_index, tmp_path, pairs, args) + expected = paired_summary(len(pairs), **expect) + assert summary_block(proc.stderr) == expected + yt = dict((name, r1.tags['YT']) for name, (r1, r2) in mates_by_name(records).items()) + assert sorted(n for n in yt if yt[n] == 'CP') == ['conc1a', 'conc1b', 'conc1c', 'conc2a', 'conc2b'] + assert sorted(n for n in yt if yt[n] == 'DP') == ( + ['disc_chr', 'disc_ff'] if with_disc and expect['disc'] else []) + + +def test_paired_summary_manual_example_layout(bt2, genome_dict, genome_index, tmp_path): + """The summary for an all-concordant input follows the MANUAL's layout line by line.""" + c = genome_dict['chr1'] + pairs = [Fragment(c, s, 300, 50).pair('p%d' % s) for s in (200, 1200, 2200, 3200)] + _, proc = align_pairs(bt2, genome_index, tmp_path, pairs) + assert summary_block(proc.stderr) == [ + '4 reads; of these:', + ' 4 (100.00%) were paired; of these:', + ' 0 (0.00%) aligned concordantly 0 times', + ' 4 (100.00%) aligned concordantly exactly 1 time', + ' 0 (0.00%) aligned concordantly >1 times', + ' ----', + ' 0 pairs aligned concordantly 0 times; of these:', + ' 0 (0.00%) aligned discordantly 1 time', + ' ----', + ' 0 pairs aligned 0 times concordantly or discordantly; of these:', + ' 0 mates make up the pairs; of these:', + ' 0 (0.00%) aligned 0 times', + ' 0 (0.00%) aligned exactly 1 time', + ' 0 (0.00%) aligned >1 times', + '100.00% overall alignment rate'] + + +# --------------------------------------------------------------------------- +# Unequal mate counts +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize('short_mate', [1, 2]) +def test_unequal_mate_counts(bt2, genome_dict, genome_index, tmp_path, short_mate): + """A -1/-2 file pair with different read counts is an error naming the shorter file.""" + c = genome_dict['chr1'] + frags = [Fragment(c, 300 * i, 250, 40) for i in range(1, 6)] + d = new_dir(tmp_path) + r1 = [('p%d' % i, f.mate1) for i, f in enumerate(frags)] + r2 = [('p%d' % i, f.mate2) for i, f in enumerate(frags)] + if short_mate == 1: + r1 = r1[:3] + else: + r2 = r2[:3] + m1 = h.write_fastq(os.path.join(d, 'r1.fq'), r1) + m2 = h.write_fastq(os.path.join(d, 'r2.fq'), r2) + proc = bt2.run('bowtie2', ['-x', genome_index, '-p', '1', '-1', m1, '-2', m2, + '-S', os.path.join(d, 'out.sam')], check=False, cwd=d) + assert proc.returncode != 0 + other = 3 - short_mate + assert ('Error, fewer reads in file specified with -%d than in file specified with -%d' + % (short_mate, other)) in proc.stderr + + +# --------------------------------------------------------------------------- +# Same pairing from --interleaved, -b --align-paired-reads, and file lists +# --------------------------------------------------------------------------- + +def input_pairs(genome_dict): + c1, c2 = genome_dict['chr1'], genome_dict['chr2'] + return [ + Fragment(c1, 1000, 300, 50).pair('conc'), + Fragment(c1, 2000, 300, 50, 40, strand='-').pair('crick'), + ('m1only', c1[1500:1550], junk(50, 9)), + ('disc', c1[2600:2650], h.revcomp(c2[500:550])), + ('none', junk(45, 10), junk(45, 11)), + ] + + +def test_interleaved_and_bam_pair_like_separate_files(bt2, genome_dict, genome_index, tmp_path): + """--interleaved FASTQ and -b --align-paired-reads give the same SAM records as -1/-2 for the same pairs.""" + pairs = input_pairs(genome_dict) + d = new_dir(tmp_path) + m1, m2 = write_pair_files(d, pairs) + il = h.write_fastq(os.path.join(d, 'il.fq'), + [r for name, s1, s2 in pairs for r in ((name, s1), (name, s2))]) + bam_records = [] + for name, s1, s2 in pairs: + bam_records.append((name, PAIRED | UNMAPPED | MATE_UNMAPPED | FIRST_IN_PAIR, s1, 'I' * len(s1))) + bam_records.append((name, PAIRED | UNMAPPED | MATE_UNMAPPED | SECOND_IN_PAIR, s2, 'I' * len(s2))) + bam = write_unaligned_bam(os.path.join(d, 'reads.bam'), bam_records) + base = ['-x', genome_index, '-p', '1'] + _, sep, _ = bt2.align(base + ['-1', m1, '-2', m2]) + _, inter, _ = bt2.align(base + ['--interleaved', il]) + _, frombam, _ = bt2.align(base + ['-b', bam, '--align-paired-reads']) + assert [r.tags['YT'] for r in sep] == ['CP'] * 4 + ['UP'] * 2 + ['DP'] * 2 + ['UP'] * 2 + assert [r.line for r in inter] == [r.line for r in sep] + assert [r.line for r in frombam] == [r.line for r in sep] + + +def test_mate_file_lists_pair_in_order(bt2, genome_dict, genome_index, tmp_path): + """-1 a,b -2 c,d pairs a with c and b with d: same SAM records as one file per mate.""" + pairs = input_pairs(genome_dict) + d1, d2, d3 = new_dir(tmp_path), new_dir(tmp_path), new_dir(tmp_path) + m1, m2 = write_pair_files(d1, pairs) + a1, a2 = write_pair_files(d2, pairs[:2]) + b1, b2 = write_pair_files(d3, pairs[2:]) + base = ['-x', genome_index, '-p', '1'] + _, single, _ = bt2.align(base + ['-1', m1, '-2', m2]) + _, listed, _ = bt2.align(base + ['-1', a1 + ',' + b1, '-2', a2 + ',' + b2]) + assert [r.qname for r in single] == [p[0] for p in pairs for _ in (1, 2)] + assert [r.line for r in listed] == [r.line for r in single] From 9aa3d6161a6eb9e1b86f338696821c1833a76c8c Mon Sep 17 00:00:00 2001 From: Jacob Schreiber Date: Fri, 16 Oct 2026 12:27:01 -0400 Subject: [PATCH 2/3] Speed up alignment about 3x with byte-identical output bowtie2-align is 1.7x to 4.3x faster on every input timed (end-to-end and local, single- and paired-end, -p 1 to -p 16), except a 12-read input at -p 1, where index loading dominates. Its output is byte for byte the same as before. Peak memory is unchanged, except that with --mm a small run touches more of the mapped index. Dynamic programming (aligner_swsse_ee_u8.cpp, aligner_swsse_loc_u8.cpp, aligner_swsse.cpp, aligner_sw*.cpp): - The 8-bit end-to-end fill computes the vertical-gap term with an exact prefix scan across lanes instead of the lazy-F loop. Both 8-bit fills have a shorter dependency chain from one column to the next. Reads of up to 128 rows are filled as two interleaved chains. - The query profile is built by table lookup. The 8-bit backtraces clear only the intervals they wrote, not whole matrices. - Local mode gathers its candidate cells with vector instructions, sorts them by counting, tests domination vectorized, and backtraces without a frame stack. FM index and seeds (aligner_seed.cpp, group_walk.h, bt2_idx.*): - Occurrence counts in every LF step are branch-free, and the large index arrays are 64-byte aligned. - Exact and one-mismatch seed searches are batched across the reads of an input batch and across both strands. Independent walks are stepped in turn, with each walk's next BWT side prefetched, so memory waits overlap. - Seed instantiation is redone only for inputs that changed. Seed-hit ranking and sampling no longer scan lists linearly or sort per call. - SA resolution runs single-row walks ahead of their use, and walks large BW ranges a 64-bit BWT word at a time. Sampled SA elements are built on first use. Every random draw is unchanged. - joinedToTextOff uses a bucket table built at index load instead of a binary search over the reference fragments. Elsewhere: RedundantAlns keeps per-read runs along diagonals, BtBranchTracer resets lazily, reference windows are unpacked straight to masks, and the output queue and SAM writer do less work per record. Index loading (index_load.h): at -p > 1, the four large arrays (.4 file, forward and mirror BWT, SA sample) are read in 64 MB pieces by up to four helper threads and the main thread. Read errors print the same messages as the serial load. The load stays serial at -p 1, and with --mm, --shmem, --verbose, -t or --sanity. AVX2 dispatch (bowtie_main.cpp, Makefile): the dispatch tested AVX2 in CPUID leaf 1, so bowtie2-align-*-v256 ran only on CPUs where an unrelated bit was set. It now requires every x86-64-v3 feature (CPUID leaves 1, 7 and 0x80000001) and OS support for the AVX registers (XCR0). The -v256 build compiles with -ffp-contract=off, so its floating-point results, --score-min and MAPQ, are rounded as in the 128-bit build. No OS-specific calls are added. Co-Authored-By: Claude Opus 5.5 --- Makefile | 4 +- aligner_bt.h | 33 +- aligner_cache.cpp | 13 +- aligner_result.cpp | 250 ++++-- aligner_result.h | 66 +- aligner_seed.cpp | 1391 +++++++++++++++++++++++++-------- aligner_seed.h | 502 ++++++++++-- aligner_sw.cpp | 180 ++++- aligner_sw.h | 49 ++ aligner_sw_driver.cpp | 531 ++++++++----- aligner_sw_driver.h | 207 ++++- aligner_swsse.cpp | 57 ++ aligner_swsse.h | 73 +- aligner_swsse_ee_u8.cpp | 1585 ++++++++++++++++++++++++++++++-------- aligner_swsse_loc_u8.cpp | 1115 ++++++++++++++++++--------- aln_sink.cpp | 34 +- bowtie_main.cpp | 87 ++- bt2_idx.cpp | 190 +++-- bt2_idx.h | 160 +++- bt2_io.cpp | 30 +- bt2_search.cpp | 222 +++++- ds.h | 86 ++- group_walk.h | 953 ++++++++++++++++++++++- index_load.h | 398 ++++++++++ mm.h | 24 + outq.cpp | 32 +- outq.h | 95 ++- pat.cpp | 70 +- pat.h | 50 +- random_util.h | 98 ++- read.h | 33 +- reference.cpp | 224 +++++- reference.h | 23 +- sam.cpp | 334 ++++---- scoring.h | 56 ++ sse_wrap.h | 3 + sstring.h | 29 + 37 files changed, 7472 insertions(+), 1815 deletions(-) create mode 100644 index_load.h diff --git a/Makefile b/Makefile index 53534610..1cc258b4 100644 --- a/Makefile +++ b/Makefile @@ -85,7 +85,9 @@ ifneq (,$(findstring $(ARCH), x86_64 amd64)) POPCNT_CAPABILITY ?= 1 ifneq (0, $(SSE_AVX2)) ENABLE_V256 := 1 - V256_FLAG := -march=x86-64-v3 -mtune=znver3 -faligned-new -DSSE_AVX2 -DPOPCNT_CAPABILITY + # -ffp-contract=off: no fused multiply-adds, so floating-point results (e.g. + # --score-min, MAPQ) are rounded as in the 128-bit build and outputs match it + V256_FLAG := -march=x86-64-v3 -mtune=znver3 -ffp-contract=off -faligned-new -DSSE_AVX2 -DPOPCNT_CAPABILITY # tell 128-bit version to check for v256 capability and use it, if found SSE_FLAG += -DENABLE_x86_64_v3 endif diff --git a/aligner_bt.h b/aligner_bt.h index 8056b7ab..e35c699e 100644 --- a/aligner_bt.h +++ b/aligner_bt.h @@ -546,7 +546,8 @@ class BtBranchTracer { public: explicit BtBranchTracer() : - prob_(), bs_(), seenPaths_(DP_CAT), sawcell_(DP_CAT), doTri_() { } + prob_(), bs_(), seenPaths_(DP_CAT), sawcell_(DP_CAT), doTri_(), + resetPending_(false), resetNdiag_(0), resetRflen_(0) { } /** * Add a branch to the queue. @@ -642,7 +643,10 @@ class BtBranchTracer { /** * Initialize the tracer with respect to a new read. This involves - * resetting all the state relating to the set of cells already visited + * resetting all the state relating to the set of cells already visited. + * The reset itself is deferred to the next initBt(), since most + * reference windows are never backtraced with this tracer; + * seenPaths_ and sawcell_ are read only from initBt() on. */ void initRef( const char* rd, // in: read sequence @@ -660,7 +664,22 @@ class BtBranchTracer { size_t nceil) // in: N ceiling { prob_.initRef(rd, qu, rdlen, rf, rflen, trflen, refid, refoff, fw, rect, cper, &sc, nceil); - const size_t ndiag = rflen + rdlen - 1; + resetNdiag_ = rflen + rdlen - 1; + // An eager reset clears every per-column set below the largest rflen + // seen since the last backtrace, so keep the largest. + if(!resetPending_ || rflen > resetRflen_) { + resetRflen_ = rflen; + } + resetPending_ = true; + } + + /** + * Perform the reset of seenPaths_ and sawcell_ that the last initRef() + * (and any initRef() calls before it since the last initBt()) deferred. + */ + void resetSeen() { + const size_t ndiag = resetNdiag_; + const size_t rflen = resetRflen_; seenPaths_.resize(ndiag); for(size_t i = 0; i < ndiag; i++) { seenPaths_[i].clear(); @@ -677,6 +696,7 @@ class BtBranchTracer { sawcell_[i].setCat(DP_CAT); sawcell_[i].clear(); // clear the set } + resetPending_ = false; } /** @@ -691,6 +711,9 @@ class BtBranchTracer { bool doTri, // in: triangle-shaped mini-fills? RandomSource& rnd) // in: random gen, to choose among equal paths { + if(resetPending_) { + resetSeen(); + } prob_.initBt(row, col, fill, usecp, escore); Edit e; e.reset(); unsorted_.clear(); @@ -939,6 +962,10 @@ class BtBranchTracer { ELList tri_; // triangle to fill when doing mini-fills EList ndep_; // # triangles mini-filled at various depths + bool resetPending_; // seenPaths_/sawcell_ reset owed by initRef + size_t resetNdiag_; // # diagonals of the last initRef's window + size_t resetRflen_; // largest rflen since the last reset + #ifndef NDEBUG ESet seen_; // seedn branch ids; should never see same twice #endif diff --git a/aligner_cache.cpp b/aligner_cache.cpp index 06d23dc0..0fe766e7 100644 --- a/aligner_cache.cpp +++ b/aligner_cache.cpp @@ -88,12 +88,13 @@ bool AlignmentCache::addOnTheFlyImpl( s->payload.len = botf - topf; s->payload.topf = topf; s->payload.topb = topb; - for(size_t j = 0; j < (botf-topf); j++) { - if(!salist_.add(pool(), OFF_MASK)) { - // Change the payload's len field - s->payload.len = (TIndexOffU)j; - return false; // Exhausted pool memory - } + // One OFF_MASK (offset not yet resolved) per element of the range + const size_t len = botf - topf; + const size_t j = salist_.addFillUpTo(pool(), len, OFF_MASK); + if(j < len) { + // Change the payload's len field + s->payload.len = (TIndexOffU)j; + return false; // Exhausted pool memory } assert(s->payload.repOk(*this)); } diff --git a/aligner_result.cpp b/aligner_result.cpp index 6697b97c..17f836c5 100644 --- a/aligner_result.cpp +++ b/aligner_result.cpp @@ -774,14 +774,15 @@ void StackedAln::writeCigar( assert_eq(op.size(), run.size()); if(o != NULL || occ != NULL) { char buf[128]; + size_t nbuf = 0; // length of the number itoa10 last wrote into buf ASSERT_ONLY(bool printed = false); for(size_t i = 0; i < op.size(); i++) { size_t r = run[i]; if(r > 0) { - itoa10(r, buf); + nbuf = (size_t)(itoa10(r, buf) - buf); ASSERT_ONLY(printed = true); if(o != NULL) { - o->append(buf); + o->append(buf, nbuf); o->append(op[i]); } if(occ != NULL) { @@ -804,6 +805,7 @@ void StackedAln::writeCigar( */ void StackedAln::writeMdz(BTString* o, char* occ) const { char buf[128]; + size_t nbuf = 0; // length of the number itoa10 last wrote into buf bool mm_last = false; bool rdgap_last = false; bool first_print = true; @@ -815,8 +817,8 @@ void StackedAln::writeMdz(BTString* o, char* occ) const { if(r > 0) { if(op[i] == '=') { // Write run length - itoa10(r, buf); - if(o != NULL) { o->append(buf); } + nbuf = (size_t)(itoa10(r, buf) - buf); + if(o != NULL) { o->append(buf, nbuf); } if(occ != NULL) { COPY_BUF(); } first_print = false; mm_last = false; @@ -924,53 +926,162 @@ void AlnRes::printQuals( } /** - * Add all of the cells involved in the given alignment to the database. + * Append the runs of res's cells to runs and set a's key and bounds. Returns + * false iff res covers no rows. + * + * The cells are the ones the per-cell walk this replaces visits: row i, from + * the first row not soft-trimmed on the left for readExtentRows() rows, covers + * columns [left, right). right is left + 1 plus the number of read gaps among + * the edits at pos i+1 that directly follow the edits consumed so far (none on + * the last row); the next row's left is right, or right - 1 if an edit at pos + * i is a reference gap. Edits are consumed in list order, those at pos i while + * on row i. So a row before the next edit's pos - 1, and every row once the + * next edit's pos is behind the current row, covers one cell one column right + * of the row above's: those rows are emitted in bulk, and the rows next to an + * edit run the per-cell walk's code unchanged. */ -void RedundantAlns::add(const AlnRes& res) { - assert(!cells_.empty()); - TRefOff left = res.refoff(), right; +bool RedundantAlns::makeRuns(const AlnRes& res, EList& runs, Aln& a) { const size_t len = res.readExtentRows(); + if(len == 0) { + return false; + } + TRefOff left = res.refoff(), right; const size_t alignmentStart = res.trimmedLeft(true); - if(!res.fw()) { + const size_t end = alignmentStart + len; + const bool fw = res.fw(); + if(!fw) { const_cast(res).invertEdits(); } const EList& ned = res.ned(); + const size_t nned = ned.size(); size_t nedidx = 0; - assert_leq(len, cells_.size()); - // For each row... - for(size_t i = alignmentStart; i < alignmentStart + len; i++) { - size_t diff = 1; // amount to shift to right for next round - right = left + 1; - while(nedidx < ned.size() && ned[nedidx].pos == i) { - if(ned[nedidx].isRefGap()) { - // Next my_left will be in same column as this round - diff = 0; + Run cur; + cur.dlo = cur.dhi = 0; + cur.r0 = cur.r1 = 0; + bool open = false; + int64_t dmin = 0, dmax = 0; + a.run0 = runs.size(); + size_t i = alignmentStart; + while(i < end) { + // Rows [i, stop) cover one cell each, on the diagonal of left at row i + size_t stop = end; + if(nedidx < nned) { + size_t pos = ned[nedidx].pos; + if(pos >= i) { + stop = (pos > i) ? pos - 1 : i; + if(stop > end) { + stop = end; + } } - nedidx++; } - if(i < alignmentStart + len - 1) { - // See how many inserts there are before the next read - // character - size_t nedidx_next = nedidx; - while(nedidx_next < ned.size() && ned[nedidx_next].pos == i+1) - { - if(ned[nedidx_next].isReadGap()) { - right++; + int64_t dlo, dhi; + size_t rfirst = i, rlast; + if(stop > i) { + dlo = (int64_t)left - (int64_t)i; + dhi = dlo + 1; + rlast = stop - 1; + left += (TRefOff)(stop - i); + i = stop; + } else { + size_t diff = 1; // amount to shift to right for next round + right = left + 1; + while(nedidx < nned && ned[nedidx].pos == i) { + if(ned[nedidx].isRefGap()) { + // Next my_left will be in same column as this round + diff = 0; + } + nedidx++; + } + if(i < end - 1) { + // See how many inserts there are before the next read + // character + size_t nedidx_next = nedidx; + while(nedidx_next < nned && ned[nedidx_next].pos == i+1) + { + if(ned[nedidx_next].isReadGap()) { + right++; + } + nedidx_next++; } - nedidx_next++; } + dlo = (int64_t)left - (int64_t)i; + dhi = (int64_t)right - (int64_t)i; + rlast = i; + left = right + diff - 1; + i++; } - for(TRefOff j = left; j < right; j++) { - // Add to db - RedundantCell c(res.refid(), res.fw(), j, i); - ASSERT_ONLY(bool ret =) cells_[i].insert(c); - assert(ret); + if(open && dlo == cur.dlo && dhi == cur.dhi) { + cur.r1 = rlast; // rows are consecutive: extend the open run + continue; } - left = right + diff - 1; - } - if(!res.fw()) { + if(open) { + runs.push_back(cur); + if(cur.dlo < dmin) dmin = cur.dlo; + if(cur.dhi > dmax) dmax = cur.dhi; + } else { + dmin = dlo; + dmax = dhi; + } + cur.dlo = dlo; + cur.dhi = dhi; + cur.r0 = rfirst; + cur.r1 = rlast; + open = true; + } + runs.push_back(cur); + if(cur.dlo < dmin) dmin = cur.dlo; + if(cur.dhi > dmax) dmax = cur.dhi; + if(!fw) { const_cast(res).invertEdits(); } + a.rfid = res.refid(); + a.fw = fw; + a.dlo = dmin; + a.dhi = dmax; + a.r0 = alignmentStart; + a.r1 = end - 1; + a.nrun = runs.size() - a.run0; + return true; +} + +/** + * Return true iff the two run lists, each in ascending row order, share a + * cell: some pair of runs shares a row and a diagonal. + */ +bool RedundantAlns::runsOverlap( + const Run* a, + size_t na, + const Run* b, + size_t nb) +{ + size_t i = 0, j = 0; + while(i < na && j < nb) { + if(a[i].r1 < b[j].r0) { + i++; + } else if(b[j].r1 < a[i].r0) { + j++; + } else { + if(a[i].dlo < b[j].dhi && b[j].dlo < a[i].dhi) { + return true; + } + if(a[i].r1 < b[j].r1) { + i++; + } else { + j++; + } + } + } + return false; +} + +/** + * Add all of the cells involved in the given alignment to the database. + */ +void RedundantAlns::add(const AlnRes& res) { + Aln a; + if(makeRuns(res, runs_, a)) { + alns_.push_back(a); + } } /** @@ -978,57 +1089,28 @@ void RedundantAlns::add(const AlnRes& res) { * one of the cells in the database. */ bool RedundantAlns::overlap(const AlnRes& res) { - assert(!cells_.empty()); - TRefOff left = res.refoff(), right; - const size_t len = res.readExtentRows(); - const size_t alignmentStart = res.trimmedLeft(true); - if(!res.fw()) { - const_cast(res).invertEdits(); + if(alns_.empty()) { + return false; } - const EList& ned = res.ned(); - size_t nedidx = 0; - // For each row... - bool olap = false; - assert_leq(len, cells_.size()); - for(size_t i = alignmentStart; i < alignmentStart + len; i++) { - size_t diff = 1; // amount to shift to right for next round - right = left + 1; - while(nedidx < ned.size() && ned[nedidx].pos == i) { - if(ned[nedidx].isRefGap()) { - // Next my_left will be in same column as this round - diff = 0; - } - nedidx++; - } - if(i < alignmentStart + len - 1) { - // See how many inserts there are before the next read - // character - size_t nedidx_next = nedidx; - while(nedidx_next < ned.size() && ned[nedidx_next].pos == i+1) - { - if(ned[nedidx_next].isReadGap()) { - right++; - } - nedidx_next++; - } - } - for(TRefOff j = left; j < right; j++) { - // Add to db - RedundantCell c(res.refid(), res.fw(), j, i); - if(cells_[i].contains(c)) { - olap = true; - break; - } - } - if(olap) { - break; - } - left = right + diff - 1; + qruns_.clear(); + Aln q; + if(!makeRuns(res, qruns_, q)) { + return false; } - if(!res.fw()) { - const_cast(res).invertEdits(); + const size_t nalns = alns_.size(); + for(size_t k = 0; k < nalns; k++) { + const Aln& a = alns_[k]; + // One branch on all the bounds: each test alone is unpredictable + bool cand = (a.dlo < q.dhi) & (q.dlo < a.dhi) & + (a.rfid == q.rfid) & (a.fw == q.fw) & + (a.r0 <= q.r1) & (q.r0 <= a.r1); + if(cand && + runsOverlap(runs_.ptr() + a.run0, a.nrun, qruns_.ptr(), q.nrun)) + { + return true; + } } - return olap; + return false; } /** diff --git a/aligner_result.h b/aligner_result.h index 9bfc7492..4434bc0a 100644 --- a/aligner_result.h +++ b/aligner_result.h @@ -1650,29 +1650,34 @@ struct RedundantCell { * whether one alignment is redundant (has a DP cell in common with) with a set * of others. * - * Adding cells to and checking cell against this data structure can get rather - * slow when there are many alignments in play. Dividing the burden over - * read-position bins helps some. + * The cells of an alignment are stored as runs: maximal stretches of + * consecutive rows whose cells cover the same range of diagonals (diagonal = + * column - row). A gapless alignment is one run; each gap adds a run or two. + * Two alignments share a cell iff they have the same reference id and + * orientation and some pair of their runs shares a row and a diagonal, so + * the database answers exactly what a set of RedundantCells per row would. */ class RedundantAlns { public: - RedundantAlns(int cat = DP_CAT) : cells_(cat) { } + RedundantAlns(int cat = DP_CAT) : alns_(cat), runs_(cat), qruns_(cat) { } /** * Empty the cell database. */ - void reset() { cells_.clear(); } + void reset() { + alns_.clear(); + runs_.clear(); + } /** - * Initialize and set the list of sets to equal the read length. + * Empty the cell database in preparation for a read of length npos. */ void init(size_t npos) { - cells_.resize(npos); - for(size_t i = 0; i < npos; i++) { - cells_[i].clear(); - } + (void)npos; + alns_.clear(); + runs_.clear(); } /** @@ -1688,7 +1693,46 @@ class RedundantAlns { protected: - EList > cells_; + /** + * Rows r0..r1 (inclusive), each with the cells of diagonals [dlo, dhi). + */ + struct Run { + int64_t dlo; + int64_t dhi; + size_t r0; + size_t r1; + }; + + /** + * One stored alignment: its key, the bounding rows and diagonals of its + * cells, and its runs, runs_[run0 .. run0+nrun). + */ + struct Aln { + TRefId rfid; + int64_t dlo; + int64_t dhi; + size_t r0; + size_t r1; + size_t run0; + size_t nrun; + bool fw; + }; + + /** + * Append the runs of res's cells to runs and set a's key and bounds. + * Returns false iff res covers no rows. + */ + static bool makeRuns(const AlnRes& res, EList& runs, Aln& a); + + /** + * Return true iff the two run lists, each in ascending row order, share + * a cell. + */ + static bool runsOverlap(const Run* a, size_t na, const Run* b, size_t nb); + + EList alns_; // stored alignments + EList runs_; // their runs + EList qruns_; // runs of the alignment being queried }; typedef uint64_t TNumAlns; diff --git a/aligner_seed.cpp b/aligner_seed.cpp index 4338f0cc..845ef6f5 100644 --- a/aligner_seed.cpp +++ b/aligner_seed.cpp @@ -18,6 +18,8 @@ */ #include +#include +#include #include "aligner_cache.h" #include "aligner_seed.h" #include "search_globals.h" @@ -357,6 +359,106 @@ Seed::instantiate( return ret; } +/** + * Bitwise equality of two doubles. + */ +static inline bool sameDouble(double a, double b) { + uint64_t x, y; + memcpy(&x, &a, sizeof(x)); + memcpy(&y, &b, sizeof(y)); + return x == y; +} + +/** + * Return true iff two penalty functions have the same type and parameters. + */ +static bool sameFunc(const SimpleFunc& a, const SimpleFunc& b) { + return a.getType() == b.getType() && + sameDouble(a.getMin(), b.getMin()) && + sameDouble(a.getMax(), b.getMax()) && + sameDouble(a.getConst(), b.getConst()) && + sameDouble(a.getCoeff(), b.getCoeff()); +} + +/** + * Return true iff every field of two constraints is the same. + */ +static bool sameConstraint(const Constraint& a, const Constraint& b) { + return a.edits == b.edits && + a.mms == b.mms && + a.ins == b.ins && + a.dels == b.dels && + a.penalty == b.penalty && + a.editsCeil == b.editsCeil && + a.mmsCeil == b.mmsCeil && + a.insCeil == b.insCeil && + a.delsCeil == b.delsCeil && + a.penaltyCeil == b.penaltyCeil && + a.instantiated == b.instantiated && + sameFunc(a.penFunc, b.penFunc); +} + +/** + * Return true iff every field of two seed policies is the same (the + * overall constraint is compared by pointer). + */ +static bool sameSeed(const Seed& a, const Seed& b) { + return a.len == b.len && + a.type == b.type && + a.overall == b.overall && + sameConstraint(a.zones[0], b.zones[0]) && + sameConstraint(a.zones[1], b.zones[1]) && + sameConstraint(a.zones[2], b.zones[2]); +} + +#ifndef NDEBUG +/** + * Return true iff two InstantiatedSeeds hold the same fields as + * Seed::instantiate writes them. + */ +static bool sameInstantiation(const InstantiatedSeed& a, const InstantiatedSeed& b) { + if(a.steps.size() != b.steps.size() || a.zones.size() != b.zones.size()) { + return false; + } + for(size_t k = 0; k < a.steps.size(); k++) { + if(a.steps[k] != b.steps[k]) return false; + } + for(size_t k = 0; k < a.zones.size(); k++) { + if(a.zones[k] != b.zones[k]) return false; + } + return sameConstraint(a.cons[0], b.cons[0]) && + sameConstraint(a.cons[1], b.cons[1]) && + sameConstraint(a.cons[2], b.cons[2]) && + sameConstraint(a.overall, b.overall) && + a.maxjump == b.maxjump && + a.seedoff == b.seedoff && + a.seedoffidx == b.seedoffidx && + a.fw == b.fw && + sameSeed(a.s, b.s); +} +#endif + +/** + * Source of SeedInstKey generation numbers; 0 is never handed out. + */ +static std::atomic seedInstGens(0); + +bool SeedInstKey::matches(const Seed& sd, size_t rdl) const { + assert(sd.overall != NULL); + return gen != 0 && + rdlen == rdl && + sameSeed(seed, sd) && + sameConstraint(overall, *sd.overall); +} + +void SeedInstKey::set(const Seed& sd, size_t rdl) { + assert(sd.overall != NULL); + seed = sd; + overall = *sd.overall; + rdlen = rdl; + gen = seedInstGens.fetch_add(1, std::memory_order_relaxed) + 1; +} + /** * Return a set consisting of 1 seed encapsulating an exact matching * strategy. @@ -468,7 +570,7 @@ enum { * reference strand. I.e. if fw is false, then the sequence that appears in * 'seq' is the reverse complement of the raw read substring. */ -void +bool SeedAligner::instantiateSeq( const Read& read, // input read BTDnaString& seq, // output sequence @@ -484,10 +586,14 @@ SeedAligner::instantiateSeq( qual.resize(len); // If fw is false, we take characters starting at the 3' end of the // reverse complement of the read. + int cor = 0; // OR of the characters (0-3, N = 4) for(int i = 0; i < len; i++) { - seq.set(read.patFw.windowGetDna(i, fw, depth, len), i); + const int c = read.patFw.windowGetDna(i, fw, depth, len); + cor |= c; + seq.set(c, i); qual.set(read.qual.windowGet(i, fw, depth, len), i); } + return (cor & 4) != 0; } /** @@ -532,6 +638,14 @@ pair SeedAligner::instantiateSeeds( ret.second = 0; // # seeds that hit in cache with non-empty results sr.reset(read, offIdx2off_, nseeds); assert(sr.repOk(&cache.current(), true)); // require that SeedResult be initialized + // The instantiation keys of sr's InstantiatedSeeds, one per seed policy + std::vector& keys = sr.instKeys(); + if(keys.size() < seeds.size()) keys.resize(seeds.size()); + for(size_t j = 0; j < seeds.size(); j++) { + if(!keys[j].matches(seeds[j], read.length())) { + keys[j].set(seeds[j], read.length()); + } + } // For each seed position for(int fwi = 0; fwi < 2; fwi++) { bool fw = (fwi == 0); @@ -546,7 +660,7 @@ pair SeedAligner::instantiateSeeds( // Extract the seed sequence at this offset // If fw == true, we extract the characters from i*per to // i*(per-1) (exclusive). If fw == false, - instantiateSeq( + const bool hasN = instantiateSeq( read, sr.seqs(fw)[i], sr.quals(fw)[i], @@ -560,7 +674,25 @@ pair SeedAligner::instantiateSeeds( iss.expand(); assert_eq(seedlen, seeds[j].len); InstantiatedSeed* is = &iss.back(); - if(seeds[j].instantiate( + const uint64_t gen = keys[j].gen; + if(!hasN && is->instGen == gen) { + // 'is' already holds this policy's instantiation for + // this read length, and there are no Ns to charge to + // its constraints: only the offset fields differ + is->seedoff = depth; + is->seedoffidx = i; + is->fw = fw; +#ifndef NDEBUG + InstantiatedSeed chk; + const bool chkok = seeds[j].instantiate( + read, sr.seqs(fw)[i], sr.quals(fw)[i], pens, + depth, i, j, fw, chk); + assert(chkok); + assert(sameInstantiation(chk, *is)); +#endif + ret.first++; + if(fwi == 0) { instFw.first++; } else { instRc.first++; } + } else if(seeds[j].instantiate( read, sr.seqs(fw)[i], sr.quals(fw)[i], @@ -571,12 +703,16 @@ pair SeedAligner::instantiateSeeds( fw, *is)) { + // Without Ns the constraints are uncharged and 'is' + // holds the key's instantiation + is->instGen = hasN ? 0 : gen; // Can we fill this seed hit in from the cache? ret.first++; if(fwi == 0) { instFw.first++; } else { instRc.first++; } } else { // Seed may fail to instantiate if there are Ns // that prevent it from matching + is->instGen = 0; met.filteredseed++; iss.pop_back(); } @@ -654,6 +790,18 @@ class SeedAlignerSearchState { }; +// searchAllSeeds scratch, kept across calls so that it allocates only +// when a read needs more seed searches than any read before it +struct SeedAligner::SeedSearchScratch { + std::vector params; // one per seed search + std::vector states; // one per seed search + std::vector first; // per cache: its first search +}; + +SeedAligner::~SeedAligner() { + delete sss_; +} + /** * We assume that all seeds are the same length. * @@ -661,6 +809,14 @@ class SeedAlignerSearchState { * * 1. Instantiate all seeds, retracting them if necessary. * 2. Calculate zone boundaries for each seed + * + * The seed offsets of both strands are searched together, up to + * SEED_BATCH_CACHES offsets per searchSeedBi batch, so that more searches + * are in flight than with one batch per strand. The searches write only + * their SeedSearchCaches and the bwops_/bwedits_ counts, and the searches + * of one offset stay in one batch, in their order; the results are then + * handed to the alignment cache and to sr in the order the per-strand + * batches used: forward seed offsets in order, then reverse-complement ones. */ void SeedAligner::searchAllSeeds( const EList& seeds, // search seeds @@ -686,93 +842,84 @@ void SeedAligner::searchAllSeeds( bwops_ = bwedits_ = 0; uint64_t possearches = 0, seedsearches = 0, intrahits = 0, interhits = 0, ooms = 0; - /** - * TODO: Define is somewhere else - * Note: The ideal may be dependent on the CPU model, but 8 seems to work fine. - * 2 is too small for prefetch to be fully effective, 4 seems already OK, - * and 32 is too big (cache trashing). - **/ - const int ibatch_size = 8; - - SeedSearchMultiCache mcache; - std::vector paramVec; - // Scratch space for searchSeedBi, grown on demand and reused across - // batches and reads so that the search itself never allocates. - std::vector sstateVec; - - mcache.reserve(ibatch_size); - paramVec.reserve(ibatch_size*16); // assume no more than 16 iss per cache, on average - sstateVec.reserve(ibatch_size*16); - + if(sss_ == NULL) sss_ = new SeedSearchScratch(); + SeedSearchMultiCache& mcache = mcache_; + std::vector& paramVec = sss_->params; + std::vector& sstateVec = sss_->states; + std::vector& first = sss_->first; + mcache.clear(); + paramVec.clear(); + first.clear(); + + // One cache per seed offset with something to search, forward offsets + // first. All caches are taken before the searches refer to them. for(int fwi = 0; fwi < 2; fwi++) { const bool fw = (fwi == 0); - int i =0; - // For each instantiated seed, but batched - while (i < (int)sr.numOffs()) { - const int ibatch_max = std::min(i+ibatch_size,(int)sr.numOffs()); - mcache.clear(); - paramVec.clear(); - // start aligning and find list of seeds to search - for(; i < ibatch_max; i++) { + for(int i = 0; i < (int)sr.numOffs(); i++) { assert(sr.repOk(&cache.current())); - EList& iss = sr.instantiatedSeeds(fw, i); - if(iss.empty()) { + if(sr.instantiatedSeeds(fw, i).empty()) { // Cache hit in an across-read cache continue; } - const BTDnaString& seq = sr.seqs(fw)[i]; // seed sequence - const BTString& qual = sr.quals(fw)[i]; // seed qualities - mcache.emplace_back(seq, qual, i, fw); - const size_t mnr = mcache.size()-1; - SeedSearchCache &srcache = mcache[mnr]; - { - possearches++; - for(size_t j = 0; j < iss.size(); j++) { - // Set seq and qual appropriately, using the seed sequences - // and qualities already installed in SeedResults - assert_eq(fw, iss[j].fw); - assert_eq(i, (int)iss[j].seedoffidx); - paramVec.emplace_back(srcache, iss[j], ebwtFw_, ebwtBw_); - seedsearches++; - } - } - } // internal i (batch) loop - - // do the searches - if (!paramVec.empty()) { - if(sstateVec.size() < paramVec.size()) sstateVec.resize(paramVec.size()); - searchSeedBi(paramVec.size(), &(paramVec[0]), &(sstateVec[0])); - } - - // finish aligning and add to SeedResult - for (size_t mnr=0; mnr& iss = sr.instantiatedSeeds(fw, i); + SeedSearchCache &srcache = mcache[mnr]; + first.push_back(paramVec.size()); + for(size_t j = 0; j < iss.size(); j++) { + // Set seq and qual appropriately, using the seed sequences + // and qualities already installed in SeedResults + assert_eq(fw, iss[j].fw); + assert_eq(i, (int)iss[j].seedoffidx); + paramVec.emplace_back(srcache, iss[j], ebwtFw_, ebwtBw_); + seedsearches++; + } + } + first.push_back(paramVec.size()); + + // do the searches + if(sstateVec.size() < paramVec.size()) sstateVec.resize(paramVec.size()); + for(size_t c0 = 0; c0 < mcache.size(); c0 += SEED_BATCH_CACHES) { + const size_t c1 = std::min(c0 + SEED_BATCH_CACHES, mcache.size()); + const size_t p0 = first[c0], p1 = first[c1]; + if(p1 > p0) { + searchSeedBi(p1 - p0, &(paramVec[p0]), &(sstateVec[p0])); + } + } + + // finish aligning and add to SeedResult + for (size_t mnr=0; mnr 3) { + return false; + } + } + fi = ebwt.ftabSeqToInt(seq, left - ftabLen, false); + return true; +} + +/** + * The rest of exactSweepInit, given exactSweepFtabIdx's result for the + * same depth: the same top, bot and dep. + */ +inline void exactSweepInitIdx( + const Ebwt& ebwt, + const BTDnaString& seq, + const int ftabLen, + const size_t len, + const bool doFtab, + const TIndexOffU fi, + size_t &dep, + TIndexOffU &top, + TIndexOffU &bot + ) +{ + top = bot = 0; + if(doFtab) { + // Use ftab + if(fi != std::numeric_limits::max()) { + ebwt.ftabLoHi(fi, top, bot); + } + dep += (size_t)ftabLen; + } else { + // Use fchr + int c = seq[len-dep-1]; + if(c < 4) { + top = ebwt.fchr()[c]; + bot = ebwt.fchr()[c+1]; + } + dep++; + } +} + inline void exactSweepMapLF( const Ebwt& ebwt, const BTDnaString& seq, @@ -1045,26 +1255,270 @@ size_t SeedAligner::exactSweep( } /** - * Search for end-to-end exact hit for read. Return true iff one is found. + * exactSweep for many reads at once. exactSweep steps the sweeps of the read + * and of its reverse complement in turn, so that each step's prefetch of the + * sides its sweep needs next is hidden by the other sweep's step. Usually + * only one of the two survives past the first few steps, so for most of a + * sweep one load is in flight. Here the sweeps of all the jobs (each read, + * then its reverse complement, job after job) are stepped in turn in + * EXACT_SWEEP_LANES lanes; when a sweep ends, the next one takes its lane. + * The sweeps share no state, and each takes exactly the steps it takes in + * exactSweep, so each job's edit bounds, exact hits, element count (nelt) + * and LF operations are the ones exactSweep gives for its read. */ -bool SeedAligner::oneMmSearch( +void SeedAligner::exactSweepBatch( + const Ebwt& ebwt, // BWT index + const Scoring& sc, // scoring scheme + size_t mineMax, // don't care about edit bounds > this + bool repex, // report 0mm hits? + ExactSweepJob* const* jobs, // the reads to sweep + size_t njobs) // # jobs +{ + assert_gt(mineMax, 0); + const int ftabLen = ebwt.eh().ftabChars(); + + // One sweep in progress + struct Lane { + ExactSweepJob* job; // NULL: lane is idle + int fwi; // 0: the read, 1: its reverse complement + const BTDnaString* seq; + size_t len; + size_t dep; + size_t nedit; + TIndexOffU top; + TIndexOffU bot; + SideLocus tloc; + SideLocus bloc; + bool doInit; + bool done; + size_t prefetch_count; + bool doFtab; // the next (re)start uses the ftab + TIndexOffU fi; // at this ftab index +#ifndef NDEBUG + size_t fidep; // the depth they were computed for +#endif + }; + Lane lanes[EXACT_SWEEP_LANES]; + + // Before a lane's turn that (re)starts its sweep: compute the ftab + // index the start reads and prefetch its entries, so that they arrive + // while the other lanes take their turns + const TIndexOffU* const ftab = ebwt.ftab(); + auto prepInit = [&](Lane& ln) { + ln.doFtab = exactSweepFtabIdx(ebwt, *ln.seq, ftabLen, ln.len, ln.dep, ln.fi); + if(ln.doFtab && ln.fi != std::numeric_limits::max()) { + __builtin_prefetch(ftab + ln.fi); // ftabHi(fi) + __builtin_prefetch(ftab + ln.fi + 1); // ftabLo(fi+1) + } +#ifndef NDEBUG + ln.fidep = ln.dep; +#endif + }; + + // Sweep s is job s/2's read (s even) or its reverse complement (s odd) + const size_t nsweeps = 2 * njobs; + size_t next = 0; + + // The end of a sweep: what exactSweep does after its loop + auto finish = [&](Lane& ln) { + if( (!ln.done) && (ln.dep >= ln.len) ) { + ExactSweepJob& job = *ln.job; + // Set the minimum # edits + job.mine[ln.fwi] = ln.nedit; + // Done + if(ln.nedit == 0 && ln.bot > ln.top) { + if(repex) { + // This is an exact hit + int64_t score = ln.len * sc.match(); + job.hit[ln.fwi] = true; + job.top[ln.fwi] = ln.top; + job.bot[ln.fwi] = ln.bot; + job.score[ln.fwi] = score; + assert(ebwt.contains(*ln.seq, NULL, NULL)); + } + job.nelt += (ln.bot - ln.top); + } + } + }; + + // Start the next sweep in a lane; false if none is left + auto start = [&](Lane& ln) { + while(next < nsweeps) { + ExactSweepJob& job = *jobs[next >> 1]; + const int fwi = (int)(next & 1); + next++; + if(fwi == 0 ? job.nofw : job.norc) { + continue; + } + ln.job = &job; + ln.fwi = fwi; + ln.seq = fwi == 0 ? &job.read->patFw : &job.read->patRc; + ln.len = job.read->length(); + ln.dep = 0; + ln.nedit = 0; + ln.top = ln.bot = 0; + ln.doInit = true; + ln.done = false; + ln.prefetch_count = 0; + if(ln.len == 0) { + finish(ln); + continue; + } + const BTDnaString& seq = *ln.seq; + __builtin_prefetch(&(seq[ln.len-1])); + if (ln.len>48) __builtin_prefetch(&(seq[ln.len-49])); // HW prefetch prediction assumes forward, help it + prepInit(ln); + return true; + } + ln.job = NULL; + return false; + }; + + size_t nlanes = 0; + for(size_t l = 0; l < EXACT_SWEEP_LANES; l++) { + if(start(lanes[l])) { + nlanes++; + } + } + while(nlanes > 0) { + for(size_t l = 0; l < EXACT_SWEEP_LANES; l++) { + Lane& ln = lanes[l]; + if(ln.job == NULL) { + continue; + } + const BTDnaString& seq = *ln.seq; + if (++ln.prefetch_count >= 48) { // cache line is 64 bytes, but we may skip some deps + const size_t left = ln.len-ln.dep; + if (left>48) { + __builtin_prefetch(&(seq[left-49])); // HW prefetch prediction assumes forward, help it + } + ln.prefetch_count = 0; + } + // One turn of exactSweep's loop for this sweep + bool step = true; + if (ln.doInit) { + assert_eq(ln.fidep, ln.dep); + exactSweepInitIdx(ebwt, seq, ftabLen, ln.len, // in + ln.doFtab, ln.fi, // in + ln.dep, ln.top, ln.bot); // out + if ( exactSweepStep(ebwt, ln.top, ln.bot, mineMax, + ln.tloc, ln.bloc, + ln.job->mine[ln.fwi], + ln.nedit, ln.done) ) { + step = false; + } else { + ln.doInit=false; + // Its first LF step reads the side INIT_LOCS has just + // prefetched: leave it to the lane's next turn + step = false; + } + } + if (step && ln.dep < ln.len) { + exactSweepMapLF(ebwt, seq, ln.len, ln.dep, ln.tloc, ln.bloc, + ln.top, ln.bot, ln.job->bwops); + + if ( exactSweepStep(ebwt, ln.top, ln.bot, mineMax, + ln.tloc, ln.bloc, + ln.job->mine[ln.fwi], + ln.nedit, ln.done) ) { + ln.doInit=true; + } + ln.dep++; + } + if (ln.done || ln.dep >= ln.len) { + // This sweep has ended; the next one takes the lane + finish(ln); + if(!start(ln)) { + nlanes--; + } + } else if (ln.doInit) { + // The sweep restarts on the lane's next turn + prepInit(ln); + } + } + } +} + +/** + * Compute the ftab indices oneMmStart reads for the job's directions (each + * as ftabLoHi computes it from the read), store them in the job, and + * prefetch their ftab entries. Directions oneMmStart skips get an index + * too; it is not read. + */ +void SeedAligner::oneMmStartPrefetch( + const Ebwt* ebwtFw, // BWT index + const Ebwt* ebwtBw, // BWT' index + bool rep1mm, // report 1mm hits? + OneMmJob& job) +{ + const Read& read = *job.read; + const size_t len = read.length(); + const int ftabLen = ebwtFw->eh().ftabChars(); + job.ftabPre = true; + job.ftabPreRep1mm = rep1mm; + if(ftabLen <= 1) { + return; // oneMmStart uses fchr + } + const TIndexOffU nomatch = std::numeric_limits::max(); + for(int fwi = 0; fwi < 2; fwi++) { + bool fw = (fwi == 0); + if( fw && job.nofw) continue; + if(!fw && job.norc) continue; + int lim = rep1mm ? 2 : 1; + for(int ebwtfwi = 0; ebwtfwi < lim; ebwtfwi++) { + bool ebwtfw = (ebwtfwi == 0); + // The near half's length, as in oneMmStart + const size_t nea = (len >> 1) + (ebwtfw ? 0 : (len & 1)); + if((size_t)ftabLen > nea) { + continue; // oneMmStart uses fchr + } + const Ebwt* ebwt = (ebwtfw ? ebwtFw : ebwtBw); + const Ebwt* ebwtp = (ebwtfw ? ebwtBw : ebwtFw); + const BTDnaString& seq = + (fw ? (ebwtfw ? read.patFw : read.patFwRev) : + (ebwtfw ? read.patRc : read.patRcRev)); + const bool rev = !ebwtfw; + const int k = 2 * fwi + ebwtfwi; + const TIndexOffU fi = ebwt->ftabSeqToInt(seq, len - ftabLen, rev); + job.fi[k] = fi; + if(fi != nomatch) { + __builtin_prefetch(ebwt->ftab() + fi); + __builtin_prefetch(ebwt->ftab() + fi + 1); + } + if(rep1mm) { + const TIndexOffU fip = ebwtp->ftabSeqToInt(seq, len - ftabLen, rev); + job.fip[k] = fip; + if(fip != nomatch) { + __builtin_prefetch(ebwtp->ftab() + fip); + __builtin_prefetch(ebwtp->ftab() + fip + 1); + } + } + } + } +} + +/** + * Set up the directions of a 1-mismatch search and their main walks, which + * are the first job.ndir walks. Returns false if the read cannot have a hit + * (too many Ns); the job then has no walks. + */ +bool SeedAligner::oneMmStart( const Ebwt* ebwtFw, // BWT index const Ebwt* ebwtBw, // BWT' index - const Read& read, // read to align const Scoring& sc, // scoring - int64_t minsc, // minimum score - bool nofw, // don't align forward read - bool norc, // don't align revcomp read - bool local, // 1mm hits must be legal local alignments bool repex, // report 0mm hits? bool rep1mm, // report 1mm hits? - SeedResults& hits, // holds all the seed hits (and exact hit) - SeedSearchMetrics& met) // metrics + OneMmJob& job) { assert(!rep1mm || ebwtBw != NULL); + const Read& read = *job.read; const size_t len = read.length(); - int nceil = sc.nCeil.f((double)len); + job.repex = repex; + job.rep1mm = rep1mm; + job.len = len; + job.nceil = sc.nCeil.f((double)len); size_t ns = read.ns(); + job.ns = ns; if(ns > 1) { // Can't align this with <= 1 mismatches return false; @@ -1088,20 +1542,11 @@ bool SeedAligner::oneMmSearch( } assert_geq(halfFw, 1); assert_geq(halfBw, 1); - SideLocus tloc, bloc; - TIndexOffU t[4], b[4]; // dest BW ranges for BWT - t[0] = t[1] = t[2] = t[3] = 0; - b[0] = b[1] = b[2] = b[3] = 0; - TIndexOffU tp[4], bp[4]; // dest BW ranges for BWT' - tp[0] = tp[1] = tp[2] = tp[3] = 0; - bp[0] = bp[1] = bp[2] = bp[3] = 0; - TIndexOffU top = 0, bot = 0, topp = 0, botp = 0; - // Align fw read / rc read - bool results = false; + int ndir = 0; for(int fwi = 0; fwi < 2; fwi++) { bool fw = (fwi == 0); - if( fw && nofw) continue; - if(!fw && norc) continue; + if( fw && job.nofw) continue; + if(!fw && job.norc) continue; // Align going right-to-left, left-to-right int lim = rep1mm ? 2 : 1; for(int ebwtfwi = 0; ebwtfwi < lim; ebwtfwi++) { @@ -1130,275 +1575,537 @@ bool SeedAligner::oneMmSearch( continue; } size_t dep = 0; + TIndexOffU top = 0, bot = 0, topp = 0, botp = 0; // Align near half if(ftabLen > 1 && (size_t)ftabLen <= nea) { // Use ftab to jump partway into near half bool rev = !ebwtfw; - ebwt->ftabLoHi(seq, len - ftabLen, rev, top, bot); - if(rep1mm) { - ebwtp->ftabLoHi(seq, len - ftabLen, rev, topp, botp); - assert_eq(bot - top, botp - topp); + if(job.ftabPre && job.ftabPreRep1mm == rep1mm) { + // The indices ftabLoHi would compute, computed earlier + const TIndexOffU nomatch = std::numeric_limits::max(); + const int k = 2 * fwi + ebwtfwi; + assert_eq(job.fi[k], ebwt->ftabSeqToInt(seq, len - ftabLen, rev)); + if(job.fi[k] != nomatch) { + ebwt->ftabLoHi(job.fi[k], top, bot); + } + if(rep1mm) { + assert_eq(job.fip[k], ebwtp->ftabSeqToInt(seq, len - ftabLen, rev)); + if(job.fip[k] != nomatch) { + ebwtp->ftabLoHi(job.fip[k], topp, botp); + } + assert_eq(bot - top, botp - topp); + } + } else { + ebwt->ftabLoHi(seq, len - ftabLen, rev, top, bot); + if(rep1mm) { + ebwtp->ftabLoHi(seq, len - ftabLen, rev, topp, botp); + assert_eq(bot - top, botp - topp); + } } if(bot - top == 0) { continue; } - int c = seq[len - ftabLen]; - t[c] = top; b[c] = bot; - tp[c] = topp; bp[c] = botp; dep = ftabLen; - // initialize tloc, bloc?? } else { // Use fchr to jump in by 1 pos int c = seq[len-1]; assert_range(0, 3, c); - top = topp = tp[c] = ebwt->fchr()[c]; - bot = botp = bp[c] = ebwt->fchr()[c+1]; + top = topp = ebwt->fchr()[c]; + bot = botp = ebwt->fchr()[c+1]; if(bot - top == 0) { continue; } dep = 1; - // initialize tloc, bloc?? } - INIT_LOCS(top, bot, tloc, bloc, *ebwt); assert(sanityPartial(ebwt, ebwtp, seq, len-dep, len, rep1mm, top, bot, topp, botp)); - bool do_continue = false; - for(; dep < nea; dep++) { - assert_lt(dep, len); - int rdc = seq[len - dep - 1]; - tp[0] = tp[1] = tp[2] = tp[3] = topp; - bp[0] = bp[1] = bp[2] = bp[3] = botp; - if(bloc.valid()) { - bwops_++; - t[0] = t[1] = t[2] = t[3] = b[0] = b[1] = b[2] = b[3] = 0; - ebwt->mapBiLFEx(tloc, bloc, t, b, tp, bp); - SANITY_CHECK_4TUP(t, b, tp, bp); - top = t[rdc]; bot = b[rdc]; - if(bot <= top) { - do_continue = true; - break; - } - topp = tp[rdc]; botp = bp[rdc]; - assert(!rep1mm || bot - top == botp - topp); - } else { - assert_eq(bot, top+1); - assert(!rep1mm || botp == topp+1); - bwops_++; - top = ebwt->mapLF1(top, tloc, rdc); - if(top == OFF_MASK) { - do_continue = true; - break; - } - bot = top + 1; - t[rdc] = top; b[rdc] = bot; - tp[rdc] = topp; bp[rdc] = botp; - assert(!rep1mm || b[rdc] - t[rdc] == bp[rdc] - tp[rdc]); - // topp/botp stay the same - } - INIT_LOCS(top, bot, tloc, bloc, *ebwt); - assert(sanityPartial(ebwt, ebwtp, seq, len - dep - 1, len, rep1mm, top, bot, topp, botp)); + OneMmDir& d = job.dirs[ndir]; + d.ebwt = ebwt; + d.ebwtp = ebwtp; + d.seq = &seq; + d.qual = &qual; + d.nea = nea; + d.fw = fw; + d.ebwtfw = ebwtfw; + d.exact = false; + d.etop = d.ebot = 0; + job.walks.expand(); + job.walks.back().init(ndir, false, top, bot, topp, botp, dep); + ndir++; + } + } + job.ndir = ndir; + job.mainLeft = (size_t)ndir; + return true; +} + +/** + * One LF step of main walk i of a job from loci tloc and bloc, which it then + * sets for the next step; returns false when the walk has ended. It may + * append mismatch walks to the job's walks, so it works on a copy. + */ +inline bool SeedAligner::oneMmStepMain( + OneMmJob& job, + size_t i, + SideLocus& tloc, + SideLocus& bloc, + bool prefetch) +{ + OneMmWalk m = job.walks[i]; + OneMmDir& d = job.dirs[m.dir]; + const Ebwt* ebwt = d.ebwt; + const BTDnaString& seq = *d.seq; + const size_t len = job.len; + const bool rep1mm = job.rep1mm; + TIndexOffU t[4], b[4]; // dest BW ranges for BWT + t[0] = t[1] = t[2] = t[3] = 0; + b[0] = b[1] = b[2] = b[3] = 0; + TIndexOffU tp[4], bp[4]; // dest BW ranges for BWT' + tp[0] = tp[1] = tp[2] = tp[3] = m.topp; + bp[0] = bp[1] = bp[2] = bp[3] = m.botp; + if(m.dep < d.nea) { + // Near half: exact matching + assert_lt(m.dep, len); + int rdc = seq[len - m.dep - 1]; + if(bloc.valid()) { + job.bwops++; + ebwt->mapBiLFEx(tloc, bloc, t, b, tp, bp); + SANITY_CHECK_4TUP(t, b, tp, bp); + m.top = t[rdc]; m.bot = b[rdc]; + if(m.bot <= m.top) { + return false; + } + m.topp = tp[rdc]; m.botp = bp[rdc]; + assert(!rep1mm || m.bot - m.top == m.botp - m.topp); + } else { + assert_eq(m.bot, m.top+1); + assert(!rep1mm || m.botp == m.topp+1); + job.bwops++; + m.top = ebwt->mapLF1(m.top, tloc, rdc); + if(m.top == OFF_MASK) { + return false; } - if(do_continue) { + m.bot = m.top + 1; + // topp/botp stay the same + } + if(prefetch) { + INIT_LOCS(m.top, m.bot, tloc, bloc, *ebwt); + } else { + INIT_LOCS_NOPF(m.top, m.bot, tloc, bloc, *ebwt); + } + assert(sanityPartial(ebwt, d.ebwtp, seq, len - m.dep - 1, len, rep1mm, m.top, m.bot, m.topp, m.botp)); + m.dep++; + job.walks[i] = m; + return true; + } + // Far half: exact matching, and at each position a mismatch walk + // for each other character that occurs + if(m.dep >= len) { + return false; + } + int rdc = seq[len - m.dep - 1]; + int quc = (*d.qual)[len - m.dep - 1]; + if(rdc > 3 && job.nceil == 0) { + return false; + } + int clo = 0, chi = 3; + bool match = true; + if(bloc.valid()) { + job.bwops++; + ebwt->mapBiLFEx(tloc, bloc, t, b, tp, bp); + SANITY_CHECK_4TUP(t, b, tp, bp); + match = rdc < 4; + if(match) { + m.top = t[rdc]; m.bot = b[rdc]; + m.topp = tp[rdc]; m.botp = bp[rdc]; + } + } else { + assert_eq(m.bot, m.top+1); + assert(!rep1mm || m.botp == m.topp+1); + job.bwops++; + clo = ebwt->mapLF1(m.top, tloc); + match = (clo == rdc); + assert_range(-1, 3, clo); + if(clo < 0) { + return false; // Hit the $ + } + t[clo] = m.top; + b[clo] = m.bot = m.top + 1; + bp[clo] = m.botp; + tp[clo] = m.topp; + assert(!rep1mm || m.bot - m.top == m.botp - m.topp); + assert(!rep1mm || b[clo] - t[clo] == bp[clo] - tp[clo]); + chi = clo; + } + if(rep1mm && (job.ns == 0 || rdc > 3)) { + for(int j = clo; j <= chi; j++) { + if(j == rdc || b[j] == t[j]) { + // Either matches read or isn't a possibility continue; } - // Align far half - for(; dep < len; dep++) { - int rdc = seq[len-dep-1]; - int quc = qual[len-dep-1]; - if(rdc > 3 && nceil == 0) { - break; - } - tp[0] = tp[1] = tp[2] = tp[3] = topp; - bp[0] = bp[1] = bp[2] = bp[3] = botp; - int clo = 0, chi = 3; - bool match = true; - if(bloc.valid()) { - bwops_++; - t[0] = t[1] = t[2] = t[3] = b[0] = b[1] = b[2] = b[3] = 0; - ebwt->mapBiLFEx(tloc, bloc, t, b, tp, bp); - SANITY_CHECK_4TUP(t, b, tp, bp); - match = rdc < 4; - top = t[rdc]; bot = b[rdc]; - topp = tp[rdc]; botp = bp[rdc]; - } else { - assert_eq(bot, top+1); - assert(!rep1mm || botp == topp+1); - bwops_++; - clo = ebwt->mapLF1(top, tloc); - match = (clo == rdc); - assert_range(-1, 3, clo); - if(clo < 0) { - break; // Hit the $ + // Potential mismatch - next, try + assert_eq(b[j] - t[j], bp[j] - tp[j]); + job.walks.expand(); + OneMmWalk& w = job.walks.back(); + w.init(m.dir, true, t[j], b[j], tp[j], bp[j], m.dep + 1); + w.mmdep = (uint32_t)m.dep; + w.mmc = (int8_t)j; + w.rdc = (int8_t)rdc; + w.quc = quc; + if(w.dep == len) { + // Mismatch at the last position: nothing left to match + w.done = w.hit = true; + } + } + } + if(m.bot > m.top && match) { + assert_lt(rdc, 4); + if(m.dep == len-1) { + // Success; this is an exact hit + if(d.ebwtfw && job.repex) { + d.exact = true; + d.etop = m.top; + d.ebot = m.bot; + } + return false; // End of far loop + } + if(prefetch) { + INIT_LOCS(m.top, m.bot, tloc, bloc, *ebwt); + } else { + INIT_LOCS_NOPF(m.top, m.bot, tloc, bloc, *ebwt); + } + assert(sanityPartial(ebwt, d.ebwtp, seq, len - m.dep - 1, len, rep1mm, m.top, m.bot, m.topp, m.botp)); + m.dep++; + job.walks[i] = m; + return true; + } + return false; // End of far loop +} + +/** + * One LF step of mismatch walk i of a job, as oneMmStepMain. + */ +inline bool SeedAligner::oneMmStepMm( + OneMmJob& job, + size_t i, + SideLocus& tloc, + SideLocus& bloc, + bool prefetch) +{ + OneMmWalk& w = job.walks[i]; + const OneMmDir& d = job.dirs[w.dir]; + const Ebwt* ebwt = d.ebwt; + const size_t len = job.len; + int rdcm = (*d.seq)[len - w.dep - 1]; + assert_lt(rdcm, 4); + if(bloc.valid()) { + job.bwops++; + TIndexOffU tm[4], bm[4]; // dest BW ranges for BWT + tm[0] = tm[1] = tm[2] = tm[3] = 0; + bm[0] = bm[1] = bm[2] = bm[3] = 0; + TIndexOffU tmp[4], bmp[4]; // dest BW ranges for BWT' + tmp[0] = tmp[1] = tmp[2] = tmp[3] = w.topp; + bmp[0] = bmp[1] = bmp[2] = bmp[3] = w.botp; + ebwt->mapBiLFEx(tloc, bloc, tm, bm, tmp, bmp); + SANITY_CHECK_4TUP(tm, bm, tmp, bmp); + w.top = tm[rdcm]; w.bot = bm[rdcm]; + w.topp = tmp[rdcm]; w.botp = bmp[rdcm]; + if(w.bot <= w.top) { + return false; + } + } else { + assert_eq(w.bot, w.top+1); + assert_eq(w.botp, w.topp+1); + job.bwops++; + w.top = ebwt->mapLF1(w.top, tloc, rdcm); + if(w.top == OFF_MASK) { + return false; + } + w.bot = w.top + 1; + // topp/botp stay the same + } + if(++w.dep == len) { + w.hit = true; + return false; + } + if(prefetch) { + INIT_LOCS(w.top, w.bot, tloc, bloc, *ebwt); + } else { + INIT_LOCS_NOPF(w.top, w.bot, tloc, bloc, *ebwt); + } + return true; +} + +/** + * Step the walks of the jobs in turn, at most LANES at a time. The walks of + * each job are started in the order they were made, job after job; starting + * a walk sets its loci, which prefetches the sides of its first step. A + * job's main walks may make more walks for it until they end, so every job + * from the first one that may still have a walk to start is looked at + * whenever a lane is free. The walks share no state: every walk takes the + * same steps as it would alone. + * + * With GROUP, setting a lane's loci does not prefetch their sides; the + * prefetches of every lane are issued together once per round, after the + * walks that take free lanes are started and before the lanes are stepped, + * so that their page walks overlap. The same lines are prefetched. + * oneMmSearchBatch's 32-lane runs group; oneMmSearch's one-read runs do + * not (with 4 KB pages, grouping made the per-read runs slower). + */ +template +void SeedAligner::oneMmRun(OneMmJob* const* jobs, size_t njobs) +{ + OneMmJob* actj[LANES]; + size_t act[LANES]; + SideLocus tlocs[LANES], blocs[LANES]; + const uint8_t* ebs[LANES]; // with GROUP: the BWT each lane's walk steps in + size_t nact = 0, jfirst = 0; + while(true) { + while(jfirst < njobs && jobs[jfirst]->mainLeft == 0 && + jobs[jfirst]->nextw == jobs[jfirst]->walks.size()) + { + jfirst++; + } + for(size_t j = jfirst; j < njobs && nact < LANES; j++) { + OneMmJob& job = *jobs[j]; + while(nact < LANES && job.nextw < job.walks.size()) { + const OneMmWalk& w = job.walks[job.nextw]; + if(!w.done) { + const Ebwt& e = *job.dirs[w.dir].ebwt; + if(GROUP) { + INIT_LOCS_NOPF(w.top, w.bot, tlocs[nact], blocs[nact], e); + ebs[nact] = e.ebwt(); } else { - t[clo] = top; - b[clo] = bot = top + 1; + INIT_LOCS(w.top, w.bot, tlocs[nact], blocs[nact], e); } - bp[clo] = botp; - tp[clo] = topp; - assert(!rep1mm || bot - top == botp - topp); - assert(!rep1mm || b[clo] - t[clo] == bp[clo] - tp[clo]); - chi = clo; + actj[nact] = &job; + act[nact++] = job.nextw; } - //assert(sanityPartial(ebwt, ebwtp, seq, len - dep - 1, len, rep1mm, top, bot, topp, botp)); - if(rep1mm && (ns == 0 || rdc > 3)) { - for(int j = clo; j <= chi; j++) { - if(j == rdc || b[j] == t[j]) { - // Either matches read or isn't a possibility - continue; - } - // Potential mismatch - next, try - size_t depm = dep + 1; - TIndexOffU topm = t[j], botm = b[j]; - TIndexOffU topmp = tp[j], botmp = bp[j]; - assert_eq(botm - topm, botmp - topmp); - TIndexOffU tm[4], bm[4]; // dest BW ranges for BWT - tm[0] = t[0]; tm[1] = t[1]; - tm[2] = t[2]; tm[3] = t[3]; - bm[0] = b[0]; bm[1] = t[1]; - bm[2] = b[2]; bm[3] = t[3]; - TIndexOffU tmp[4], bmp[4]; // dest BW ranges for BWT' - tmp[0] = tp[0]; tmp[1] = tp[1]; - tmp[2] = tp[2]; tmp[3] = tp[3]; - bmp[0] = bp[0]; bmp[1] = tp[1]; - bmp[2] = bp[2]; bmp[3] = tp[3]; - SideLocus tlocm, blocm; - INIT_LOCS(topm, botm, tlocm, blocm, *ebwt); - for(; depm < len; depm++) { - int rdcm = seq[len - depm - 1]; - tmp[0] = tmp[1] = tmp[2] = tmp[3] = topmp; - bmp[0] = bmp[1] = bmp[2] = bmp[3] = botmp; - if(blocm.valid()) { - bwops_++; - tm[0] = tm[1] = tm[2] = tm[3] = - bm[0] = bm[1] = bm[2] = bm[3] = 0; - ebwt->mapBiLFEx(tlocm, blocm, tm, bm, tmp, bmp); - SANITY_CHECK_4TUP(tm, bm, tmp, bmp); - topm = tm[rdcm]; botm = bm[rdcm]; - topmp = tmp[rdcm]; botmp = bmp[rdcm]; - if(botm <= topm) { - break; - } - } else { - assert_eq(botm, topm+1); - assert_eq(botmp, topmp+1); - bwops_++; - topm = ebwt->mapLF1(topm, tlocm, rdcm); - if(topm == OFF_MASK) { - break; - } - botm = topm + 1; - // topp/botp stay the same - } - INIT_LOCS(topm, botm, tlocm, blocm, *ebwt); - } - if(depm == len) { - // Success; this is a 1MM hit - size_t off5p = dep; // offset from 5' end of read - size_t offstr = dep; // offset into patFw/patRc - if(fw == ebwtfw) { - off5p = len - off5p - 1; - } - if(!ebwtfw) { - offstr = len - offstr - 1; - } - Edit e((uint32_t)off5p, j, rdc, EDIT_TYPE_MM, false); - results = true; - int64_t score = (len - 1) * sc.match(); - // In --local mode, need to double-check that - // end-to-end alignment doesn't violate local - // alignment principles. Specifically, it - // shouldn't to or below 0 anywhere in the middle. - int pen = sc.score(rdc, (int)(1 << j), quc - 33); - score += pen; - bool valid = true; - if(local) { - int64_t locscore_fw = 0, locscore_bw = 0; - for(size_t i = 0; i < len; i++) { - if(i == dep) { - if(locscore_fw + pen <= 0) { - valid = false; - break; - } - locscore_fw += pen; - } else { - locscore_fw += sc.match(); - } - if(len-i-1 == dep) { - if(locscore_bw + pen <= 0) { - valid = false; - break; - } - locscore_bw += pen; - } else { - locscore_bw += sc.match(); - } - } - } - if(valid) { - valid = score >= minsc; - } - if(valid) { -#ifndef NDEBUG - BTDnaString& rf = tmprfdnastr_; - rf.clear(); - edits_.clear(); - edits_.push_back(e); - if(!fw) Edit::invertPoss(edits_, len, false); - Edit::toRef(fw ? read.patFw : read.patRc, edits_, rf); - if(!fw) Edit::invertPoss(edits_, len, false); - assert_eq(len, rf.length()); - for(size_t i = 0; i < len; i++) { - assert_lt((int)rf[i], 4); - } - ASSERT_ONLY(TIndexOffU toptmp = 0); - ASSERT_ONLY(TIndexOffU bottmp = 0); - assert(ebwtFw->contains(rf, &toptmp, &bottmp)); -#endif - TIndexOffU toprep = ebwtfw ? topm : topmp; - TIndexOffU botrep = ebwtfw ? botm : botmp; - assert_eq(toprep, toptmp); - assert_eq(botrep, bottmp); - hits.add1mmEe(toprep, botrep, &e, NULL, fw, score); - } + job.nextw++; + } + } + if(nact == 0) { + break; + } + if(GROUP) { + // Every lane's loci were set, without prefetches, by its step + // in the last round or by the start of its walk above + for(size_t k = 0; k < nact; k++) { + SIDE_LOCI_PREFETCH(tlocs[k], blocs[k], ebs[k]); + } + } + for(size_t k = 0; k < nact;) { + OneMmJob& job = *actj[k]; + const size_t i = act[k]; + const bool mm = job.walks[i].mm; + if(mm ? oneMmStepMm(job, i, tlocs[k], blocs[k], !GROUP) : + oneMmStepMain(job, i, tlocs[k], blocs[k], !GROUP)) + { + k++; + } else { + job.walks[i].done = true; + if(!mm) { + job.mainLeft--; + } + nact--; + actj[k] = actj[nact]; + act[k] = act[nact]; + tlocs[k] = tlocs[nact]; + blocs[k] = blocs[nact]; + if(GROUP) { + ebs[k] = ebs[nact]; + } + } + } + } +} + +/** + * Report the hits of a job whose walks have all ended, in the order of a + * one-walk-at-a-time search: by direction, then mismatch position, then + * character. Mismatch walks are made in that order within a direction. + */ +bool SeedAligner::oneMmReport( + const OneMmJob& job, + const Ebwt* ebwtFw, // BWT index + const Scoring& sc, // scoring + int64_t minsc, // minimum score + bool local, // 1mm hits must be legal local alignments + SeedResults& hits) // holds all the seed hits (and exact hit) +{ + const Read& read = *job.read; + (void)read; + (void)ebwtFw; // read only by the debug checks + const size_t len = job.len; + const int ndir = job.ndir; + const bool repex = job.repex; + (void)repex; + bool results = false; + for(int di = 0; di < ndir; di++) { + const OneMmDir& d = job.dirs[di]; + const bool fw = d.fw; + const bool ebwtfw = d.ebwtfw; + for(size_t i = (size_t)ndir; i < job.walks.size(); i++) { + const OneMmWalk& w = job.walks[i]; + if(w.dir != di || !w.hit) { + continue; + } + assert(w.mm); + const size_t dep = w.mmdep; + const int j = w.mmc; + const int rdc = w.rdc; + const int quc = w.quc; + const TIndexOffU topm = w.top, botm = w.bot; + const TIndexOffU topmp = w.topp, botmp = w.botp; + // Success; this is a 1MM hit + size_t off5p = dep; // offset from 5' end of read + if(fw == ebwtfw) { + off5p = len - off5p - 1; + } + Edit e((uint32_t)off5p, j, rdc, EDIT_TYPE_MM, false); + results = true; + int64_t score = (len - 1) * sc.match(); + // In --local mode, need to double-check that + // end-to-end alignment doesn't violate local + // alignment principles. Specifically, it + // shouldn't to or below 0 anywhere in the middle. + int pen = sc.score(rdc, (int)(1 << j), quc - 33); + score += pen; + bool valid = true; + if(local) { + int64_t locscore_fw = 0, locscore_bw = 0; + for(size_t i = 0; i < len; i++) { + if(i == dep) { + if(locscore_fw + pen <= 0) { + valid = false; + break; } + locscore_fw += pen; + } else { + locscore_fw += sc.match(); } - } - if(bot > top && match) { - assert_lt(rdc, 4); - if(dep == len-1) { - // Success; this is an exact hit - if(ebwtfw && repex) { - if(fw) { - results = true; - int64_t score = len * sc.match(); - hits.addExactEeFw( - ebwtfw ? top : topp, - ebwtfw ? bot : botp, - NULL, NULL, fw, score); - assert(ebwtFw->contains(seq, NULL, NULL)); - } else { - results = true; - int64_t score = len * sc.match(); - hits.addExactEeRc( - ebwtfw ? top : topp, - ebwtfw ? bot : botp, - NULL, NULL, fw, score); - assert(ebwtFw->contains(seq, NULL, NULL)); - } + if(len-i-1 == dep) { + if(locscore_bw + pen <= 0) { + valid = false; + break; } - break; // End of far loop + locscore_bw += pen; } else { - INIT_LOCS(top, bot, tloc, bloc, *ebwt); - assert(sanityPartial(ebwt, ebwtp, seq, len - dep - 1, len, rep1mm, top, bot, topp, botp)); + locscore_bw += sc.match(); } - } else { - break; // End of far loop } - } // for(; dep < len; dep++) - } // for(int ebwtfw = 0; ebwtfw < 2; ebwtfw++) - } // for(int fw = 0; fw < 2; fw++) + } + if(valid) { + valid = score >= minsc; + } + if(valid) { +#ifndef NDEBUG + BTDnaString& rf = tmprfdnastr_; + rf.clear(); + edits_.clear(); + edits_.push_back(e); + if(!fw) Edit::invertPoss(edits_, len, false); + Edit::toRef(fw ? read.patFw : read.patRc, edits_, rf); + if(!fw) Edit::invertPoss(edits_, len, false); + assert_eq(len, rf.length()); + for(size_t i = 0; i < len; i++) { + assert_lt((int)rf[i], 4); + } + ASSERT_ONLY(TIndexOffU toptmp = 0); + ASSERT_ONLY(TIndexOffU bottmp = 0); + assert(ebwtFw->contains(rf, &toptmp, &bottmp)); +#endif + TIndexOffU toprep = ebwtfw ? topm : topmp; + TIndexOffU botrep = ebwtfw ? botm : botmp; + assert_eq(toprep, toptmp); + assert_eq(botrep, bottmp); + hits.add1mmEe(toprep, botrep, &e, NULL, fw, score); + } + } + if(d.exact) { + assert(ebwtfw && repex); + const BTDnaString& seq = *d.seq; + (void)seq; + results = true; + int64_t score = len * sc.match(); + if(fw) { + hits.addExactEeFw(d.etop, d.ebot, NULL, NULL, fw, score); + assert(ebwtFw->contains(seq, NULL, NULL)); + } else { + hits.addExactEeRc(d.etop, d.ebot, NULL, NULL, fw, score); + assert(ebwtFw->contains(seq, NULL, NULL)); + } + } + } return results; } +/** + * Search for end-to-end alignments with up to 1 mismatch. Return true iff + * one is found. + * + * There are up to four directions: each strand of the read, matched + * right-to-left in the forward index and, when looking for 1-mismatch hits, + * in the mirror index. Each direction has a main walk that matches the read + * exactly, and every mismatch the main walk tries in the far half starts + * another walk that matches the rest of the read exactly. These walks are + * independent of each other, and each LF step waits on a load from a random + * place in the index. So the walks are stepped in turn, one LF step each: + * each step prefetches the sides its walk needs next, and the steps of the + * other walks hide that load. Every walk takes the same steps as it would + * alone, and the hits are reported afterwards in the order in which a + * one-walk-at-a-time search finds them. + */ +bool SeedAligner::oneMmSearch( + const Ebwt* ebwtFw, // BWT index + const Ebwt* ebwtBw, // BWT' index + const Read& read, // read to align + const Scoring& sc, // scoring + int64_t minsc, // minimum score + bool nofw, // don't align forward read + bool norc, // don't align revcomp read + bool local, // 1mm hits must be legal local alignments + bool repex, // report 0mm hits? + bool rep1mm, // report 1mm hits? + SeedResults& hits, // holds all the seed hits (and exact hit) + SeedSearchMetrics& met) // metrics +{ + OneMmJob& job = mmJob_; + job.init(&read, nofw, norc); + if(!oneMmStart(ebwtFw, ebwtBw, sc, repex, rep1mm, job)) { + return false; + } + OneMmJob* jobs[1] = { &job }; + oneMmRun(jobs, 1); + bwops_ += job.bwops; + return oneMmReport(job, ebwtFw, sc, minsc, local, hits); +} + +/** + * oneMmSearch's walks for many reads at once. oneMmSearch steps the walks + * of one read in turn, but most reads have only two or three walks going at + * a time. Here the walks of all the jobs are stepped in turn in + * ONEMM_BATCH_LANES lanes. Each walk takes the same steps as in + * oneMmSearch, so each job ends with the walks, hits and LF operations + * oneMmSearch has for its read; oneMmSearchApply reports them. + */ +void SeedAligner::oneMmSearchBatch( + const Ebwt* ebwtFw, // BWT index + const Ebwt* ebwtBw, // BWT' index + const Scoring& sc, // scoring + bool repex, // report 0mm hits? + bool rep1mm, // report 1mm hits? + OneMmJob* const* jobs, // the reads to search + size_t njobs) // # jobs +{ + for(size_t j = 0; j < njobs; j++) { + oneMmStartPrefetch(ebwtFw, ebwtBw, rep1mm, *jobs[j]); + } + for(size_t j = 0; j < njobs; j++) { + oneMmStart(ebwtFw, ebwtBw, sc, repex, rep1mm, *jobs[j]); + } + oneMmRun(jobs, njobs); +} + inline void SeedAligner::prefetchNextLocsBi( const InstantiatedSeed& seed, // current instantiated seed diff --git a/aligner_seed.h b/aligner_seed.h index 3459771e..dace2df9 100644 --- a/aligner_seed.h +++ b/aligner_seed.h @@ -47,6 +47,9 @@ * * The constraint can put both caps and ceilings on the number and * types of edits allowed. + * + * A field added here must also be compared in sameConstraint() + * (aligner_seed.cpp), which keys cached seed instantiations. */ struct Constraint { @@ -318,6 +321,9 @@ struct InstantiatedSeed; * * If the read is shorter than a single seed, one seed is used and it * is shrunk to accommodate the read. + * + * A field added here must also be compared in sameSeed() + * (aligner_seed.cpp), which keys cached seed instantiations. */ struct Seed { @@ -410,7 +416,7 @@ struct Seed { */ struct InstantiatedSeed { - InstantiatedSeed() : steps(AL_CAT), zones(AL_CAT) { } + InstantiatedSeed() : steps(AL_CAT), zones(AL_CAT), instGen(0) { } // Steps map. There are as many steps as there are positions in // the seed. The map is a helpful abstraction because we sometimes @@ -464,6 +470,12 @@ struct InstantiatedSeed { // Seed this was instantiated from Seed s; + + // Generation of the SeedInstKey whose instantiation this object holds + // (steps, zones, cons, overall, maxjump and s as Seed::instantiate + // writes them for a seed without Ns), or 0 if it holds none. Set by + // SeedAligner::instantiateSeeds only. + uint64_t instGen; #ifndef NDEBUG /** @@ -475,6 +487,37 @@ struct InstantiatedSeed { #endif }; +/** + * The inputs to the part of Seed::instantiate that is the same for every + * seed without Ns: the seed policy (length, type, zone constraints and the + * overall-constraint pointer), the overall constraint it points to, and the + * read length. Every distinct value set gets a new generation number, + * unique in the process. An InstantiatedSeed whose instGen equals gen + * holds Seed::instantiate's output for these inputs, so instantiating a + * seed without Ns into it again only has to set its offset fields. + */ +struct SeedInstKey { + + SeedInstKey() : rdlen(0), gen(0) { } + + /** + * Return true iff 'sd', the constraint it points to and 'rdl' are + * the inputs this key was set from. + */ + bool matches(const Seed& sd, size_t rdl) const; + + /** + * Set the key from 'sd', the constraint it points to and 'rdl', and + * take a new generation number. + */ + void set(const Seed& sd, size_t rdl); + + Seed seed; // the seed policy + Constraint overall; // *seed.overall + size_t rdlen; // read length + uint64_t gen; // 0: never set +}; + /** * Simple struct for holding a end-to-end alignments for the read with at most * 2 edits. @@ -626,10 +669,17 @@ class SeedResults { offIdx2off_(AL_CAT), rankOffs_(AL_CAT), rankFws_(AL_CAT), + rankCnt_(AL_CAT), mm1Hit_(AL_CAT) { clear(); } + + /** + * Keys of the seed instantiations held in this object's + * InstantiatedSeed lists, one per seed policy (see SeedInstKey). + */ + std::vector& instKeys() { return instKeys_; } /** * Set the current read. @@ -1009,6 +1059,22 @@ class SeedResults { } #endif + /** + * The element count rankSeedHits ranks a QVal by: its count if it is + * valid, non-empty and not ranked yet (held as at most 'none'), else + * 'none'. + */ + static uint32_t rankCount(const QVal& qv, bool sorted, uint32_t none) { + if(!qv.valid() || sorted) { + return none; + } + TIndexOffU ne = qv.numElts(); + if(ne == 0 || ne >= (TIndexOffU)none) { + return none; + } + return (uint32_t)ne; + } + /** * Populate rankOffs_ and rankFws_ with the list of QVals that need to be * examined for this SeedResults, in order. The order is ascending by @@ -1037,8 +1103,23 @@ class SeedResults { rankFws_.push_back(false); } } else { + // Each pick draws a coin for which strand is scanned first, then + // per strand a random start and a circular scan for the first + // strictly smaller element count. The scans read a compact array + // of counts in which a QVal that is invalid, empty or already + // ranked holds MAX_U32, which is never below the running minimum + // (it starts at MAX_U32); counts at or above MAX_U32 are held as + // MAX_U32 for the same reason. Same draws, same picks. + const uint32_t none = MAX_U32; + const uint32_t n = (uint32_t)numOffs_; + rankCnt_.resize(2 * (size_t)n); + uint32_t* const cnt = rankCnt_.ptr(); // fw [0, n), rc [n, 2n) + for(uint32_t i = 0; i < n; i++) { + cnt[i] = rankCount(hitsFw_[i], sortedFw_[i], none); + cnt[n + i] = rankCount(hitsRc_[i], sortedRc_[i], none); + } while(rankOffs_.size() < nonzTot_) { - TIndexOffU minsz = MAX_U32; + uint32_t minsz = none; uint32_t minidx = 0; bool minfw = true; // Rank seed-hit positions in ascending order by number of elements @@ -1047,25 +1128,29 @@ class SeedResults { assert(rb == 0 || rb == 1); for(int fwi = 0; fwi <= 1; fwi++) { bool fw = (fwi == (rb ? 1 : 0)); - EList& rrs = (fw ? hitsFw_ : hitsRc_); - EList& sorted = (fw ? sortedFw_ : sortedRc_); - uint32_t i = (rnd.nextU32() % (uint32_t)numOffs_); - for(uint32_t ii = 0; ii < numOffs_; ii++) { - if(rrs[i].valid() && // valid QVal - rrs[i].numElts() > 0 && // non-empty - !sorted[i] && // not already sorted - rrs[i].numElts() < minsz) // least elts so far? - { - minsz = rrs[i].numElts(); - minidx = i; - minfw = (fw == 1); - } - if((++i) == numOffs_) { - i = 0; - } + const uint32_t* c = cnt + (fw ? 0 : n); + uint32_t i = (rnd.nextU32() % n); + // First strict minimum of this strand in circular order + // from i: a later equal count does not replace it + uint32_t sm = none, si = 0; + for(uint32_t j = i; j < n; j++) { + const bool lt = c[j] < sm; + sm = lt ? c[j] : sm; + si = lt ? j : si; + } + for(uint32_t j = 0; j < i; j++) { + const bool lt = c[j] < sm; + sm = lt ? c[j] : sm; + si = lt ? j : si; + } + if(sm < minsz) { + minsz = sm; + minidx = si; + minfw = fw; } } - assert_neq(MAX_U32, minsz); + assert_neq(none, minsz); + cnt[(minfw ? 0 : n) + minidx] = none; if(minfw) { sortedFw_[minidx] = true; } else { @@ -1370,6 +1455,7 @@ class SeedResults { // are ignored. EList rankOffs_; // sorted offests of seeds to try EList rankFws_; // sorted orientations assoc. with rankOffs_ + EList rankCnt_; // rankSeedHits scratch: counts, fw then rc bool sorted_; // true if sort() called since last reset // These fields set once per read @@ -1381,6 +1467,7 @@ class SeedResults { EList mm1Hit_; // 1-mismatch end-to-end hits size_t mm1Elt_; // number of 1-mismatch hit rows bool mm1Sorted_; // true iff we've sorted the mm1Hit_ list + std::vector instKeys_; // per seed policy; kept across reads EList tmpMedian_; // temporary storage for calculating median }; @@ -1457,23 +1544,38 @@ struct SeedSearchMetrics { /** * Wrap the search cache with all the relevant objects + * + * Reused across seeds and reads (see SeedSearchMultiCache): reset() points + * it at a new seed and empties it. */ class SeedSearchCache { public: - SeedSearchCache( - const BTDnaString& _seq, // sequence of current seed - const BTString& _qual // quality string for current seed - ) + SeedSearchCache() : qv() - , seq(_seq) - , qual(_qual) + , seq(NULL) + , qual(NULL) , cachedEls() , cachep(NULL) { cachedEls.reserve(16); // do not expect I will need more } + /** + * Point this cache at a new seed and forget everything about the + * previous one. + */ + void reset( + const BTDnaString* _seq, // sequence of current seed + const BTString* _qual) // quality string for current seed + { + qv.reset(); + seq = _seq; + qual = _qual; + cachedEls.clear(); + cachep = NULL; + } + /** * This function is called whenever we start to align a new read or * read substring. @@ -1482,7 +1584,7 @@ class SeedSearchCache { */ int beginAlign(AlignmentCacheIface& cache) { - int ret = cache.beginAlign(seq, qual, qv); + int ret = cache.beginAlign(*seq, *qual, qv); if (ret>=0) { cachep = &cache; } @@ -1502,7 +1604,7 @@ class SeedSearchCache { AddEl &el = cachedEls[i]; success &= cachep->addOnTheFly(el.sak, el.topf, el.botf, el.topb, el.botb, getLock); } - cachedEls.clear(); + // cachedEls is emptied by the next reset() return success; } @@ -1545,8 +1647,8 @@ class SeedSearchCache { bool qvValid() const { return qv.valid();} const QVal& getQv() const {return qv;} - const BTDnaString& getSeq() const {return seq;} - const BTString& getQual() const {return qual;} + const BTDnaString& getSeq() const {return *seq;} + const BTString& getQual() const {return *qual;} protected: class AddEl { @@ -1572,39 +1674,47 @@ class SeedSearchCache { }; QVal qv; - const BTDnaString& seq; // sequence of current seed - const BTString& qual; // quality string for current seed + const BTDnaString* seq; // sequence of current seed + const BTString* qual; // quality string for current seed std::vector cachedEls; // tmp storage of values that will go in the cache AlignmentCacheIface* cachep; // local alignment cache for seed alignment, set at beginAliginings }; /** - * Wrap the search cache with all the relevant objects + * The SeedSearchCaches of one searchAllSeeds call, in search order. The + * caches are kept, with their buffers, across calls: clear() only forgets + * how many are in use. */ class SeedSearchMultiCache { public: SeedSearchMultiCache( ) - : cacheVec() + : cacheVec(), n_(0) {} - void emplace_back( + /** + * Take the next cache and point it at the given seed. May move the + * caches in memory, so take every cache before holding references to + * any of them. + */ + void add( const BTDnaString& seq, // sequence of current seed const BTString& qual, // quality string for current seed int seedoffidx, // seed index bool fw // is it fw? ) { - cacheVec.emplace_back(seq, qual, seedoffidx, fw); + if(n_ == cacheVec.size()) cacheVec.emplace_back(); + CacheEl& el = cacheVec[n_++]; + el.srcache.reset(&seq, &qual); + el.seedoffidx = seedoffidx; + el.fw = fw; } - // Same semantics as std::vector - void reserve(size_t new_cap) { cacheVec.reserve(new_cap); } - size_t size() const {return cacheVec.size(); } - void clear() { cacheVec.clear(); } - void pop_back() { cacheVec.pop_back(); } + size_t size() const {return n_; } + void clear() { n_ = 0; } // Access one of the search caches const SeedSearchCache& operator[](size_t idx) const { return cacheVec[idx].srcache; } @@ -1616,16 +1726,7 @@ class SeedSearchMultiCache { protected: class CacheEl { public: - CacheEl( - const BTDnaString& _seq, // sequence of current seed - const BTString& _qual, // quality string for current seed - int _seedoffidx, // seed index - bool _fw // is it fw? - ) - : srcache(_seq, _qual) - , seedoffidx(_seedoffidx) - , fw(_fw) {} - + CacheEl() : srcache(), seedoffidx(0), fw(false) {} SeedSearchCache srcache; // search wrapper int seedoffidx; // seed index @@ -1633,13 +1734,164 @@ class SeedSearchMultiCache { }; std::vector cacheVec; + size_t n_; // caches in use }; +// How many seed offsets (both strands together) SeedAligner::searchAllSeeds +// searches in one searchSeedBi batch +static const size_t SEED_BATCH_CACHES = 32; + // Per-parameter scratch space for SeedAligner::searchSeedBi. Defined in // aligner_seed.cpp; the caller owns the storage so that neither the batched // nor the recursive entry point has to allocate. class SeedAlignerSearchState; +/** + * One walk of SeedAligner::oneMmSearch: the main walk of one direction, which + * matches the read exactly, or a walk that matches the rest of the read + * exactly after a mismatch. + */ +struct OneMmWalk { + + void init( + int dir_, + bool mm_, + TIndexOffU top_, + TIndexOffU bot_, + TIndexOffU topp_, + TIndexOffU botp_, + size_t dep_) + { + top = top_; bot = bot_; + topp = topp_; botp = botp_; + dep = (uint32_t)dep_; + mmdep = 0; + dir = (uint8_t)dir_; + mm = mm_; + mmc = rdc = 0; + quc = 0; + done = hit = false; + } + + TIndexOffU top, bot; // range in the index walked + TIndexOffU topp, botp; // range in the other index + uint32_t dep; // depth of the next read character to match + uint32_t mmdep; // mismatch walk: depth of the mismatch + int quc; // mismatch walk: quality at the mismatch + uint8_t dir; // index of the direction + bool mm; // a mismatch walk (else a main walk) + int8_t mmc; // mismatch walk: reference character + int8_t rdc; // mismatch walk: read character + bool done; // the walk has ended + bool hit; // mismatch walk: matched to the end of the read +}; + +// How many walks SeedAligner::oneMmSearch steps in turn +static const size_t ONEMM_WALKS = 16; + +// How many walks SeedAligner::oneMmSearchBatch steps in turn +#ifndef ONEMM_BATCH_LANES +#define ONEMM_BATCH_LANES 32 +#endif + +/** + * One direction of a 1-mismatch search: a strand of the read matched + * right-to-left in the forward index or in the mirror index. + */ +struct OneMmDir { + const Ebwt* ebwt; // index walked + const Ebwt* ebwtp; // the other index + const BTDnaString* seq; + const BTString* qual; + size_t nea; // length of the near half + bool fw; // strand of the read + bool ebwtfw; // ebwt is the forward index + bool exact; // the main walk matched the whole read + TIndexOffU etop, ebot; // its range in ebwt +}; + +/** + * The 1-mismatch end-to-end search of one read (one mate): its inputs, its + * directions and walks, and the LF operations they took. oneMmSearch runs + * one; oneMmSearchBatch runs many with their walks stepped in turn, and + * oneMmSearchApply then reports a job's hits as oneMmSearch would have. + */ +struct OneMmJob { + + void init(const Read* read_, bool nofw_, bool norc_) { + read = read_; + nofw = nofw_; + norc = norc_; + repex = false; + rep1mm = true; + len = 0; + nceil = 0; + ns = 0; + ndir = 0; + walks.clear(); + nextw = 0; + mainLeft = 0; + bwops = 0; + ftabPre = false; + } + + const Read* read; // read to align + bool nofw; // don't align the read + bool norc; // don't align its reverse complement + bool repex; // report 0mm hits? + bool rep1mm; // report 1mm hits? + size_t len; // read length + int nceil; // N ceiling for the read length + size_t ns; // Ns in the read + OneMmDir dirs[4]; // directions, in search order + int ndir; // # directions; their main walks are walks[0, ndir) + EList walks;// all walks, in the order they were made + size_t nextw; // next walk to start + size_t mainLeft; // main walks that have not ended + uint64_t bwops; // Burrows-Wheeler operations + // ftab indices of the directions, computed (and their entries + // prefetched) by oneMmStartPrefetch before oneMmStart reads them; + // direction (fw, ebwtfw) is slot 2 * !fw + !ebwtfw + bool ftabPre; // fi/fip are set + bool ftabPreRep1mm; // the rep1mm they were set for + TIndexOffU fi[4]; // ftab index in the direction's index + TIndexOffU fip[4]; // ftab index in the other index (rep1mm only) +}; + +/** + * The exact end-to-end sweep of one read (one mate), as exactSweep does it, + * for SeedAligner::exactSweepBatch: its inputs, and the results exactSweep + * gives for them. SeedAligner::exactSweepApply hands the results over. + */ +struct ExactSweepJob { + + void init(const Read* read_, bool nofw_, bool norc_) { + read = read_; + nofw = nofw_; + norc = norc_; + mine[0] = mine[1] = 0; + nelt = 0; + bwops = 0; + hit[0] = hit[1] = false; + top[0] = top[1] = bot[0] = bot[1] = 0; + score[0] = score[1] = 0; + } + + const Read* read; // read to sweep + bool nofw; // don't sweep the read + bool norc; // don't sweep its reverse complement + size_t mine[2]; // minimum # edits: read, reverse complement + size_t nelt; // # exact hit elements (exactSweep's return value) + uint64_t bwops; // Burrows-Wheeler operations + bool hit[2]; // exact hit: read, reverse complement + TIndexOffU top[2]; // exact hit ranges + TIndexOffU bot[2]; + int64_t score[2]; // exact hit scores +}; + +// How many sweeps SeedAligner::exactSweepBatch steps in turn +static const size_t EXACT_SWEEP_LANES = 16; + /** * Given an index and a seeding scheme, searches for seed hits. */ @@ -1650,14 +1902,21 @@ class SeedAligner { /** * Initialize with index. */ - SeedAligner() : edits_(AL_CAT), offIdx2off_(AL_CAT) { } + SeedAligner() : edits_(AL_CAT), offIdx2off_(AL_CAT), sss_(NULL) { } + + ~SeedAligner(); + + // Owns its seed-search scratch (sss_) + SeedAligner(const SeedAligner&) = delete; + SeedAligner& operator=(const SeedAligner&) = delete; /** * Given a read and a few coordinates that describe a substring of the * read (or its reverse complement), fill in 'seq' and 'qual' objects - * with the seed sequence and qualities. + * with the seed sequence and qualities. Return true iff the seed + * sequence contains an N. */ - void instantiateSeq( + bool instantiateSeq( const Read& read, // input read BTDnaString& seq, // output sequence BTString& qual, // output qualities @@ -1731,6 +1990,43 @@ class SeedAligner { SeedResults& hits, // holds all the seed hits (and exact hit) SeedSearchMetrics& met); // metrics + /** + * exactSweep for many reads at once, with up to EXACT_SWEEP_LANES of + * their sweeps stepped in turn. Each job gets the results exactSweep + * gives for its read; exactSweepApply hands them over. + */ + void exactSweepBatch( + const Ebwt& ebwt, // BWT index + const Scoring& sc, // scoring scheme + size_t mineMax, // don't care about edit bounds > this + bool repex, // report 0mm hits? + ExactSweepJob* const* jobs, // the reads to sweep + size_t njobs); // # jobs + + /** + * Hand over the results of a job swept by exactSweepBatch exactly as + * exactSweep would have produced them: the edit bounds of the strands + * swept, the exact hits, the LF operations; returns exactSweep's return + * value. + */ + size_t exactSweepApply( + const ExactSweepJob& job, + size_t& mineFw, // minimum # edits for forward read + size_t& mineRc, // minimum # edits for revcomp read + SeedResults& hits) // holds all the seed hits (and exact hit) + { + if(!job.nofw) mineFw = job.mine[0]; + if(!job.norc) mineRc = job.mine[1]; + if(job.hit[0]) { + hits.addExactEeFw(job.top[0], job.bot[0], NULL, NULL, true, job.score[0]); + } + if(job.hit[1]) { + hits.addExactEeRc(job.top[1], job.bot[1], NULL, NULL, false, job.score[1]); + } + bwops_ += job.bwops; + return job.nelt; + } + /** * Search for end-to-end alignments with up to 1 mismatch. */ @@ -1748,7 +2044,87 @@ class SeedAligner { SeedResults& hits, // holds all the seed hits (and exact hit) SeedSearchMetrics& met); // metrics + /** + * The walks of oneMmSearch for many reads at once, with up to + * ONEMM_BATCH_LANES of them stepped in turn. Each job must have been + * initialized with its read and strands; it then holds the walks + * oneMmSearch takes for that read, and oneMmSearchApply reports them. + */ + void oneMmSearchBatch( + const Ebwt* ebwtFw, // BWT index + const Ebwt* ebwtBw, // BWT' index + const Scoring& sc, // scoring + bool repex, // report 0mm hits? + bool rep1mm, // report 1mm hits? + OneMmJob* const* jobs, // the reads to search + size_t njobs); // # jobs + + /** + * Compute the ftab indices oneMmStart reads for the job's directions, + * store them in the job and prefetch their ftab entries, so that + * starting a batch of searches does not wait on them one at a time. + */ + void oneMmStartPrefetch( + const Ebwt* ebwtFw, // BWT index + const Ebwt* ebwtBw, // BWT' index + bool rep1mm, // report 1mm hits? + OneMmJob& job); + + /** + * Hand over the results of a job searched by oneMmSearchBatch exactly + * as oneMmSearch would have produced them for the job's read, strands + * and repex/rep1mm and the given minimum score and local flag: the hits + * in the same order, and the LF operations; returns oneMmSearch's + * return value. + */ + bool oneMmSearchApply( + const OneMmJob& job, + const Ebwt* ebwtFw, // BWT index + const Scoring& sc, // scoring + int64_t minsc, // minimum score + bool local, // 1mm hits must be legal local alignments + SeedResults& hits) // holds all the seed hits (and exact hit) + { + bwops_ += job.bwops; + return oneMmReport(job, ebwtFw, sc, minsc, local, hits); + } + protected: + + /** + * Set up the directions and main walks of a 1-mismatch search; false + * if the read cannot have a hit (too many Ns), in which case the job + * has no walks. + */ + bool oneMmStart( + const Ebwt* ebwtFw, // BWT index + const Ebwt* ebwtBw, // BWT' index + const Scoring& sc, // scoring + bool repex, // report 0mm hits? + bool rep1mm, // report 1mm hits? + OneMmJob& job); + + /// One LF step of a main walk; false when the walk has ended. The loci + /// of the next step are prefetched unless 'prefetch' is false. + bool oneMmStepMain(OneMmJob& job, size_t i, SideLocus& tloc, SideLocus& bloc, bool prefetch); + + /// One LF step of a mismatch walk; false when the walk has ended + bool oneMmStepMm(OneMmJob& job, size_t i, SideLocus& tloc, SideLocus& bloc, bool prefetch); + + /// Step the walks of the jobs in turn, at most LANES at a time; GROUP: + /// each round's prefetches are issued together after the round's steps + template + void oneMmRun(OneMmJob* const* jobs, size_t njobs); + + /// Report a job's hits in the order of a one-walk-at-a-time search + bool oneMmReport( + const OneMmJob& job, + const Ebwt* ebwtFw, // BWT index + const Scoring& sc, // scoring + int64_t minsc, // minimum score + bool local, // 1mm hits must be legal local alignments + SeedResults& hits); // holds all the seed hits (and exact hit) + class SeedAlignerSearchParams; /** @@ -1852,7 +2228,13 @@ class SeedAligner { EList offIdx2off_;// offset idx to read offset map, set up instantiateSeeds() uint64_t bwops_; // Burrows-Wheeler operations uint64_t bwedits_; // Burrows-Wheeler edits + OneMmJob mmJob_; // the current oneMmSearch BTDnaString tmprfdnastr_; // used in reportHit + + // searchAllSeeds scratch, kept across calls + struct SeedSearchScratch; // its search params and states (aligner_seed.cpp) + SeedSearchMultiCache mcache_; // one SeedSearchCache per searched seed offset + SeedSearchScratch* sss_; // allocated on first use ASSERT_ONLY(ESet hits_); // Ref hits so far for seed being aligned BTDnaString tmpdnastr_; @@ -1868,6 +2250,20 @@ class SeedAligner { } \ } +/** + * INIT_LOCS without its prefetches; the caller issues them later with + * SIDE_LOCI_PREFETCH(tloc, bloc, (e).ebwt()), which prefetches the same lines. + */ +#define INIT_LOCS_NOPF(top, bot, tloc, bloc, e) { \ + if(bot - top == 1) { \ + tloc.initFromRow(top, (e).eh(), (e).ebwt(), false); \ + bloc.invalidate(); \ + } else { \ + SideLocus::initFromTopBot(top, bot, (e).eh(), (e).ebwt(), tloc, bloc, false); \ + assert(bloc.valid()); \ + } \ +} + #define SANITY_CHECK_4TUP(t, b, tp, bp) { \ ASSERT_ONLY(TIndexOffU tot = (b[0]-t[0])+(b[1]-t[1])+(b[2]-t[2])+(b[3]-t[3])); \ ASSERT_ONLY(TIndexOffU totp = (bp[0]-tp[0])+(bp[1]-tp[1])+(bp[2]-tp[2])+(bp[3]-tp[3])); \ diff --git a/aligner_sw.cpp b/aligner_sw.cpp index aa482676..363f8c5b 100644 --- a/aligner_sw.cpp +++ b/aligner_sw.cpp @@ -207,12 +207,13 @@ void SwAligner::initRef( // rfbuf_ = uint32_t list large enough to accommodate both the reference // sequence and any Ns we might add to either side. rfwbuf_.resize((rflen + 16) / 4); - int offset = refs.getStretch( + // Unpack the stretch straight to masks: 1 << c for a base c, 16 for N + int offset = refs.getStretchMasks( rfwbuf_.ptr(), // buffer to store words in refidx, // which reference (rfi < 0) ? 0 : (size_t)rfi, // starting offset (can't be < 0) rflenInner // length to grab (exclude overhang) - ASSERT_ONLY(, tmp_destU32_));// for BitPairReference::getStretch() + ASSERT_ONLY(, tmp_destU32_));// for BitPairReference::getStretchMasks() assert_leq(offset, 16); rf_ = (char*)rfwbuf_.ptr() + offset; // Shift ref chars away from 0 so we can stick Ns at the beginning @@ -223,33 +224,27 @@ void SwAligner::initRef( } // Add Ns for(size_t i = 0; i < leftNs; i++) { - rf_[i] = 4; + rf_[i] = 16; } } if(rightNs > 0) { // Add Ns to the end for(size_t i = 0; i < rightNs; i++) { - rf_[i + leftNs + rflenInner] = 4; + rf_[i + leftNs + rflenInner] = 16; } } #ifndef NDEBUG - // Sanity check reference characters + // Sanity check reference masks for(size_t i = 0; i < rflen; i++) { - assert(!haveRfbuf2 || rf_[i] == rfbuf2[i]); - assert_range(0, 4, (int)rf_[i]); + assert(!haveRfbuf2 || (int)rf_[i] == (1 << rfbuf2[i])); + assert(rf_[i] == 1 || rf_[i] == 2 || rf_[i] == 4 || rf_[i] == 8 || rf_[i] == 16); } #endif - // Count Ns and convert reference characters into A/C/G/T masks. Ambiguous - // nucleotides (IUPAC codes) have more than one mask bit set. If a - // reference scanner was provided, use it to opportunistically resolve seed - // hits. + // Count Ns (mask 16) among the first 'upto' reference characters nsUpto = 0; - for(size_t i = 0; i < rflen; i++) { - // rf_[i] gets mask version of refence char, with N=16 - if(i < upto && rf_[i] > 3) { - nsUpto++; - } - rf_[i] = (1 << rf_[i]); + const size_t nsLim = min(upto, rflen); + for(size_t i = 0; i < nsLim; i++) { + nsUpto += (rf_[i] == 16); } // Correct for having captured an extra reference character rff--; @@ -493,6 +488,84 @@ int SwAligner::ungappedAlign( return 1; } +/** + * Put btncand_ in the order btncand_.sort() gives it (std::stable_sort by + * DpBtCandidate::operator<: score descending, then row descending, then + * column descending). Called after a local-mode gather, whose candidates + * are in nondecreasing column order. + * + * For a list in that order, two stable counting sorts give the same order: + * the first by row, descending, reading the list from its end, so that the + * cells of one row come out in descending column order; the second by score, + * descending. Two elements end up in the opposite relative order to + * stable_sort's only if their row, column, score and fate are all equal, so + * the resulting list is the same. + * + * A short list is sorted by insertion sort, which is stable. A list that is + * not in column order, has a fate set, or has a row or score outside the + * rectangle and [minsc_, best] is sorted by btncand_.sort(). + */ +void SwAligner::sortLocalCandidates(TAlScore best) { + const size_t n = btncand_.size(); + DpBtCandidate *c = btncand_.ptr(); + if(n < 16) { + for(size_t i = 1; i < n; i++) { + if(c[i] < c[i-1]) { + DpBtCandidate x = c[i]; + size_t j = i; + do { + c[j] = c[j-1]; + j--; + } while(j > 0 && x < c[j-1]); + c[j] = x; + } + } + return; + } + const size_t nrow = dpRows(); + if(best < minsc_ || best - minsc_ >= 1024) { + btncand_.sort(); + return; + } + const size_t nsc = (size_t)(best - minsc_) + 1; + // Bucket counts: by row (nrow-1-row), then by score (best-score) + btncandhist_.resizeNoCopy(nrow + nsc); + btncandhist_.fillZero(); + uint32_t *hrow = btncandhist_.ptr(); + uint32_t *hsc = hrow + nrow; + size_t prevcol = 0; + for(size_t i = 0; i < n; i++) { + const DpBtCandidate& x = c[i]; + if(x.col < prevcol || x.fate != 0 || x.row >= nrow || + x.score < minsc_ || x.score > best) + { + btncand_.sort(); + return; + } + prevcol = x.col; + hrow[nrow - 1 - x.row]++; + hsc[(size_t)(best - x.score)]++; + } + // Turn the counts into each bucket's first position + uint32_t tot = 0; + for(size_t i = 0; i < nrow + nsc; i++) { + if(i == nrow) { + tot = 0; + } + uint32_t k = hrow[i]; + hrow[i] = tot; + tot += k; + } + btncandtmp_.resizeNoCopy(n); + DpBtCandidate *t = btncandtmp_.ptr(); + for(size_t i = n; i-- > 0;) { + t[hrow[nrow - 1 - c[i].row]++] = c[i]; + } + for(size_t i = 0; i < n; i++) { + c[hsc[(size_t)(best - t[i].score)]++] = t[i]; + } +} + /** * Align read 'rd' to reference using read & reference information given * last time init() was called. @@ -506,6 +579,7 @@ bool SwAligner::align( // Reset solutions lists btncand_.clear(); btncanddone_.clear(); + clearDoneMirrors(); btncanddoneSucc_ = btncanddoneFail_ = 0; best = std::numeric_limits::min(); sse8succ_ = sse16succ_ = false; @@ -719,7 +793,22 @@ bool SwAligner::align( } } if(!btncand_.empty()) { - btncand_.sort(); + if(sc_->monotone) { + btncand_.sort(); + } else { +#ifndef NDEBUG + cand_tmp_ = btncand_; + cand_tmp_.sort(); +#endif + sortLocalCandidates(best); +#ifndef NDEBUG + assert_eq(cand_tmp_.size(), btncand_.size()); + for(size_t i = 0; i < btncand_.size(); i++) { + assert(cand_tmp_[i] == btncand_[i]); + assert_eq(cand_tmp_[i].fate, btncand_[i].fate); + } +#endif + } } if(dpLog_ != NULL) { (*dpLog_) << ",1," << best; @@ -755,6 +844,9 @@ bool SwAligner::nextAlignment( if(SQ == 0) SQ = 1; size_t rdlen = rdf_ - rdi_; bool checkpointed = rdlen >= cperMinlen_; + // Rows and columns fit in 31 bits, so the domination test can use the + // 32-bit copies in btncanddonerow_ and btncanddonecol_ + const bool dom32 = dpRows() <= 0x7fffffff && (size_t)(rff_ - rfi_) <= 0x7fffffff; while(cural_ < candsz) { // Doing 'continue' anywhere in here simply causes us to move on to the // next candidate @@ -943,7 +1035,41 @@ bool SwAligner::nextAlignment( // majority of valid-but-redundant candidates lying in the // "penumbra" of a high-scoring alignment. bool dom = false; - { + if(dom32) { + // The same test as below, on every done candidate without + // an early exit: |row - row'| <= SQ and |col - col'| <= SQ, + // as (row' - (row - SQ)) mod 2^32 <= 2*SQ. + // btncanddonerow_/col_ are appended to with btncanddone_ + // and cleared with it in align(), and rebuilt if their + // count differs (btncanddone_ cleared elsewhere). They are + // padded with BT_DONE_SENTINEL to a multiple of 8, which + // never passes the row test, so the loop runs in blocks + // of 8 with no scalar remainder. + const size_t donesz = btncanddone_.size(); + if(btncanddonemir_ != donesz) { + clearDoneMirrors(); + for(size_t i = 0; i < donesz; i++) { + pushDoneMirrors(btncanddone_[i].row, btncanddone_[i].col); + } + } + const size_t n8 = btncanddonerow_.size(); + assert_eq(0, n8 & 7); + assert_geq(n8, donesz); + assert_lt(n8, donesz + 8); + const uint32_t *prow = btncanddonerow_.ptr(); + const uint32_t *pcol = btncanddonecol_.ptr(); + const uint32_t rlo = (uint32_t)row - (uint32_t)SQ; + const uint32_t clo = (uint32_t)col - (uint32_t)SQ; + const uint32_t w = 2 * (uint32_t)SQ; + uint32_t any = 0; + for(size_t i = 0; i < n8; i += 8) { + for(size_t j = 0; j < 8; j++) { + any |= (uint32_t)((prow[i + j] - rlo) <= w) & + (uint32_t)((pcol[i + j] - clo) <= w); + } + } + dom = (any != 0); + } else { size_t donesz = btncanddone_.size(); const size_t col = btncand_[cural_].col; const size_t row = btncand_[cural_].row; @@ -962,6 +1088,20 @@ bool SwAligner::nextAlignment( } } } +#ifndef NDEBUG + { + bool dom2 = false; + for(size_t i = 0; i < btncanddone_.size(); i++) { + const DpBtCandidate& o = btncanddone_[i]; + if((row > o.row ? row - o.row : o.row - row) <= SQ && + (col > o.col ? col - o.col : o.col - col) <= SQ) + { + dom2 = true; + } + } + assert_eq(dom, dom2); + } +#endif if(dom) { btncand_[cural_].fate = BT_CAND_FATE_FILT_DOMINATED; nbtfiltdo_++; @@ -1116,12 +1256,14 @@ bool SwAligner::nextAlignment( if(ret) { btncand_[cural_].fate = BT_CAND_FATE_SUCCEEDED; btncanddone_.push_back(btncand_[cural_]); + pushDoneMirrors(btncand_[cural_].row, btncand_[cural_].col); btncanddoneSucc_++; assert(res.repOk()); break; } else { btncand_[cural_].fate = BT_CAND_FATE_FAILED; btncanddone_.push_back(btncand_[cural_]); + pushDoneMirrors(btncand_[cural_].row, btncand_[cural_].col); btncanddoneFail_++; } } diff --git a/aligner_sw.h b/aligner_sw.h index 78440cbc..2f7fd70a 100644 --- a/aligner_sw.h +++ b/aligner_sw.h @@ -218,7 +218,12 @@ class SwAligner { btcells_(DP_CAT), btdiag_(), btncand_(DP_CAT), + btncandtmp_(DP_CAT), + btncandhist_(DP_CAT), btncanddone_(DP_CAT), + btncanddonerow_(DP_CAT), + btncanddonecol_(DP_CAT), + btncanddonemir_(0), btncanddoneSucc_(0), btncanddoneFail_(0), cper_(), @@ -494,6 +499,42 @@ class SwAligner { void buildQueryProfileEnd2EndSseU8(bool fw); void buildQueryProfileLocalSseU8(bool fw); + /** + * The row and column mirrors of btncanddone_ for nextAlignment's + * domination test hold btncanddonemir_ entries followed by + * BT_DONE_SENTINEL up to their size, a multiple of 8, so the test runs + * in whole blocks of 8. The sentinel is outside every domination window + * when rows, columns and the window are below 2^31 (nextAlignment's dom32 + * guard). + */ + static const uint32_t BT_DONE_SENTINEL = 0xC0000000u; + + /** + * Empty the mirrors; called where btncanddone_ is cleared. + */ + void clearDoneMirrors() { + btncanddonerow_.clear(); + btncanddonecol_.clear(); + btncanddonemir_ = 0; + } + + /** + * Append a done candidate's row and column to the mirrors. + */ + void pushDoneMirrors(size_t row, size_t col) { + size_t k = btncanddonemir_; + if(k == btncanddonerow_.size()) { + btncanddonerow_.resize(k + 8); + btncanddonecol_.resize(k + 8); + for(size_t i = k; i < k + 8; i++) { + btncanddonerow_[i] = btncanddonecol_[i] = BT_DONE_SENTINEL; + } + } + btncanddonerow_[k] = (uint32_t)row; + btncanddonecol_[k] = (uint32_t)col; + btncanddonemir_ = k + 1; + } + /** * Build query profile look up tables for the read. The query profile look * up table is organized as a 1D array indexed by [i][j] where i is the @@ -506,6 +547,8 @@ class SwAligner { bool gatherCellsNucleotidesLocalSseU8(TAlScore best); bool gatherCellsNucleotidesEnd2EndSseU8(TAlScore best); + void sortLocalCandidates(TAlScore best); + bool gatherCellsNucleotidesLocalSseI16(TAlScore best); bool gatherCellsNucleotidesEnd2EndSseI16(TAlScore best); @@ -626,7 +669,13 @@ class SwAligner { NBest btdiag_; // per-diagonal backtrace candidates EList btncand_; // cells we might backtrace from + EList btncandtmp_; // scratch for sortLocalCandidates + EList btncandhist_; // bucket counts for sortLocalCandidates EList btncanddone_; // candidates that we investigated + EList btncanddonerow_; // btncanddone_'s rows and columns + EList btncanddonecol_; // in 32 bits, for nextAlignment; + // padded to a multiple of 8 with BT_DONE_SENTINEL + size_t btncanddonemir_; // # of btncanddone_ in the mirrors size_t btncanddoneSucc_; // # investigated and succeeded size_t btncanddoneFail_; // # investigated and failed diff --git a/aligner_sw_driver.cpp b/aligner_sw_driver.cpp index afd25af8..efde5bd2 100644 --- a/aligner_sw_driver.cpp +++ b/aligner_sw_driver.cpp @@ -80,6 +80,7 @@ bool SwDriver::eeSaTups( gws_.clear(); rands_.clear(); satpos_.clear(); + lazy_.clear(); eehits_.clear(); // First, count up the total number of satpos_, rands_, eehits_, and gws_ // we're going to tuse @@ -290,11 +291,116 @@ bool SwDriver::eeSaTups( return true; } +/** + * Set up the two chains of a seed hit's extension as extend() would before + * its loops: ch[0] extends to the left with the forward index, ch[1] to the + * right with the backward index. A chain with nothing to do is not live. + */ +void SwDriver::extInit( + ExtChain* ch, // out: ch[0] left, ch[1] right + const Read& rd, // read + const Ebwt& ebwtFw, // Forward Bowtie index + const Ebwt* ebwtBw, // Backward Bowtie index + TIndexOffU topf, // top in fw index + TIndexOffU botf, // bot in fw index + TIndexOffU topb, // top in bw index + TIndexOffU botb, // bot in bw index + bool fw, // seed orientation + size_t off, // seed offset from 5' end + size_t len) // seed length +{ + const size_t rdlen = rd.length(); + const BTDnaString* seq = fw ? &rd.patFw : &rd.patRc; + ExtChain& l = ch[0]; + l.n = l.ops = 0; + l.left = fw ? off : rdlen - len - off; + l.live = l.left > 0; + if(l.live) { + // Starting to left of seed ( 0 && ebwtBw != NULL; + if(r.live) { + // Starting to right of seed and moving right + r.ebwt = ebwtBw; + r.seq = seq; + r.top = topb; + r.bot = botb; + r.tp0 = topf; + r.i = fw ? len + off : rdlen - off; + r.right = true; + INIT_LOCS(r.top, r.bot, r.tloc, r.bloc, *ebwtBw); + } +} + +/** + * One iteration of extend()'s loop on chain c. c.live turns false where + * extend() leaves the loop: the extension stops (mismatch, more than one + * character possible, range shrinks), reaches 255 positions, or reaches the + * end of the read. + */ +inline void SwDriver::extStep(ExtChain& c) { + assert(c.live); + // Get char from read + const int rdc = c.seq->get(c.i); + c.ops++; + if(c.bloc.valid()) { + TIndexOffU t[4] = { 0, 0, 0, 0 }, b[4] = { 0, 0, 0, 0 }; + TIndexOffU tp[4], bp[4]; + tp[0] = c.tp0; // the only element of tp/bp mapBiLFEx reads + c.ebwt->mapBiLFEx(c.tloc, c.bloc, t, b, tp, bp); + SANITY_CHECK_4TUP(t, b, tp, bp); + int nonz = -1; + bool abort = false; + size_t origSz = c.bot - c.top; + for(int j = 0; j < 4; j++) { + if(b[j] > t[j]) { + if(nonz >= 0) { + abort = true; + break; + } + nonz = j; + c.top = t[j]; c.bot = b[j]; + } + } + assert_leq(c.bot - c.top, origSz); + if(abort || (nonz != rdc && rdc <= 3) || c.bot - c.top < origSz) { + c.live = false; + return; + } + } else { + assert_eq(c.bot, c.top+1); + int ch = c.ebwt->mapLF1(c.top, c.tloc); + if(ch != rdc && rdc <= 3) { + c.live = false; + return; + } + c.bot = c.top + 1; + } + if(++c.n == 255 || --c.left == 0) { + c.live = false; + return; + } + c.i = c.right ? c.i + 1 : c.i - 1; + INIT_LOCS(c.top, c.bot, c.tloc, c.bloc, *c.ebwt); +} + /** * Extend a seed hit out on either side. Requires that we know the seed hit's * offset into the read and orientation. Also requires that we know top/bot * for the seed hit in both the forward and (if we want to extend to the right) - * reverse index. + * reverse index. The left and right extensions are independent and are + * stepped in turn. */ void SwDriver::extend( const Read& rd, // read @@ -311,176 +417,19 @@ void SwDriver::extend( size_t& nlex, // # positions we can extend to left w/o edit size_t& nrex) // # positions we can extend to right w/o edit { - TIndexOffU t[4], b[4]; - TIndexOffU tp[4], bp[4]; - SideLocus tloc, bloc; - size_t rdlen = rd.length(); - size_t lim = fw ? off : rdlen - len - off; - // We're about to add onto the beginning, so reverse it -#ifndef NDEBUG - if(false) { - // TODO: This will sometimes fail even when the extension is legitimate - // This is because contains() comes in from one extreme or the other, - // whereas we started from the inside and worked outwards. This - // affects which Ns are OK and which are not OK. - - // Have to do both because whether we can get through an N depends on - // which direction we're coming in -// bool fwContains = ebwtFw.contains(tmp_rdseq_); -// tmp_rdseq_.reverse(); -// bool bwContains = ebwtBw != NULL && ebwtBw->contains(tmp_rdseq_); -// tmp_rdseq_.reverse(); -// assert(fwContains || bwContains); - } -#endif - ASSERT_ONLY(tmp_rdseq_.reverse()); - if(lim > 0) { - const Ebwt *ebwt = &ebwtFw; - assert(ebwt != NULL); - // Extend left using forward index - const BTDnaString& seq = fw ? rd.patFw : rd.patRc; - // See what we get by extending - TIndexOffU top = topf, bot = botf; - t[0] = t[1] = t[2] = t[3] = 0; - b[0] = b[1] = b[2] = b[3] = 0; - tp[0] = tp[1] = tp[2] = tp[3] = topb; - bp[0] = bp[1] = bp[2] = bp[3] = botb; - SideLocus tloc, bloc; - INIT_LOCS(top, bot, tloc, bloc, *ebwt); - for(size_t ii = 0; ii < lim; ii++) { - // Starting to left of seed (mapBiLFEx(tloc, bloc, t, b, tp, bp); - SANITY_CHECK_4TUP(t, b, tp, bp); - int nonz = -1; - bool abort = false; - size_t origSz = bot - top; - for(int j = 0; j < 4; j++) { - if(b[j] > t[j]) { - if(nonz >= 0) { - abort = true; - break; - } - nonz = j; - top = t[j]; bot = b[j]; - } - } - assert_leq(bot - top, origSz); - if(abort || (nonz != rdc && rdc <= 3) || bot - top < origSz) { - break; - } - } else { - assert_eq(bot, top+1); - prm.nSdFmops++; - int c = ebwt->mapLF1(top, tloc); - if(c != rdc && rdc <= 3) { - break; - } - bot = top + 1; - } - ASSERT_ONLY(tmp_rdseq_.append(rdc)); - if(++nlex == 255) { - break; - } - INIT_LOCS(top, bot, tloc, bloc, *ebwt); - } + ExtChain ch[2]; + extInit(ch, rd, ebwtFw, ebwtBw, topf, botf, topb, botb, fw, off, len); + ch[0].n = nlex; + ch[1].n = nrex; + while(ch[0].live || ch[1].live) { + if(ch[0].live) extStep(ch[0]); + if(ch[1].live) extStep(ch[1]); } - // We're about to add onto the end, so re-reverse - ASSERT_ONLY(tmp_rdseq_.reverse()); - lim = fw ? rdlen - len - off : off; - if(lim > 0 && ebwtBw != NULL) { - const Ebwt *ebwt = ebwtBw; - assert(ebwt != NULL); - // Extend right using backward index - const BTDnaString& seq = fw ? rd.patFw : rd.patRc; - // See what we get by extending - TIndexOffU top = topb, bot = botb; - t[0] = t[1] = t[2] = t[3] = 0; - b[0] = b[1] = b[2] = b[3] = 0; - tp[0] = tp[1] = tp[2] = tp[3] = topf; - bp[0] = bp[1] = bp[2] = bp[3] = botf; - INIT_LOCS(top, bot, tloc, bloc, *ebwt); - for(size_t ii = 0; ii < lim; ii++) { - // Starting to right of seed (mapBiLFEx(tloc, bloc, t, b, tp, bp); - SANITY_CHECK_4TUP(t, b, tp, bp); - int nonz = -1; - bool abort = false; - size_t origSz = bot - top; - for(int j = 0; j < 4; j++) { - if(b[j] > t[j]) { - if(nonz >= 0) { - abort = true; - break; - } - nonz = j; - top = t[j]; bot = b[j]; - } - } - assert_leq(bot - top, origSz); - if(abort || (nonz != rdc && rdc <= 3) || bot - top < origSz) { - break; - } - } else { - assert_eq(bot, top+1); - prm.nSdFmops++; - int c = ebwt->mapLF1(top, tloc); - if(c != rdc && rdc <= 3) { - break; - } - bot = top + 1; - } - ASSERT_ONLY(tmp_rdseq_.append(rdc)); - if(++nrex == 255) { - break; - } - INIT_LOCS(top, bot, tloc, bloc, *ebwt); - } - } -#ifndef NDEBUG - if(false) { - // TODO: This will sometimes fail even when the extension is legitimate - // This is because contains() comes in from one extreme or the other, - // whereas we started from the inside and worked outwards. This - // affects which Ns are OK and which are not OK. - - // Have to do both because whether we can get through an N depends on - // which direction we're coming in -// bool fwContains = ebwtFw.contains(tmp_rdseq_); -// tmp_rdseq_.reverse(); -// bool bwContains = ebwtBw != NULL && ebwtBw->contains(tmp_rdseq_); -// tmp_rdseq_.reverse(); -// assert(fwContains || bwContains); - } -#endif - assert_lt(nlex, rdlen); - assert_lt(nrex, rdlen); - return; + nlex = ch[0].n; + nrex = ch[1].n; + prm.nSdFmops += ch[0].ops + ch[1].ops; + assert_lt(nlex, rd.length()); + assert_lt(nrex, rd.length()); } /** @@ -517,6 +466,7 @@ void SwDriver::prioritizeSATupsRands( rands2_.clear(); satpos_.clear(); satpos2_.clear(); + lazy_.clear(); size_t nrange = 0, nelt = 0, nsmall = 0, nsmall_elts = 0; bool keepWhole = false; EList& satpos = keepWhole ? satpos_ : satpos2_; @@ -690,7 +640,22 @@ void SwDriver::prioritizeSATupsRands( for(size_t j = 0; j < satpos2_.size(); j++) { rands2_[j].reset(); } - while(nelt_added < maxelt && nelt_added < nelt) { + // The loop adds one element per iteration until there are + // min(maxelt, nelt): make room for all of them at once (the lists + // already have the capacity, so no list moves) + const size_t neltEnd = min(maxelt, nelt); + const size_t nadd = nelt_added < neltEnd ? neltEnd - nelt_added : 0; + const size_t base = satpos_.size(); + lazyBase_ = base; + lazyAll_ = all; + satpos_.resize(base + nadd); + gws_.resize(base + nadd); + rands_.resize(base + nadd); + lazy_.resize(nadd); + LazyElt* const lz = lazy_.ptr(); + const TIndexOffU zOff = ebwtFw._zOff; + const TIndexOffU offMask = ebwtFw.eh()._offMask; + for(size_t k = 0; k < nadd; k++) { // Pick a non-small range to sample from size_t ri = rowsamp_.next(rnd) + nsmall; assert_geq(ri, nsmall); @@ -707,36 +672,70 @@ void SwDriver::prioritizeSATupsRands( // Tell the row sampler this range is done rowsamp_.finishedRange(ri - nsmall); } - // Add the element to the satpos_ list - SATuple sat; - TSlice o; - o.init(satpos2_[ri].sat.offs, r, r+1); - sat.init(satpos2_[ri].sat.key, (TIndexOffU)(satpos2_[ri].sat.topf + r), OFF_MASK, o); - satpos_.expand(); - satpos_.back().sat = sat; - satpos_.back().origSz = satpos2_[ri].origSz; - satpos_.back().pos = satpos2_[ri].pos; - // Initialize GroupWalk object - gws_.expand(); - SARangeWithOffs sa; - sa.topf = sat.topf; - sa.len = sat.key.len; - sa.offs = sat.offs; - gws_.back().init( - ebwtFw, // forward Bowtie index - ref, // reference sequences - sa, // SA tuples: ref hit, salist range - wlm); // metrics - assert(gws_.back().initialized()); - // Initialize random selector - rands_.expand(); - rands_.back().init(1, all); - nelt_added++; + // Add the element to the satpos_ list. Its entries are built here + // only when building them resolves its offset (an SA-sampled or '$' + // row) or finds it resolved (its slot is shared with an element + // resolved before); otherwise they are built on first use + // (materialize()), as they would have been built here. + LazyElt& le = lz[k]; + le.r = (TIndexOffU)r; + le.ri = (uint32_t)ri; + const TIndexOffU row = (TIndexOffU)(satpos2_[ri].sat.topf + r); + le.pending = !(row == zOff || (row & offMask) == row || + satpos2_[ri].sat.offs.get(r) != OFF_MASK); + if(!le.pending) { + SATuple sat; + TSlice o; + o.init(satpos2_[ri].sat.offs, r, r+1); + sat.init(satpos2_[ri].sat.key, row, OFF_MASK, o); + SATupleAndPos& sp = satpos_[base + k]; + sp.sat = sat; + sp.origSz = satpos2_[ri].origSz; + sp.pos = satpos2_[ri].pos; + // Initialize GroupWalk object + SARangeWithOffs sa; + sa.topf = sat.topf; + sa.len = sat.key.len; + sa.offs = sat.offs; + gws_[base + k].init( + ebwtFw, // forward Bowtie index + ref, // reference sequences + sa, // SA tuples: ref hit, salist range + wlm); // metrics + assert(gws_[base + k].initialized()); + // Initialize random selector + rands_[base + k].init(1, all); + } } + nelt_added += nadd; nelt_out = nelt_added; return; } +void SwDriver::materializeSampled(size_t i, const Ebwt& ebwtFw) { + LazyElt& le = lazy_[i - lazyBase_]; + assert(le.pending); + const SATupleAndPos& src = satpos2_[le.ri]; + SATupleAndPos& dst = satpos_[i]; + // What the sampling loop builds for the element + SATuple sat; + TSlice o; + o.init(src.sat.offs, le.r, le.r+1); + sat.init(src.sat.key, (TIndexOffU)(src.sat.topf + le.r), OFF_MASK, o); + dst.sat = sat; + dst.origSz = src.origSz; + dst.pos = src.pos; + SARangeWithOffs sa; + sa.topf = sat.topf; + sa.len = sat.key.len; + sa.offs = sat.offs; + // The walk as init() set it up at the draw, when the slot was unresolved + gws_[i].initFreshSingle(ebwtFw, sa); + assert(gws_[i].initialized()); + rands_[i].init(1, lazyAll_); + le.pending = false; +} + // deterministic, use only a single gws element, will use just-in-time init later void SwDriver::prioritizeSATupsIdxs( const Read& read, // read @@ -758,6 +757,7 @@ void SwDriver::prioritizeSATupsIdxs( gws_.resize(1); // will use a single element in deterministic mode useCurrIdx = true; satpos_.clear(); + lazy_.clear(); size_t nrange = 0, nelt = 0; for(size_t i = 0; i < nonz; i++) { @@ -902,6 +902,133 @@ void SwDriver::prioritizeSATups( return; } +// How many rows of the ranges after the current one resolveElement() starts +// walks for, along with the current range's +static const size_t SPEC_AHEAD = 8; +// Ranges of up to this many rows are walked row by row by spec_ before +// GroupWalk2S walks them, so that its range walk finds the index's sides in +// the cache (it is not replaced: its counts depend on how the range splits) +static const size_t SPEC_WARM_MAX = 32; +// How many ranges after the current one resolveElement() looks at +static const size_t SPEC_SCAN = 32; + +/** + * Start spec_'s walks for the rows of range j that a GroupWalk2S will walk + * (every row of a single-element range, or of a range of up to SPEC_WARM_MAX + * rows not yet walked, whose offset is not yet known); return their number. + */ +size_t SwDriver::specRange(size_t j, const Ebwt& ebwtFw) { + const size_t k = j - lazyBase_; + if(k < lazy_.size() && lazy_[k].pending) { + // A pending sampled element: one row, nothing reported, not + // SA-sampled nor '$'; walked unless its slot was resolved since + const LazyElt& le = lazy_[k]; + const SATuple& src = satpos2_[le.ri].sat; + if(src.offs.get(le.r) != OFF_MASK) { + return 0; + } + const TIndexOffU row = (TIndexOffU)(src.topf + le.r); + if(!spec_.have(ebwtFw, row)) { + spec_.start(ebwtFw, row); + } + return 1; + } + const SATuple& sat = satpos_[j].sat; + const size_t sz = sat.size(); + if(sz == 0 || sz > SPEC_WARM_MAX || gws_[j].numReported() > 0) { + return 0; + } + const EbwtParams& eh = ebwtFw.eh(); + size_t n = 0; + for(size_t e = 0; e < sz; e++) { + const TIndexOffU row = sat.topf + (TIndexOffU)e; + if(sat.offs.get(e) != OFF_MASK || row == ebwtFw._zOff || (row & eh._offMask) == row) { + continue; // no walk + } + if(!spec_.have(ebwtFw, row)) { + spec_.start(ebwtFw, row); + } + n++; + } + return n; +} + +void SwDriver::resolveElement( + size_t i, // range in satpos_ / gws_ + size_t elt, // element of the range + GroupWalk2S& gws, // its GroupWalk + SARangeWithOffs& sa, // SA range with offsets + const Ebwt& ebwtFw, // forward index + const BitPairReference& ref, // reference + WalkResult& wr, // out: result + WalkMetrics& wlm, // group walk left metrics + PerReadMetrics& prm) // per-read metrics +{ + if(!useCurrIdx && gws.numReported() == 0 && sa.size() <= SPEC_WARM_MAX) { + const bool single = gws.freshSingle(sa); + TIndexOffU k = 0, rowk = 0, rowkm1 = 0; + bool have = single && spec_.lookup(ebwtFw, sa.topf, k, rowk, rowkm1); + if(!have) { + // This range's walks, and those of the next ranges, which the + // loop reaches next unless it stops first + specRange(i, ebwtFw); + size_t ahead = 0; + const size_t jend = min(satpos_.size(), i + 1 + SPEC_SCAN); + for(size_t j = i + 1; j < jend && ahead < SPEC_AHEAD; j++) { + ahead += specRange(j, ebwtFw); + } + if(single) { + spec_.finish(ebwtFw, sa.topf); + have = spec_.lookup(ebwtFw, sa.topf, k, rowk, rowkm1); + } else { + for(size_t e = 0; e < sa.size(); e++) { + spec_.finish(ebwtFw, sa.topf + (TIndexOffU)e); + } + } + } + if(have) { + assert_eq(0, elt); + gws.commitSingle(ebwtFw, sa, k, rowk, rowkm1, wr, wlm, prm); + return; + } + // A range of several rows, or a single row whose finished walk was + // displaced by another's: GroupWalk2S walks it + } + if(!useCurrIdx) { + // The steps of the one-row states inside the walk are spec_'s + gws.advanceElementSpec((TIndexOffU)elt, ebwtFw, ref, sa, gwstate_, wr, wlm, prm, spec_); + return; + } + gws.advanceElement((TIndexOffU)elt, ebwtFw, ref, sa, gwstate_, wr, wlm, prm); +} + +void SwDriver::specBatch( + const Ebwt& ebwtFw, // forward index + ExactSweepJob* const* jobs, // the batch's sweeps + size_t njobs) // # sweeps +{ + spec_.dropLive(); + const EbwtParams& eh = ebwtFw.eh(); + for(size_t j = 0; j < njobs; j++) { + const ExactSweepJob& job = *jobs[j]; + for(int s = 0; s < 2; s++) { + // eeSaTups makes a range of the hit's rows + if(!job.hit[s] || job.bot[s] - job.top[s] > SPEC_WARM_MAX) { + continue; + } + for(TIndexOffU row = job.top[s]; row < job.bot[s]; row++) { + if(row == ebwtFw._zOff || (row & eh._offMask) == row) { + continue; // offset known without a walk + } + if(!spec_.have(ebwtFw, row)) { + spec_.start(ebwtFw, row); + } + } + } + } + spec_.finishAll(ebwtFw); +} + enum { FOUND_NONE = 0, FOUND_EE, @@ -1005,6 +1132,7 @@ int SwDriver::extendSeeds( maxIters, // max # to report all); // report all hits? assert_eq(satpos_.size(), rands_.size()); + spec_.dropLive(); // walks of the previous ranges are not needed assert_eq(gws_.size(), satpos_.size()); } else { eeMode = false; @@ -1044,6 +1172,7 @@ int SwDriver::extendSeeds( } neltLeft = nelt; firstExtend = false; + spec_.dropLive(); // walks of the previous ranges are not needed } if(neltLeft == 0) { // Finished examining gapped candidates @@ -1055,6 +1184,8 @@ int SwDriver::extendSeeds( if(eeMode && eehits_[i].score < minsc) { return EXTEND_PERFECT_SCORE; } + // Entries the sampling loop left pending are built now + materialize(i, ebwtFw); // In deterministic mode, we can reuse the same gws_ element GroupWalk2S &gws = gws_[useCurrIdx? 0 : i]; @@ -1123,7 +1254,7 @@ int SwDriver::extendSeeds( sa.topf = satpos.sat.topf; sa.len = satpos.sat.key.len; sa.offs = satpos.sat.offs; - gws.advanceElement((TIndexOffU)elt, ebwtFw, ref, sa, gwstate_, wr, wlm, prm); + resolveElement(i, elt, gws, sa, ebwtFw, ref, wr, wlm, prm); eltsDone++; if(!eeMode) { assert_gt(neltLeft, 0); @@ -1717,6 +1848,7 @@ int SwDriver::extendSeedsPaired( maxIters, // max elts to report all); // report all hits assert_eq(satpos_.size(), rands_.size()); + spec_.dropLive(); // walks of the previous ranges are not needed assert_eq(gws_.size(), satpos_.size()); neltLeft = nelt; // Initialize list that contains the mate-finding failure @@ -1763,6 +1895,7 @@ int SwDriver::extendSeedsPaired( } neltLeft = nelt; firstExtend = false; + spec_.dropLive(); // walks of the previous ranges are not needed mateStreaks_.resize(satpos_.size()); mateStreaks_.fill(0); } @@ -1776,6 +1909,8 @@ int SwDriver::extendSeedsPaired( if(eeMode && eehits_[i].score < minsc) { return EXTEND_PERFECT_SCORE; } + // Entries the sampling loop left pending are built now + materialize(i, ebwtFw); // In deterministic mode, we can reuse the same gws_ element GroupWalk2S &gws = gws_[useCurrIdx? 0 : i]; @@ -1861,7 +1996,7 @@ int SwDriver::extendSeedsPaired( sa.topf = satpos.sat.topf; sa.len = satpos.sat.key.len; sa.offs = satpos.sat.offs; - gws.advanceElement((TIndexOffU)elt, ebwtFw, ref, sa, gwstate_, wr, wlm, prm); + resolveElement(i, elt, gws, sa, ebwtFw, ref, wr, wlm, prm); eltsDone++; assert_gt(neltLeft, 0); neltLeft--; diff --git a/aligner_sw_driver.h b/aligner_sw_driver.h index 57034293..7f75d5ca 100644 --- a/aligner_sw_driver.h +++ b/aligner_sw_driver.h @@ -180,8 +180,10 @@ class RowSampler { public: - RowSampler(int cat = 0) : elim_(cat), masses_(cat) { + RowSampler(int cat = 0) : elim_(cat), masses_(cat), prefix_(cat) { mass_ = 0.0f; + dirty_ = 0; + lastUnelim_ = std::numeric_limits::max(); } /** @@ -214,6 +216,9 @@ class RowSampler { masses_[i - sai] = num / denom; mass_ += masses_[i - sai]; } + prefix_.resize(saf - sai); + dirty_ = 0; + lastUnelim_ = saf - sai - 1; } /** @@ -224,6 +229,21 @@ class RowSampler { assert_lt(i, masses_.size()); elim_[i] = true; mass_ -= masses_[i]; + if(i < dirty_) { + dirty_ = i; + } + if(i == lastUnelim_) { + // The fallback of next() moves to the previous range left + size_t j = i; + lastUnelim_ = std::numeric_limits::max(); + while(j > 0) { + j--; + if(!elim_[j]) { + lastUnelim_ = j; + break; + } + } + } } /** @@ -232,27 +252,77 @@ class RowSampler { size_t next(RandomSource& rnd) { // Throw the dart double rd = rnd.nextFloat() * mass_; - double mass_sofar = 0.0f; size_t sz = masses_.size(); - size_t last_unelim = std::numeric_limits::max(); - for(size_t i = 0; i < sz; i++) { - if(!elim_[i]) { - last_unelim = i; - mass_sofar += masses_[i]; - if(rd < mass_sofar) { - // This is the one we hit - return i; + if(dirty_ < sz) { + updatePrefix(); + } + // The first range whose running mass exceeds the dart. prefix_[i] + // is the running sum a scan over the ranges left would hold at range + // i (eliminated ranges hold the previous range's, or -inf before the + // first range left), so prefix_ never decreases and the ranges + // before the first i with rd < prefix_[i] are the ones the scan + // passes. The first such i is never an eliminated range. + const double* p = prefix_.ptr(); + size_t k = 0; + if(sz <= 32) { + for(size_t i = 0; i < sz; i++) { + k += !(rd < p[i]); + } + } else { + size_t hi = sz; + while(k < hi) { + size_t mid = k + (hi - k) / 2; + if(rd < p[mid]) { + hi = mid; + } else { + k = mid + 1; } } } - assert_neq(std::numeric_limits::max(), last_unelim); - return last_unelim; + if(k < sz) { + assert(!elim_[k]); + return k; + } + // No running mass exceeded the dart (mass_ and the running sum round + // differently): the last range left, as the scan returned + assert_neq(std::numeric_limits::max(), lastUnelim_); + return lastUnelim_; } protected: + + /** + * Recompute prefix_ from dirty_ on: the running mass of the ranges left, + * added one range at a time in index order from 0.0 exactly as the + * scan in next() added them, so every value is the same double. + */ + void updatePrefix() { + const double ninf = -std::numeric_limits::infinity(); + const size_t sz = masses_.size(); + double acc = 0.0f; + bool any = false; + if(dirty_ > 0 && prefix_[dirty_ - 1] != ninf) { + acc = prefix_[dirty_ - 1]; + any = true; + } + for(size_t i = dirty_; i < sz; i++) { + if(!elim_[i]) { + acc += masses_[i]; + any = true; + prefix_[i] = acc; + } else { + prefix_[i] = any ? acc : ninf; + } + } + dirty_ = sz; + } + double mass_; // total probability mass to throw darts at EList elim_; // whether the range is eliminated EList masses_; // mass of each range + EList prefix_; // running mass of the ranges left, by index + size_t dirty_; // prefix_ is stale from this index on + size_t lastUnelim_; // last range not eliminated }; /** @@ -291,6 +361,29 @@ struct ExtendRange { size_t sz; // # of elements in SA range }; +/** + * One direction of a seed-hit extension (SwDriver::extend()): the left + * chain walks the forward index, the right chain the mirror index. The two + * are independent, so extend() steps them in turn: one LF step per chain per + * turn, with each chain's next side prefetched (INIT_LOCS) before the other + * chain's step. + */ +struct ExtChain { + const Ebwt* ebwt; // index the chain walks + const BTDnaString* seq; // read characters (patFw or patRc) + SideLocus tloc; + SideLocus bloc; + TIndexOffU top; + TIndexOffU bot; + TIndexOffU tp0; // topsP[0] handed to mapBiLFEx + size_t i; // read position of the next step + size_t left; // steps left before the end of the read + size_t n; // positions extended (nlex or nrex) + size_t ops; // LF ops (added to prm.nSdFmops) + bool right; // walks to the right (i increases) + bool live; // has another step to take +}; + class SwDriver { typedef PList TSAList; @@ -300,6 +393,9 @@ class SwDriver { SwDriver(size_t bytes) : satups_(DP_CAT), gws_(DP_CAT), + lazy_(DP_CAT), + lazyBase_(0), + lazyAll_(false), seenDiags1_(DP_CAT), seenDiags2_(DP_CAT), redAnchor_(DP_CAT), @@ -442,8 +538,64 @@ class SwDriver { redAnchor_.init(maxlen); } + /** + * Walk, several in turn, the offset resolutions of the rows of the + * exact end-to-end hits (of up to SPEC_WARM_MAX rows) the batch's sweeps + * found. resolveElement() commits the walk of a single-row hit when, + * and if, the read's extension resolves it; a larger hit's range walk + * then finds the index's sides in the cache. Nothing is counted or + * resolved here. + */ + void specBatch( + const Ebwt& ebwtFw, // forward index + ExactSweepJob* const* jobs, // the batch's sweeps + size_t njobs); // # sweeps + protected: + /** + * Resolve element 'elt' of the GroupWalk of range 'i' (of satpos_) as + * GroupWalk2S::advanceElement() does. A single-element walk is walked by + * spec_ instead, together with the walks of the rows of the next ranges + * the loop will reach (SPEC_AHEAD), and committed here; a small range + * is walked by advanceElement() once spec_ has walked its rows. + */ + void resolveElement( + size_t i, // range in satpos_ / gws_ + size_t elt, // element of the range + GroupWalk2S& gws, // its GroupWalk + SARangeWithOffs& sa, // SA range with offsets + const Ebwt& ebwtFw, // forward index + const BitPairReference& ref, // reference + WalkResult& wr, // out: result + WalkMetrics& wlm, // group walk left metrics + PerReadMetrics& prm); // per-read metrics + + /** + * Start spec_'s walks for the rows of range j (of satpos_) that its + * GroupWalk2S will walk; return their number. + */ + size_t specRange(size_t j, const Ebwt& ebwtFw); + + /** + * Build the satpos_, gws_ and rands_ entries of range i (of satpos_) if + * prioritizeSATupsRands() left them pending. Call before any of the + * three is read. + */ + inline void materialize(size_t i, const Ebwt& ebwtFw) { + const size_t k = i - lazyBase_; + if(k < lazy_.size() && lazy_[k].pending) { + materializeSampled(i, ebwtFw); + } + } + + /** + * Build the entries of the pending sampled element i as the sampling + * loop of prioritizeSATupsRands() would have built them when it drew + * the element. + */ + void materializeSampled(size_t i, const Ebwt& ebwtFw); + bool eeSaTups( const Read& rd, // read SeedResults& sh, // seed hits to extend into full alignments @@ -538,11 +690,41 @@ class SwDriver { EList > gws_; // list of GroupWalks; no particular order EList mateStreaks_; // mate-find fail streaks RowSampler rowsamp_; // row sampler + + // An element drawn by the sampling loop of prioritizeSATupsRands(): row + // r of range ri of satpos2_. Pending: its satpos_, gws_ and rands_ + // entries are not built yet (materialize() builds them); only elements + // whose row is neither SA-sampled nor the '$' row, and whose offset slot + // was unresolved when drawn, are left pending (building them then wrote + // nothing outside their own entries). + struct LazyElt { + TIndexOffU r; // element of the range + uint32_t ri; // range in satpos2_ + bool pending; + }; + EList lazy_; // the sampled elements, satpos_ index lazyBase_ + k + size_t lazyBase_; // satpos_ index of lazy_[0] + bool lazyAll_; // 'all' of the prioritizeSATupsRands() call // Ranges that we've extended through when extending seed hits EList seedExRangeFw_[2]; EList seedExRangeRc_[2]; + // Chains of a seed-hit extension (extend()) + static void extInit( + ExtChain* ch, // out: ch[0] left, ch[1] right + const Read& rd, + const Ebwt& ebwtFw, + const Ebwt* ebwtBw, + TIndexOffU topf, + TIndexOffU botf, + TIndexOffU topb, + TIndexOffU botb, + bool fw, + size_t off, + size_t len); + static void extStep(ExtChain& c); + // Data structures encapsulating the diagonals that have already been used // to seed alignment for mate 1 and mate 2. EIvalMergeListBinned seenDiags1_; @@ -564,6 +746,7 @@ class SwDriver { Pool pool_; // memory pages for salistExact_ TSAList salistEe_; // PList for offsets for end-to-end hits GroupWalkState gwstate_; // some per-thread state shared by all GroupWalks + SpecWalk spec_; // single-row offset walks, stepped in turn // For AlnRes::matchesRef: ASSERT_ONLY(SStringExpandable raw_refbuf_); diff --git a/aligner_swsse.cpp b/aligner_swsse.cpp index 5d9c727c..3c54a05e 100644 --- a/aligner_swsse.cpp +++ b/aligner_swsse.cpp @@ -21,6 +21,51 @@ #include "aligner_sw_common.h" #include "aligner_swsse.h" +/** + * For each nonzero 5-bit H-cell backtrace mask, the move the u8 backtraces + * take from the cell (an SW_BT_* value, bits 0-2) and the mask they leave + * in the cell (bits 3-7). Mask bits: 0 H above (ref gap open), 1 H left + * (read gap open), 2 F above (ref gap extend), 3 E left (read gap extend), 4 + * H diagonal. A mask with one bit set takes that move and leaves 0 (alts5, + * firsts5); one with several takes the first of diagonal, H above, F above, + * H left and E left, and leaves the others. btHSelectBit is the same choice + * made the parent's way, for the asserts. + */ +const uint8_t btHSelect[32] = { + 0, // 0: no move (not used) + SW_BT_OALL_REF_OPEN | ( 0 << 3), // 1 + SW_BT_OALL_READ_OPEN | ( 0 << 3), // 2 + SW_BT_OALL_REF_OPEN | ( 2 << 3), // 3 + SW_BT_RFGAP_EXTEND | ( 0 << 3), // 4 + SW_BT_OALL_REF_OPEN | ( 4 << 3), // 5 + SW_BT_RFGAP_EXTEND | ( 2 << 3), // 6 + SW_BT_OALL_REF_OPEN | ( 6 << 3), // 7 + SW_BT_RDGAP_EXTEND | ( 0 << 3), // 8 + SW_BT_OALL_REF_OPEN | ( 8 << 3), // 9 + SW_BT_OALL_READ_OPEN | ( 8 << 3), // 10 + SW_BT_OALL_REF_OPEN | (10 << 3), // 11 + SW_BT_RFGAP_EXTEND | ( 8 << 3), // 12 + SW_BT_OALL_REF_OPEN | (12 << 3), // 13 + SW_BT_RFGAP_EXTEND | (10 << 3), // 14 + SW_BT_OALL_REF_OPEN | (14 << 3), // 15 + SW_BT_OALL_DIAG | ( 0 << 3), // 16 + SW_BT_OALL_DIAG | ( 1 << 3), // 17 + SW_BT_OALL_DIAG | ( 2 << 3), // 18 + SW_BT_OALL_DIAG | ( 3 << 3), // 19 + SW_BT_OALL_DIAG | ( 4 << 3), // 20 + SW_BT_OALL_DIAG | ( 5 << 3), // 21 + SW_BT_OALL_DIAG | ( 6 << 3), // 22 + SW_BT_OALL_DIAG | ( 7 << 3), // 23 + SW_BT_OALL_DIAG | ( 8 << 3), // 24 + SW_BT_OALL_DIAG | ( 9 << 3), // 25 + SW_BT_OALL_DIAG | (10 << 3), // 26 + SW_BT_OALL_DIAG | (11 << 3), // 27 + SW_BT_OALL_DIAG | (12 << 3), // 28 + SW_BT_OALL_DIAG | (13 << 3), // 29 + SW_BT_OALL_DIAG | (14 << 3), // 30 + SW_BT_OALL_DIAG | (15 << 3) // 31 +}; + /** * Given a number of rows (nrow), a number of columns (ncol), and the * number of words to fit inside a single SSERegI vector, initialize the @@ -68,6 +113,18 @@ void SSEMatrix::initMasks() { assert_gt(nrow_, 0); assert_gt(ncol_, 0); masks_.resize(nrow_); + // The dirty intervals only grow, so every row masks_ has ever exposed, + // and so every row that may hold a buffer, keeps its record. A row + // without one yet gets [0, SIZE_MAX): resetRow then clears all of it. + size_t ndirty = dirtylo_.size(); + if(ndirty < nrow_) { + dirtylo_.resize(nrow_); + dirtyhi_.resize(nrow_); + for(size_t i = ndirty; i < nrow_; i++) { + dirtylo_[i] = 0; + dirtyhi_[i] = std::numeric_limits::max(); + } + } reset_.resizeNoCopy(nrow_); reset_.fill(false); } diff --git a/aligner_swsse.h b/aligner_swsse.h index aa8d1e3b..7fe1cde5 100644 --- a/aligner_swsse.h +++ b/aligner_swsse.h @@ -268,13 +268,30 @@ struct SSEMatrix { } /** - * Set the given cell's reportedThru bit. + * Set the given cell's reportedThru bit. Every backtrace calls this (or + * markDirty) on each cell whose mask word it writes, so it also records + * the cell in the row's dirty interval; see resetRow. */ inline void setReportedThrough( size_t row, // current row size_t col) // current column { masks_[row][col] |= (1 << 0); + markDirty(row, col); + } + + /** + * Record that the mask word of the given cell may be nonzero, by widening + * the row's dirty interval [dirtylo_[row], dirtyhi_[row]) to include col. + */ + inline void markDirty( + size_t row, // current row + size_t col) // current column + { + size_t& lo = dirtylo_[row]; + size_t& hi = dirtyhi_[row]; + lo = (col < lo) ? col : lo; + hi = (col >= hi) ? col + 1 : hi; } /** @@ -375,12 +392,36 @@ struct SSEMatrix { } /** - * Prepare a row so we can use it to store masks. + * Prepare a row so we can use it to store masks: afterwards the row has + * ncol_ elements, all zero. + * + * A row's buffer is zero everywhere outside its dirty interval, the + * columns written since the row was last reset (markDirty), so only that + * interval is cleared. A buffer that is about to be allocated is + * cleared in full. A row with no dirty record yet has the interval + * [0, SIZE_MAX), so it is cleared in full too (initMasks). */ void resetRow(size_t i) { assert(!reset_[i]); - masks_[i].resizeNoCopy(ncol_); - masks_[i].fillZero(); + EList& m = masks_[i]; + if(m.null() || m.capacity() < ncol_) { + m.resizeNoCopy(ncol_); + memset(m.ptr(), 0, sizeof(uint16_t) * m.capacity()); + } else { + m.resizeNoCopy(ncol_); + size_t lo = dirtylo_[i]; + size_t hi = dirtyhi_[i] < m.capacity() ? dirtyhi_[i] : m.capacity(); + if(lo < hi) { + memset(m.ptr() + lo, 0, sizeof(uint16_t) * (hi - lo)); + } + } + dirtylo_[i] = std::numeric_limits::max(); + dirtyhi_[i] = 0; +#ifndef NDEBUG + for(size_t j = 0; j < m.capacity(); j++) { + assert_eq(0, m.ptr()[j]); + } +#endif reset_[i] = true; } @@ -398,6 +439,8 @@ struct SSEMatrix { EList_sse matbuf_; // buffer for holding vectors ELList masks_; // buffer for masks/backtracking flags EList reset_; // true iff row in masks_ has been reset + EList dirtylo_; // per row of masks_: first column that may be nonzero + EList dirtyhi_; // per row of masks_: one past the last such column }; /** @@ -493,6 +536,28 @@ inline void SSEMatrix::fMaskSet( masks_[row][col] |= (1 << 10 | mask << 11); } +/** + * The move and remaining mask for each H-cell backtrace mask; see + * aligner_swsse.cpp. + */ +extern const uint8_t btHSelect[32]; + +/** + * The bit of a nonzero H-cell backtrace mask that a backtrace takes, chosen + * as the parent code did: the only bit (firsts5) when one is set, otherwise + * the first of diagonal (16), H above (1), F above (4), H left (2) and E + * left (8). For asserts. + */ +static inline int btHSelectBit(int mask) { + assert_gt(mask, 0); + if(alts5[mask] == 1) return 1 << firsts5[mask]; + if((mask & 16) != 0) return 16; + if((mask & 1) != 0) return 1; + if((mask & 4) != 0) return 4; + if((mask & 2) != 0) return 2; + return 8; +} + #define ROWSTRIDE_2COL 4 #define ROWSTRIDE 4 #define ROWSTRIDE_LOG2 2 diff --git a/aligner_swsse_ee_u8.cpp b/aligner_swsse_ee_u8.cpp index 86c0c38e..88f636ad 100644 --- a/aligner_swsse_ee_u8.cpp +++ b/aligner_swsse_ee_u8.cpp @@ -95,50 +95,87 @@ void SwAligner::buildQueryProfileEnd2EndSseU8(bool fw) { d.lastIter_ = d.lastWord_ = 0; d.qprofStride_ = d.gbarStride_ = 2; d.bias_ = 0; // no bias needed for end-to-end alignment; just use subtraction - // For each reference character A, C, G, T, N ... - for(size_t refc = 0; refc < ALPHA_SIZE; refc++) { - // For each segment ... - for(size_t i = 0; i < seglen; i++) { - size_t j = i; - uint8_t *qprofWords = - reinterpret_cast(d.profbuf_.ptr() + (refc * seglen * 2) + (i * 2)); - uint8_t *gbarWords = - reinterpret_cast(d.profbuf_.ptr() + (refc * seglen * 2) + (i * 2) + 1); - // For each sub-word (byte) ... - for(size_t k = 0; k < NWORDS_PER_REG; k++) { - int sc = 0; - *gbarWords = 0; - if(j < len) { - int readc = (*rd)[j]; - int readq = (*qu)[j]; - sc = sc_->score(readc, (int)(1 << refc), readq - 33); - // Make score positive, to fit in an unsigned - sc = -sc; - assert_range(0, 255, sc); - size_t j_from_end = len - j - 1; - if(j < (size_t)sc_->gapbar || - j_from_end < (size_t)sc_->gapbar) - { - // Inside the gap barrier - *gbarWords = 0xff; - } - } - if(refc == 0 && j == len-1) { - // Remember which 128-bit word and which smaller word has - // the final row - d.lastIter_ = i; - d.lastWord_ = k; + if(len > 0) { + // Remember which 128-bit word and which smaller word has the final + // row: read position j is in segment j % seglen, word j / seglen + d.lastIter_ = (len - 1) % seglen; + d.lastWord_ = (len - 1) / seglen; + } + // For each reference character A, C, G, T, N there are seglen segments, + // each a query-profile register followed by a gap-barrier register. + // Word k of segment i holds read position j = k * seglen + i. + SSERegI *pvec = d.profbuf_.ptr(); + uint8_t *prof = reinterpret_cast(pvec); + const size_t segBytes = 2 * NBYTES_PER_REG; // one segment's two registers + const size_t refBytes = seglen * segBytes; // one reference character's segments + // The gap barrier depends on the read position alone: 0xff where j < len + // and j or len - j - 1 is below gapbar, 0 elsewhere. Zero it, then mark + // the barrier positions at both ends. + SSERegI vzero = sse_setzero_siall(); + for(size_t s = 0; s < seglen * ALPHA_SIZE; s++) { + sse_store_siall(pvec + 2 * s + 1, vzero); + } + { + const size_t gapbar = (size_t)sc_->gapbar; + const size_t nbar = std::min(len, gapbar); + // j < gapbar for j in [0, nbar); len - j - 1 < gapbar for j in + // [len - nbar, len) + const size_t ranges[2][2] = { + { 0, nbar }, + { std::max(nbar, len - nbar), len } }; + for(size_t r = 0; r < 2; r++) { + if(ranges[r][0] >= ranges[r][1]) { + continue; + } + size_t i = ranges[r][0] % seglen, k = ranges[r][0] / seglen; + for(size_t j = ranges[r][0]; j < ranges[r][1]; j++) { + uint8_t *g = prof + i * segBytes + NBYTES_PER_REG + k; + for(size_t refc = 0; refc < ALPHA_SIZE; refc++) { + g[refc * refBytes] = 0xff; } - if((size_t)sc > d.maxPen_) { - d.maxPen_ = (size_t)sc; + if(++i == seglen) { + i = 0; + k++; } - *qprofWords = (uint8_t)sc; - gbarWords++; - qprofWords++; - j += seglen; // update offset into query } } } + // Query profile: sc_->score(readc, 1 << refc, readq - 33), negated to a + // positive penalty that fits in an unsigned byte; 0 past the read's end. + // The scorer's qualProf table holds the five bytes for every reference + // character at once, and the largest penalty among them. + const char *rdbuf = rd->buf(); + const char *qubuf = qu->buf(); + size_t maxPen = 0; + for(size_t i = 0; i < seglen; i++) { + uint8_t *p = prof + i * segBytes; + size_t j = i, k = 0; + for(; j < len; j += seglen, k++) { + const int readc = rdbuf[j]; + const int readq = qubuf[j]; + assert_range(0, 4, readc); + assert_range(0, 255, readq - 33); + const Scoring::QualProf& e = sc_->qualProf[readc][readq - 33]; + const uint64_t w = e.ee; + assert_eq((uint8_t)w, (uint8_t)-sc_->score(readc, 1, readq - 33)); + assert_eq((uint8_t)(w >> 8), (uint8_t)-sc_->score(readc, 2, readq - 33)); + assert_eq((uint8_t)(w >> 16), (uint8_t)-sc_->score(readc, 4, readq - 33)); + assert_eq((uint8_t)(w >> 24), (uint8_t)-sc_->score(readc, 8, readq - 33)); + assert_eq((uint8_t)(w >> 32), (uint8_t)-sc_->score(readc, 16, readq - 33)); + maxPen = std::max(maxPen, (size_t)e.eeMax); + p[k] = (uint8_t)w; + p[k + refBytes] = (uint8_t)(w >> 8); + p[k + 2 * refBytes] = (uint8_t)(w >> 16); + p[k + 3 * refBytes] = (uint8_t)(w >> 24); + p[k + 4 * refBytes] = (uint8_t)(w >> 32); + } + for(; k < NWORDS_PER_REG; k++) { + for(size_t refc = 0; refc < ALPHA_SIZE; refc++) { + p[k + refc * refBytes] = 0; + } + } + } + d.maxPen_ = maxPen; } #ifndef NDEBUG @@ -768,6 +805,873 @@ TAlScore SwAligner::alignGatherEE8(int& flag, bool debug) { return score; } +#if NBYTES_PER_REG == 32 +/** + * The fill of alignNucleotidesEnd2EndSseU8 for a column of ITER = 2 to 4 + * 256-bit vectors (33 to 128 rows). It computes the general loop's saturating + * u8 recurrence (see the comments there) and stores the same E, F and H + * vectors in the same places. The general loop's speed is set by the chain + * of dependent instructions from one column to the next: the first pass's F + * through every row, a scan with a shift across the 128-bit halves in every + * step, and the fix-up pass. Here that chain is the scan, with in-half + * shifts, and a few instructions, and every value passed from one column to + * the next stays in a register. + * + * Row j of lane L is read row L * ITER + j. For each column: + * - H0[j] = max(diag - profile, E[j]), H before the reference gaps (F); + * - within a lane, F flows from row j to row j + 1 at the cost of rfgapo + * from H0 (opening from H = max(H0, F) costs no less than extending F, as + * rfgapo >= rfgape) or of the barrier and rfgape from F; HL and FL are H + * and F with these carries only; + * - the carry c into each lane from the lanes above is the general loop's + * prefix scan of the lanes' carry-outs, done within each 128-bit half (two + * steps of four lanes, with shifts that do not cross the halves) and then + * across the halves once; + * - each row takes c less the cost from the top of its lane (vpen): F = + * max(FL, that), H = max(HL, that), and the next column's E = max(E - + * rdgape, H - rdgapo - barrier). + * The next column's carry-out, the scan's input, is max_j(H0[j] - cpen[j]). + * Subtraction distributes over max, so it is also the max of terms in HL + * and E, which are ready while the scan runs, and of terms in c and in c + * shifted down a lane, each with its smallest cost (zc, wc, kb). The chain + * from one column to the next is then the scan and one subtraction. All + * arithmetic is saturating u8, and sat(sat(x - a) - b) = sat(x - sat(a + b)), + * so costs may be summed before they are subtracted. Returns the maximum of + * H over the final row. + */ +template +static uint8_t alignEE8V256Short( + const __m256i *prof, // query profile: per reference character, ITER (score, barrier) pairs + __m256i *mat, // matrix: column 0, row 0, E (SSEMatrix layout) + size_t colstride, // vectors per matrix column + const char *rf, // reference + size_t rfi, // first column + size_t rff, // last column (excl.) + const Scoring& sc, // gap penalties + size_t lastWordIdx) // final row: byte offset from a column's first H vector +{ + const size_t E = SSEMatrix::E, F = SSEMatrix::F, H = SSEMatrix::H; + const __m256i zero = _mm256_setzero_si256(); + const __m256i vrfgapo = _mm256_set1_epi8((char)sc.refGapOpen()); + const __m256i vrfgape = _mm256_set1_epi8((char)sc.refGapExtend()); + const __m256i vrdgapo = _mm256_set1_epi8((char)sc.readGapOpen()); + const __m256i vrdgape = _mm256_set1_epi8((char)sc.readGapExtend()); + // Gap barrier rows (the same for every reference character), and the costs + // derived from them, per vector row j: + // bpe: F carried from row j to row j + 1 (barrier, then rfgape); + // cpen: H0 at row j to the lane's carry-out (rfgapo, then bpe below); + // vpen: the carry into the lane to row j (as the general loop's vpen); + // epen: H to the next column's E (rdgapo, then barrier). + __m256i bar[ITER], bpe[ITER], cpen[ITER], vpen[ITER], epen[ITER]; + __m256i vbar = zero; + for(size_t j = 0; j < ITER; j++) { + bar[j] = _mm256_load_si256(prof + 2 * j + 1); + bpe[j] = _mm256_adds_epu8(bar[j], vrfgape); + epen[j] = _mm256_adds_epu8(vrdgapo, bar[j]); + vpen[j] = j == 0 ? bar[0] : _mm256_adds_epu8(_mm256_adds_epu8(vpen[j-1], vrfgape), bar[j]); + vbar = _mm256_or_si256(vbar, bar[j]); + } + cpen[ITER-1] = vrfgapo; + for(size_t j = ITER-1; j > 0; j--) { + cpen[j-1] = _mm256_adds_epu8(cpen[j], bpe[j]); + } + // The scan's costs: passing lane L whole is pass(L) (as the general loop's + // vpass); qN(L) is the cost of passing the N lanes that end at L, within + // a half (lanes outside the half are never read). chi(L) is the cost of + // passing lanes 16 to L - 1, for the carry from lane 15 into lane L > 16. + size_t passPen = ITER * (size_t)sc.refGapExtend(); + if(passPen > 0xff) { + passPen = 0xff; + } + const __m256i q1 = _mm256_or_si256(_mm256_set1_epi8((char)passPen), vbar); + const __m256i q2 = _mm256_adds_epu8(q1, _mm256_slli_si256(q1, 1)); + const __m256i q3 = _mm256_adds_epu8(q2, _mm256_slli_si256(q1, 2)); + const __m256i q4 = _mm256_adds_epu8(q2, _mm256_slli_si256(q2, 2)); + const __m256i q8 = _mm256_adds_epu8(q4, _mm256_slli_si256(q4, 4)); + const __m256i q12 = _mm256_adds_epu8(q8, _mm256_slli_si256(q4, 8)); + const __m256i q16 = _mm256_adds_epu8(q8, _mm256_slli_si256(q8, 8)); + const __m256i chi = _mm256_slli_si256(q16, 1); + // Selects byte 15 of the upper half (lane 15, once moved there) into + // every byte of the upper half, and zero into the lower half + const __m256i vidx15 = _mm256_set_epi64x( + 0x0f0f0f0f0f0f0f0fLL, 0x0f0f0f0f0f0f0f0fLL, + (long long)0x8080808080808080ULL, (long long)0x8080808080808080ULL); + // Byte 15 of the lower half is 0xff: shifted into lane 0, it is row 0's + // diagonal + const __m256i vtop = _mm256_set_epi64x(0, 0, (long long)0xff00000000000000ULL, 0); + // Costs to the next column's carry-out, max_j(H0'[j] - cpen[j]), where + // H0'[j] = max(diag[j] - profile'[j], E'[j]), E'[j] = max(E[j] - rdgape, + // H[j] - epen[j]), H[j] = max(HL[j], c - vpen[j]) and diag[j] = H[j - 1] + // or, for j = 0, H[ITER - 1] shifted down a lane: + // rc[j]: from E[j] (rdgape, cpen); ec[j]: from H[j] (epen, cpen); + // pc[r][j]: from diag[j], for reference character r (profile, cpen); + // zc[r]: from c, the least over its paths; wc[r]: from c shifted down a + // lane; kb[r]: from lane 15 of the scan, carried into the upper half, + // for both (c and c shifted are taken from the scan's values directly). + __m256i rc[ITER], ec[ITER], pc[5][ITER], zc[5], wc[5], kb[5]; + // chi for c shifted down a lane: lane 16 takes lane 14's value, with no + // carry from lane 15 + const __m256i chish = _mm256_or_si256(_mm256_slli_si256(chi, 1), + _mm256_set_epi64x(0, 0xff, 0, 0)); + const __m256i vpenSh = _mm256_alignr_epi8(vpen[ITER-1], + _mm256_permute2x128_si256(vpen[ITER-1], vpen[ITER-1], 0x08), 15); + for(size_t j = 0; j < ITER; j++) { + rc[j] = _mm256_adds_epu8(vrdgape, cpen[j]); + ec[j] = _mm256_adds_epu8(epen[j], cpen[j]); + } + for(size_t r = 0; r < 5; r++) { + __m256i z = _mm256_adds_epu8(vpen[0], ec[0]); + for(size_t j = 0; j < ITER; j++) { + pc[r][j] = _mm256_adds_epu8(_mm256_load_si256(prof + (r * ITER + j) * 2), cpen[j]); + if(j > 0) { + z = _mm256_min_epu8(z, _mm256_adds_epu8(vpen[j-1], pc[r][j])); + z = _mm256_min_epu8(z, _mm256_adds_epu8(vpen[j], ec[j])); + } + } + zc[r] = z; + wc[r] = _mm256_adds_epu8(vpenSh, pc[r][0]); + kb[r] = _mm256_min_epu8(_mm256_adds_epu8(chi, zc[r]), _mm256_adds_epu8(chish, wc[r])); + } + // lpen[j]: H0 at row j to the carry into the lane's last row + __m256i lpen[ITER-1]; + lpen[ITER-2] = vrfgapo; + for(size_t j = ITER-2; j > 0; j--) { + lpen[j-1] = _mm256_adds_epu8(lpen[j], bpe[j]); + } + // H0 and E of the column, and its carry-out: for the first, from the + // staged initial column (zero), whose diagonal is 0xff in row 0 only + __m256i h0[ITER], ecur[ITER], co; + { + const __m256i *pvS = prof + (size_t)firsts5[(int)rf[rfi]] * ITER * 2; + const __m256i p = _mm256_permute2x128_si256(zero, vtop, 0x02); + const __m256i d0 = _mm256_alignr_epi8(zero, p, 15); + h0[0] = _mm256_subs_epu8(d0, _mm256_load_si256(pvS)); + co = _mm256_subs_epu8(h0[0], cpen[0]); + for(size_t j = 1; j < ITER; j++) { + h0[j] = zero; + } + for(size_t j = 0; j < ITER; j++) { + ecur[j] = zero; + } + } + uint8_t lrmax = 0; + __m256i *pvCol = mat; + for(size_t i = rfi; i < rff; i++) { + // The next column's reference character. (After the last column, + // the values computed with it are not used.) + const size_t rn = (size_t)firsts5[(int)rf[i + 1 < rff ? i + 1 : i]]; + // The scan within each half, four lanes per step: over the 4 lanes + // ending at each lane, then over 16 + __m256i y = co; + y = _mm256_max_epu8( + _mm256_max_epu8(y, _mm256_subs_epu8(_mm256_slli_si256(y, 1), q1)), + _mm256_max_epu8(_mm256_subs_epu8(_mm256_slli_si256(y, 2), q2), + _mm256_subs_epu8(_mm256_slli_si256(y, 3), q3))); + y = _mm256_max_epu8( + _mm256_max_epu8(y, _mm256_subs_epu8(_mm256_slli_si256(y, 4), q4)), + _mm256_max_epu8(_mm256_subs_epu8(_mm256_slli_si256(y, 8), q8), + _mm256_subs_epu8(_mm256_slli_si256(y, 12), q12))); + // The carry into each lane is the scan's value one lane up (y15), and + // into the upper half also lane 15's (yc), carried down + const __m256i py = _mm256_permute2x128_si256(y, y, 0x08); + const __m256i y15 = _mm256_alignr_epi8(y, py, 15); + const __m256i y14 = _mm256_alignr_epi8(y, py, 14); + const __m256i yc = _mm256_shuffle_epi8(py, vidx15); + const __m256i c = _mm256_max_epu8(y15, _mm256_subs_epu8(yc, chi)); + // The next column's carry-out: its terms in c and in c shifted down a + // lane (from y14 and yc) + const __m256i cc = _mm256_max_epu8( + _mm256_subs_epu8(y14, wc[rn]), _mm256_subs_epu8(yc, kb[rn])); + // F and H with the carries from within the lane only + __m256i fl[ITER], hl[ITER]; + fl[0] = zero; + hl[0] = h0[0]; + { + __m256i vf = _mm256_subs_epu8(h0[0], vrfgapo); + for(size_t j = 1; j < ITER; j++) { + fl[j] = _mm256_subs_epu8(vf, bar[j]); + hl[j] = _mm256_max_epu8(h0[j], fl[j]); + if(j + 2 < ITER) { + vf = _mm256_max_epu8( + _mm256_subs_epu8(vf, bpe[j]), _mm256_subs_epu8(h0[j], vrfgapo)); + } else if(j + 2 == ITER) { + // into the last row, as a maximum over the rows above + __m256i fz = _mm256_subs_epu8(h0[0], lpen[0]); + for(size_t k = 1; k < ITER-1; k++) { + fz = _mm256_max_epu8(fz, _mm256_subs_epu8(h0[k], lpen[k])); + } + vf = fz; + } + } + } + // ... and its other terms, the latest (from the last row of HL) last + { + __m256i a = _mm256_max_epu8( + _mm256_subs_epu8(ecur[0], rc[0]), _mm256_subs_epu8(ecur[1], rc[1])); + for(size_t j = 2; j < ITER; j++) { + a = _mm256_max_epu8(a, _mm256_subs_epu8(ecur[j], rc[j])); + } + for(size_t j = 0; j + 1 < ITER; j++) { + a = _mm256_max_epu8(a, _mm256_max_epu8( + _mm256_subs_epu8(hl[j], ec[j]), _mm256_subs_epu8(hl[j], pc[rn][j+1]))); + } + const __m256i hlsh = _mm256_alignr_epi8(hl[ITER-1], + _mm256_permute2x128_si256(hl[ITER-1], vtop, 0x02), 15); + a = _mm256_max_epu8(a, _mm256_max_epu8( + _mm256_subs_epu8(hl[ITER-1], ec[ITER-1]), _mm256_subs_epu8(hlsh, pc[rn][0]))); + co = _mm256_max_epu8(_mm256_max_epu8(a, _mm256_subs_epu8(y15, zc[rn])), cc); + } + // F, H and the next column's E with the carry from the lanes above + __m256i hcur[ITER]; + __m256i *pv = pvCol; + for(size_t j = 0; j < ITER; j++) { + const __m256i s = _mm256_subs_epu8(c, vpen[j]); + const __m256i f = j == 0 ? s : _mm256_max_epu8(fl[j], s); + const __m256i h = _mm256_max_epu8(hl[j], s); + const __m256i e = _mm256_max_epu8( + _mm256_subs_epu8(ecur[j], vrdgape), _mm256_subs_epu8(h, epen[j])); + _mm256_store_si256(pv + F, f); + _mm256_store_si256(pv + H, h); + _mm256_store_si256(pv + colstride + E, e); + pv += ROWSTRIDE; + hcur[j] = h; + ecur[j] = e; + } + const uint8_t lr = reinterpret_cast(pvCol + H)[lastWordIdx]; + if(lr > lrmax) { + lrmax = lr; + } + pvCol += colstride; + // H0 of the next column. Its diagonal is row j - 1 of this column, + // or for j = 0 the last vector row shifted down a lane, with 0xff in + // lane 0. + const __m256i *pvS = prof + rn * ITER * 2; + { + const __m256i p = _mm256_permute2x128_si256(hcur[ITER-1], vtop, 0x02); + const __m256i d0 = _mm256_alignr_epi8(hcur[ITER-1], p, 15); + h0[0] = _mm256_max_epu8(_mm256_subs_epu8(d0, _mm256_load_si256(pvS)), ecur[0]); + } + for(size_t j = 1; j < ITER; j++) { + h0[j] = _mm256_max_epu8( + _mm256_subs_epu8(hcur[j-1], _mm256_load_si256(pvS + 2 * j)), ecur[j]); + } + } + return lrmax; +} + +/** + * How many columns before its first exact column alignEE8V256ShortSplit + * starts its second chain. A fill started at column s computes every cell + * as the maximum over the paths that begin in row 0 at column s or later; the + * true fill also has paths from earlier columns. Such a path reaches column + * s + W only after advancing W + 1 columns or more while descending at most + * 32 * ITER - 1 rows (padding lanes included), so it takes at least + * W + 2 - 32 * ITER read-gap steps, each costing rdgape or more. With + * W = 32 * ITER - 2 + ceil(255 / rdgape) its score has saturated to 0, which + * no cell is below, so from column s + W on the two fills are equal. (With + * free diagonals and rdgapo = rdgape the bound is reached: one column fewer + * is not exact.) + */ +template +static inline size_t alignEE8V256ShortWarmup(const Scoring& sc) { + const size_t rdgape = (size_t)sc.readGapExtend(); + assert_gt(rdgape, 0); + return 32 * ITER - 2 + (255 + rdgape - 1) / rdgape; +} + +/** + * alignEE8V256Short for a fill of many columns (e.g. a mate-rescue window), + * run as two chains of columns at once: chain A over the first n columns + * from rfi, and chain B over the last n columns to rff, started as a fresh + * fill W = alignEE8V256ShortWarmup() columns before the first column A does + * not cover, so it is exact from there on (see alignEE8V256ShortWarmup). + * Where the two overlap, A writes each column at a later step than B (A + * started earlier), so the matrix ends with A's values there; B's values in + * the warm-up are at most the true ones, so they do not raise the final-row + * maximum. + * + * Each step computes alignEE8V256Short's column, with one difference: the + * next column's carry-out, the scan's input, is taken directly as + * max_j(H0'[j] - cpen[j]) from the next column's H0, rather than as the + * maximum of early terms (in HL and E) and late terms (in the scan's + * values). That puts H, E and H0 on the chain from one column to the next, + * but needs about 25 fewer instructions per column; with two chains in + * flight the fill is bound by issue rather than by one chain's latency, and + * the alignEE8V256Short form of two chains was no faster than one chain. + * Requires rff - rfi > W + 1. + */ +template +static uint8_t alignEE8V256ShortSplit( + const __m256i *prof, // query profile: per reference character, ITER (score, barrier) pairs + __m256i *mat, // matrix: column 0, row 0, E (SSEMatrix layout) + size_t colstride, // vectors per matrix column + const char *rf, // reference + size_t rfi, // first column + size_t rff, // last column (excl.) + const Scoring& sc, // gap penalties + size_t lastWordIdx, // final row: byte offset from a column's first H vector + size_t W) // alignEE8V256ShortWarmup(sc) +{ + assert_gt(rff - rfi, W + 1); + const size_t E = SSEMatrix::E, F = SSEMatrix::F, H = SSEMatrix::H; + const __m256i zero = _mm256_setzero_si256(); + const __m256i vrfgapo = _mm256_set1_epi8((char)sc.refGapOpen()); + const __m256i vrfgape = _mm256_set1_epi8((char)sc.refGapExtend()); + const __m256i vrdgapo = _mm256_set1_epi8((char)sc.readGapOpen()); + const __m256i vrdgape = _mm256_set1_epi8((char)sc.readGapExtend()); + // The costs of alignEE8V256Short, per vector row j (bpe, cpen, vpen, epen + // as there; lpen: H0 at row j to the carry into the lane's last row) + __m256i bar[ITER], bpe[ITER], cpen[ITER], vpen[ITER], epen[ITER]; + __m256i vbar = zero; + for(size_t j = 0; j < ITER; j++) { + bar[j] = _mm256_load_si256(prof + 2 * j + 1); + bpe[j] = _mm256_adds_epu8(bar[j], vrfgape); + epen[j] = _mm256_adds_epu8(vrdgapo, bar[j]); + vpen[j] = j == 0 ? bar[0] : _mm256_adds_epu8(_mm256_adds_epu8(vpen[j-1], vrfgape), bar[j]); + vbar = _mm256_or_si256(vbar, bar[j]); + } + cpen[ITER-1] = vrfgapo; + for(size_t j = ITER-1; j > 0; j--) { + cpen[j-1] = _mm256_adds_epu8(cpen[j], bpe[j]); + } + __m256i lpen[ITER-1]; + lpen[ITER-2] = vrfgapo; + for(size_t j = ITER-2; j > 0; j--) { + lpen[j-1] = _mm256_adds_epu8(lpen[j], bpe[j]); + } + // The scan's costs (q1-q12, chi) and constants, as in alignEE8V256Short + size_t passPen = ITER * (size_t)sc.refGapExtend(); + if(passPen > 0xff) { + passPen = 0xff; + } + const __m256i q1 = _mm256_or_si256(_mm256_set1_epi8((char)passPen), vbar); + const __m256i q2 = _mm256_adds_epu8(q1, _mm256_slli_si256(q1, 1)); + const __m256i q3 = _mm256_adds_epu8(q2, _mm256_slli_si256(q1, 2)); + const __m256i q4 = _mm256_adds_epu8(q2, _mm256_slli_si256(q2, 2)); + const __m256i q8 = _mm256_adds_epu8(q4, _mm256_slli_si256(q4, 4)); + const __m256i q12 = _mm256_adds_epu8(q8, _mm256_slli_si256(q4, 8)); + const __m256i q16 = _mm256_adds_epu8(q8, _mm256_slli_si256(q8, 8)); + const __m256i chi = _mm256_slli_si256(q16, 1); + const __m256i vidx15 = _mm256_set_epi64x( + 0x0f0f0f0f0f0f0f0fLL, 0x0f0f0f0f0f0f0f0fLL, + (long long)0x8080808080808080ULL, (long long)0x8080808080808080ULL); + const __m256i vtop = _mm256_set_epi64x(0, 0, (long long)0xff00000000000000ULL, 0); + const size_t n = (rff - rfi + W + 1) / 2; // columns per chain + const size_t sb = rff - n; // B's first column + // One chain's state between columns: H0 and E of its next column, and + // that column's carry-out + __m256i h0a[ITER], ea[ITER], coa, h0b[ITER], eb[ITER], cob; + // A chain's first column, from a staged initial column (zero), whose + // diagonal is 0xff in row 0 only + auto start = [&](__m256i *h0, __m256i *ecur, __m256i& co, size_t col) __attribute__((always_inline)) { + const __m256i *pvS = prof + (size_t)firsts5[(int)rf[col]] * ITER * 2; + const __m256i p = _mm256_permute2x128_si256(zero, vtop, 0x02); + const __m256i d0 = _mm256_alignr_epi8(zero, p, 15); + h0[0] = _mm256_subs_epu8(d0, _mm256_load_si256(pvS)); + co = _mm256_subs_epu8(h0[0], cpen[0]); + for(size_t j = 1; j < ITER; j++) { + h0[j] = zero; + } + for(size_t j = 0; j < ITER; j++) { + ecur[j] = zero; + } + }; + // One column of a chain: stores its F, H and the next column's E at + // pvCol, advances the state to the next column (reference character rn) + // and returns the column's final-row H + auto step = [&](__m256i *h0, __m256i *ecur, __m256i& co, size_t rn, + __m256i *pvCol) __attribute__((always_inline)) -> uint8_t + { + // The scan within each half, then the carry into each lane (as + // alignEE8V256Short) + __m256i y = co; + y = _mm256_max_epu8( + _mm256_max_epu8(y, _mm256_subs_epu8(_mm256_slli_si256(y, 1), q1)), + _mm256_max_epu8(_mm256_subs_epu8(_mm256_slli_si256(y, 2), q2), + _mm256_subs_epu8(_mm256_slli_si256(y, 3), q3))); + y = _mm256_max_epu8( + _mm256_max_epu8(y, _mm256_subs_epu8(_mm256_slli_si256(y, 4), q4)), + _mm256_max_epu8(_mm256_subs_epu8(_mm256_slli_si256(y, 8), q8), + _mm256_subs_epu8(_mm256_slli_si256(y, 12), q12))); + const __m256i py = _mm256_permute2x128_si256(y, y, 0x08); + const __m256i y15 = _mm256_alignr_epi8(y, py, 15); + const __m256i yc = _mm256_shuffle_epi8(py, vidx15); + const __m256i c = _mm256_max_epu8(y15, _mm256_subs_epu8(yc, chi)); + // F and H with the carries from within the lane only + __m256i fl[ITER], hl[ITER]; + fl[0] = zero; + hl[0] = h0[0]; + { + __m256i vf = _mm256_subs_epu8(h0[0], vrfgapo); + for(size_t j = 1; j < ITER; j++) { + fl[j] = _mm256_subs_epu8(vf, bar[j]); + hl[j] = _mm256_max_epu8(h0[j], fl[j]); + if(j + 2 < ITER) { + vf = _mm256_max_epu8( + _mm256_subs_epu8(vf, bpe[j]), _mm256_subs_epu8(h0[j], vrfgapo)); + } else if(j + 2 == ITER) { + // into the last row, as a maximum over the rows above + __m256i fz = _mm256_subs_epu8(h0[0], lpen[0]); + for(size_t k = 1; k < ITER-1; k++) { + fz = _mm256_max_epu8(fz, _mm256_subs_epu8(h0[k], lpen[k])); + } + vf = fz; + } + } + } + // F, H and the next column's E with the carry from the lanes above + __m256i hcur[ITER]; + __m256i *pv = pvCol; + for(size_t j = 0; j < ITER; j++) { + const __m256i s = _mm256_subs_epu8(c, vpen[j]); + const __m256i f = j == 0 ? s : _mm256_max_epu8(fl[j], s); + const __m256i h = _mm256_max_epu8(hl[j], s); + const __m256i e = _mm256_max_epu8( + _mm256_subs_epu8(ecur[j], vrdgape), _mm256_subs_epu8(h, epen[j])); + _mm256_store_si256(pv + F, f); + _mm256_store_si256(pv + H, h); + _mm256_store_si256(pv + colstride + E, e); + pv += ROWSTRIDE; + hcur[j] = h; + ecur[j] = e; + } + const uint8_t lr = reinterpret_cast(pvCol + H)[lastWordIdx]; + // H0 of the next column, and its carry-out + const __m256i *pvS = prof + rn * ITER * 2; + { + const __m256i p = _mm256_permute2x128_si256(hcur[ITER-1], vtop, 0x02); + const __m256i d0 = _mm256_alignr_epi8(hcur[ITER-1], p, 15); + h0[0] = _mm256_max_epu8(_mm256_subs_epu8(d0, _mm256_load_si256(pvS)), ecur[0]); + } + for(size_t j = 1; j < ITER; j++) { + h0[j] = _mm256_max_epu8( + _mm256_subs_epu8(hcur[j-1], _mm256_load_si256(pvS + 2 * j)), ecur[j]); + } + __m256i m = _mm256_max_epu8( + _mm256_subs_epu8(h0[0], cpen[0]), _mm256_subs_epu8(h0[1], cpen[1])); + for(size_t j = 2; j + 1 < ITER; j += 2) { + m = _mm256_max_epu8(m, _mm256_max_epu8( + _mm256_subs_epu8(h0[j], cpen[j]), _mm256_subs_epu8(h0[j+1], cpen[j+1]))); + } + if(ITER % 2 == 1) { + m = _mm256_max_epu8(m, _mm256_subs_epu8(h0[ITER-1], cpen[ITER-1])); + } + co = m; + return lr; + }; + start(h0a, ea, coa, rfi); + start(h0b, eb, cob, sb); + uint8_t lrmax = 0; + __m256i *pa = mat, *pb = mat + colstride * (sb - rfi); + for(size_t k = 0; k < n; k++) { + // The next columns' reference characters. A's next column is at + // most rfi + n < rff; after B's last column, the values computed with + // its character are not used. + const size_t ra = (size_t)firsts5[(int)rf[rfi + k + 1]]; + const size_t ib = sb + k + 1 < rff ? sb + k + 1 : sb + k; + const size_t rb = (size_t)firsts5[(int)rf[ib]]; + const uint8_t la = step(h0a, ea, coa, ra, pa); + const uint8_t lb = step(h0b, eb, cob, rb, pb); + const uint8_t l = la > lb ? la : lb; + lrmax = l > lrmax ? l : lrmax; + pa += colstride; + pb += colstride; + } + return lrmax; +} + +/** + * The fill of alignNucleotidesEnd2EndSseU8 for a column of one 256-bit vector + * (up to 32 rows), whose lanes are the rows in order. The column is held as + * two 128-bit halves (rows 0-15 and 16-31), so that a shift across them is a + * single in-lane instruction. It computes the general loop's saturating u8 + * recurrence and stores the same E, F and H vectors in the same places: + * - H0 = max(diag - profile, E), with F = 0 from the first pass, as every + * row is the first of its lane; + * - F is the general loop's prefix scan of the carries H0 - rfgapo, run + * within each half, four rows per step, and then carried from row 15 into + * the upper half; rfgapo is subtracted after the scan, with the barrier; + * - H = max(H0, F), and the next column's E = max(E - rdgape, H - rdgapo - + * barrier). + * The speed is set by the chain of dependent instructions from one column to + * the next. The next column's H0, the scan's input, is max(diag' - + * profile', E') with diag' = H shifted down a row; as subtraction + * distributes over max, it is also the max of terms in this column's H0 and + * E, which are ready while the scan runs, and of terms in the scan's values + * shifted down one and two rows, each with its whole cost. The chain is + * then the scan, one shift and one subtraction. sat(sat(x - a) - b) = + * sat(x - sat(a + b)), so costs may be summed before they are subtracted. + * Returns the maximum of H over the final row. + */ +static uint8_t alignEE8V256Row32( + const __m128i *pvProf, // query profile: per reference character, the score and barrier halves + __m128i *pvCol, // matrix: column 0, row 0, E (SSEMatrix layout, in halves) + size_t colstride, // 256-bit vectors per matrix column + const char *rf, // reference + size_t rfi, // first column + size_t rff, // last column (excl.) + const Scoring& sc, // gap penalties + size_t lastWordIdx) // final row +{ + const size_t E = SSEMatrix::E, F = SSEMatrix::F, H = SSEMatrix::H; + // Gap barrier rows (the same for every reference character) + const __m128i barlo = _mm_load_si128(pvProf + 2); + const __m128i barhi = _mm_load_si128(pvProf + 3); + // Cost of a carry passing a row: rfgape, or all of it in the barrier. + // passN: the cost of passing the N rows that end at each row, within a + // half. + const __m128i vrfgape = _mm_set1_epi8((char)sc.refGapExtend()); + const __m128i pass1lo = _mm_or_si128(vrfgape, barlo); + const __m128i pass1hi = _mm_or_si128(vrfgape, barhi); + const __m128i pass2lo = _mm_adds_epu8(pass1lo, _mm_slli_si128(pass1lo, 1)); + const __m128i pass2hi = _mm_adds_epu8(pass1hi, _mm_slli_si128(pass1hi, 1)); + const __m128i pass3lo = _mm_adds_epu8(pass2lo, _mm_slli_si128(pass1lo, 2)); + const __m128i pass3hi = _mm_adds_epu8(pass2hi, _mm_slli_si128(pass1hi, 2)); + const __m128i pass4lo = _mm_adds_epu8(pass2lo, _mm_slli_si128(pass2lo, 2)); + const __m128i pass4hi = _mm_adds_epu8(pass2hi, _mm_slli_si128(pass2hi, 2)); + const __m128i pass8lo = _mm_adds_epu8(pass4lo, _mm_slli_si128(pass4lo, 4)); + const __m128i pass8hi = _mm_adds_epu8(pass4hi, _mm_slli_si128(pass4hi, 4)); + const __m128i pass12lo = _mm_adds_epu8(pass8lo, _mm_slli_si128(pass4lo, 8)); + const __m128i pass12hi = _mm_adds_epu8(pass8hi, _mm_slli_si128(pass4hi, 8)); + const __m128i pass16hi = _mm_adds_epu8(pass8hi, _mm_slli_si128(pass8hi, 8)); + // rfgapo or rdgapo, plus the gap barrier + const __m128i vrfgapo = _mm_set1_epi8((char)sc.refGapOpen()); + const __m128i vrdgapo = _mm_set1_epi8((char)sc.readGapOpen()); + const __m128i fpenlo = _mm_adds_epu8(vrfgapo, barlo); + const __m128i fpenhi = _mm_adds_epu8(vrfgapo, barhi); + const __m128i epenlo = _mm_adds_epu8(vrdgapo, barlo); + const __m128i epenhi = _mm_adds_epu8(vrdgapo, barhi); + // The carry from row 15 to row 16 + k, k >= 1, loses the cost of + // passing rows 16 to 15 + k, then rfgapo and row 16 + k's barrier; row + // 16 itself takes row 15's value through the shift. carryhiSh: the same + // for F shifted down a row. + const __m128i carryhi = _mm_adds_epu8( + _mm_or_si128(_mm_slli_si128(pass16hi, 1), _mm_cvtsi32_si128(0xff)), fpenhi); + const __m128i carryhiSh = _mm_or_si128(_mm_slli_si128(carryhi, 1), _mm_cvtsi32_si128(0xff)); + const __m128i vrdgape = _mm_set1_epi8((char)sc.readGapExtend()); + // Byte 15 is 0xff: shifted into row 0, it is row 0's diagonal + const __m128i vtop = _mm_slli_si128(_mm_cvtsi32_si128(0xff), 15); + const __m128i vrow15 = _mm_set1_epi8(15); + // Costs to the next column's H0 from the scan's values: + // k1: one row down (F, then rdgapo and the barrier into E'); + // k2[r]: two rows down (F one row up, then the diagonal into H0 for + // reference character r); + // kbc[r]: from row 15 carried into the upper half, for both. + const __m128i k1lo = _mm_adds_epu8(fpenlo, epenlo); + const __m128i k1hi = _mm_adds_epu8(fpenhi, epenhi); + const __m128i fpenShlo = _mm_slli_si128(fpenlo, 1); + const __m128i fpenShhi = _mm_alignr_epi8(fpenhi, fpenlo, 15); + __m128i k2lo[5], k2hi[5], kbc[5]; + for(size_t r = 0; r < 5; r++) { + k2lo[r] = _mm_adds_epu8(fpenShlo, _mm_load_si128(pvProf + 4 * r)); + k2hi[r] = _mm_adds_epu8(fpenShhi, _mm_load_si128(pvProf + 4 * r + 1)); + kbc[r] = _mm_min_epu8(_mm_adds_epu8(carryhi, epenhi), + _mm_adds_epu8(carryhiSh, _mm_load_si128(pvProf + 4 * r + 1))); + } + // H0 and E of the column: for the first, from the staged initial column + // (zero) + __m128i h0lo, h0hi; + { + const size_t r = (size_t)firsts5[(int)rf[rfi]]; + h0lo = _mm_subs_epu8(_mm_cvtsi32_si128(0xff), _mm_load_si128(pvProf + 4 * r)); + h0hi = _mm_setzero_si128(); + } + __m128i elo = _mm_setzero_si128(), ehi = _mm_setzero_si128(); + // Maximum of H over the columns, for the final row's maximum + __m128i hmaxlo = _mm_setzero_si128(), hmaxhi = _mm_setzero_si128(); + for(size_t i = rfi; i < rff; i++) { + // The next column's reference character. (After the last column, + // the values computed with it are not used.) + const size_t rn = (size_t)firsts5[(int)rf[i + 1 < rff ? i + 1 : i]]; + // The scan within each half, four rows per step: over the 4 rows + // ending at each row, then over 16 + __m128i ylo = h0lo, yhi = h0hi; + ylo = _mm_max_epu8( + _mm_max_epu8(ylo, _mm_subs_epu8(_mm_slli_si128(ylo, 1), pass1lo)), + _mm_max_epu8(_mm_subs_epu8(_mm_slli_si128(ylo, 2), pass2lo), + _mm_subs_epu8(_mm_slli_si128(ylo, 3), pass3lo))); + yhi = _mm_max_epu8( + _mm_max_epu8(yhi, _mm_subs_epu8(_mm_slli_si128(yhi, 1), pass1hi)), + _mm_max_epu8(_mm_subs_epu8(_mm_slli_si128(yhi, 2), pass2hi), + _mm_subs_epu8(_mm_slli_si128(yhi, 3), pass3hi))); + ylo = _mm_max_epu8( + _mm_max_epu8(ylo, _mm_subs_epu8(_mm_slli_si128(ylo, 4), pass4lo)), + _mm_max_epu8(_mm_subs_epu8(_mm_slli_si128(ylo, 8), pass8lo), + _mm_subs_epu8(_mm_slli_si128(ylo, 12), pass12lo))); + yhi = _mm_max_epu8( + _mm_max_epu8(yhi, _mm_subs_epu8(_mm_slli_si128(yhi, 4), pass4hi)), + _mm_max_epu8(_mm_subs_epu8(_mm_slli_si128(yhi, 8), pass8hi), + _mm_subs_epu8(_mm_slli_si128(yhi, 12), pass12hi))); + // The scan's values one row down (the carry into each row), two rows + // down, and row 15's, for the upper half + const __m128i y1lo = _mm_slli_si128(ylo, 1); + const __m128i y2lo = _mm_slli_si128(ylo, 2); + const __m128i y1hi = _mm_alignr_epi8(yhi, ylo, 15); + const __m128i y2hi = _mm_alignr_epi8(yhi, ylo, 14); + const __m128i y15 = _mm_shuffle_epi8(ylo, vrow15); + // The next column's H0: its terms in this column's H0 and E ... + const __m128i glo = _mm_max_epu8( + _mm_max_epu8( + _mm_subs_epu8(_mm_alignr_epi8(h0lo, vtop, 15), _mm_load_si128(pvProf + 4 * rn)), + _mm_subs_epu8(elo, vrdgape)), + _mm_subs_epu8(h0lo, epenlo)); + const __m128i ghi = _mm_max_epu8( + _mm_max_epu8( + _mm_subs_epu8(_mm_alignr_epi8(h0hi, h0lo, 15), _mm_load_si128(pvProf + 4 * rn + 1)), + _mm_subs_epu8(ehi, vrdgape)), + _mm_subs_epu8(h0hi, epenhi)); + // ... and in F, from the scan + const __m128i nlo = _mm_max_epu8( + _mm_max_epu8(glo, _mm_subs_epu8(y1lo, k1lo)), _mm_subs_epu8(y2lo, k2lo[rn])); + const __m128i nhi = _mm_max_epu8( + _mm_max_epu8(ghi, _mm_subs_epu8(y1hi, k1hi)), + _mm_max_epu8(_mm_subs_epu8(y2hi, k2hi[rn]), _mm_subs_epu8(y15, kbc[rn]))); + // This column's F, H and the next column's E + const __m128i flo = _mm_subs_epu8(y1lo, fpenlo); + const __m128i fhi = _mm_max_epu8( + _mm_subs_epu8(y1hi, fpenhi), _mm_subs_epu8(y15, carryhi)); + const __m128i hlo = _mm_max_epu8(h0lo, flo); + const __m128i hhi = _mm_max_epu8(h0hi, fhi); + elo = _mm_max_epu8(_mm_subs_epu8(elo, vrdgape), _mm_subs_epu8(hlo, epenlo)); + ehi = _mm_max_epu8(_mm_subs_epu8(ehi, vrdgape), _mm_subs_epu8(hhi, epenhi)); + _mm_store_si128(pvCol + 2 * F, flo); + _mm_store_si128(pvCol + 2 * F + 1, fhi); + _mm_store_si128(pvCol + 2 * H, hlo); + _mm_store_si128(pvCol + 2 * H + 1, hhi); + pvCol += 2 * colstride; + _mm_store_si128(pvCol + 2 * E, elo); + _mm_store_si128(pvCol + 2 * E + 1, ehi); + hmaxlo = _mm_max_epu8(hmaxlo, hlo); + hmaxhi = _mm_max_epu8(hmaxhi, hhi); + h0lo = nlo; + h0hi = nhi; + } + uint8_t hmax[32]; + _mm_storeu_si128(reinterpret_cast<__m128i*>(hmax), hmaxlo); + _mm_storeu_si128(reinterpret_cast<__m128i*>(hmax + 16), hmaxhi); + return hmax[lastWordIdx]; +} + +/** + * How many columns before its first exact column alignEE8V256Row32Split + * starts its second chain. A fill started at column s computes every cell + * as the maximum over the paths that begin in row 0 at column s or later; the + * true fill also has paths from earlier columns. Such a path reaches column + * s + W only after advancing W + 1 columns or more while descending at most + * 31 rows, so it takes at least W - 30 read-gap steps, each costing rdgape or + * more. With W - 30 >= 255 / rdgape its score has saturated to 0, which no + * cell is below, so from column s + W on the two fills are equal. + */ +static inline size_t alignEE8V256Row32Warmup(const Scoring& sc) { + const size_t rdgape = (size_t)sc.readGapExtend(); + assert_gt(rdgape, 0); + return 32 + (255 + rdgape - 1) / rdgape; +} + +/** + * alignEE8V256Row32 for a fill of many columns. alignEE8V256Row32 is bound + * by the chain of dependent instructions from one column to the next, and + * uses only one 128-bit lane of the vector units. Here the fill runs as two + * chains, one in each 128-bit lane of 256-bit registers, doing the same + * column arithmetic: chain A in lane 0 over the first n columns from rfi, + * and chain B in lane 1 over the last n columns to rff. B starts as a fresh + * fill W = alignEE8V256Row32Warmup() columns before the first column A does + * not cover, so it is exact from there on (see alignEE8V256Row32Warmup). + * Where the two overlap, A writes each column at a later step than B (A + * started earlier), so the matrix ends with A's values there; B's values in + * the warm-up are at most the true ones, so they do not raise the final-row + * maximum. Register "lo" holds rows 0-15 of both chains and "hi" rows + * 16-31, so every shift of alignEE8V256Row32's column stays within a lane. + * The per-column costs that depend on the reference character are looked up + * per pair (A's character, B's). In each iteration the next column's scan + * comes before this column's F, H, E and stores, so that it is older than + * them when both are ready and goes first. Requires rff - rfi > W + 1. + */ +static uint8_t alignEE8V256Row32Split( + const __m128i *pvProf, // query profile: per reference character, the score and barrier halves + __m128i *pvCol, // matrix: column 0, row 0, E (SSEMatrix layout, in halves) + size_t colstride, // 256-bit vectors per matrix column + const char *rf, // reference + size_t rfi, // first column + size_t rff, // last column (excl.) + const Scoring& sc, // gap penalties + size_t lastWordIdx, // final row + size_t W) // alignEE8V256Row32Warmup(sc) +{ + assert_gt(rff - rfi, W + 1); + const size_t E = SSEMatrix::E, F = SSEMatrix::F, H = SSEMatrix::H; + // The costs of alignEE8V256Row32, for one chain (128 bits) + const __m128i barlo = _mm_load_si128(pvProf + 2); + const __m128i barhi = _mm_load_si128(pvProf + 3); + const __m128i vrfgape = _mm_set1_epi8((char)sc.refGapExtend()); + const __m128i pass1lo = _mm_or_si128(vrfgape, barlo); + const __m128i pass1hi = _mm_or_si128(vrfgape, barhi); + const __m128i pass2lo = _mm_adds_epu8(pass1lo, _mm_slli_si128(pass1lo, 1)); + const __m128i pass2hi = _mm_adds_epu8(pass1hi, _mm_slli_si128(pass1hi, 1)); + const __m128i pass3lo = _mm_adds_epu8(pass2lo, _mm_slli_si128(pass1lo, 2)); + const __m128i pass3hi = _mm_adds_epu8(pass2hi, _mm_slli_si128(pass1hi, 2)); + const __m128i pass4lo = _mm_adds_epu8(pass2lo, _mm_slli_si128(pass2lo, 2)); + const __m128i pass4hi = _mm_adds_epu8(pass2hi, _mm_slli_si128(pass2hi, 2)); + const __m128i pass8lo = _mm_adds_epu8(pass4lo, _mm_slli_si128(pass4lo, 4)); + const __m128i pass8hi = _mm_adds_epu8(pass4hi, _mm_slli_si128(pass4hi, 4)); + const __m128i pass12lo = _mm_adds_epu8(pass8lo, _mm_slli_si128(pass4lo, 8)); + const __m128i pass12hi = _mm_adds_epu8(pass8hi, _mm_slli_si128(pass4hi, 8)); + const __m128i pass16hi = _mm_adds_epu8(pass8hi, _mm_slli_si128(pass8hi, 8)); + const __m128i vrfgapo = _mm_set1_epi8((char)sc.refGapOpen()); + const __m128i vrdgapo = _mm_set1_epi8((char)sc.readGapOpen()); + const __m128i fpenlo = _mm_adds_epu8(vrfgapo, barlo); + const __m128i fpenhi = _mm_adds_epu8(vrfgapo, barhi); + const __m128i epenlo = _mm_adds_epu8(vrdgapo, barlo); + const __m128i epenhi = _mm_adds_epu8(vrdgapo, barhi); + const __m128i carryhi = _mm_adds_epu8( + _mm_or_si128(_mm_slli_si128(pass16hi, 1), _mm_cvtsi32_si128(0xff)), fpenhi); + const __m128i carryhiSh = _mm_or_si128(_mm_slli_si128(carryhi, 1), _mm_cvtsi32_si128(0xff)); + const __m128i k1lo = _mm_adds_epu8(fpenlo, epenlo); + const __m128i k1hi = _mm_adds_epu8(fpenhi, epenhi); + const __m128i fpenShlo = _mm_slli_si128(fpenlo, 1); + const __m128i fpenShhi = _mm_alignr_epi8(fpenhi, fpenlo, 15); + __m128i k2lo[5], k2hi[5], kbc[5]; + for(size_t r = 0; r < 5; r++) { + k2lo[r] = _mm_adds_epu8(fpenShlo, _mm_load_si128(pvProf + 4 * r)); + k2hi[r] = _mm_adds_epu8(fpenShhi, _mm_load_si128(pvProf + 4 * r + 1)); + kbc[r] = _mm_min_epu8(_mm_adds_epu8(carryhi, epenhi), + _mm_adds_epu8(carryhiSh, _mm_load_si128(pvProf + 4 * r + 1))); + } + // The same costs in both lanes + const __m256i wpass1lo = _mm256_broadcastsi128_si256(pass1lo); + const __m256i wpass1hi = _mm256_broadcastsi128_si256(pass1hi); + const __m256i wpass2lo = _mm256_broadcastsi128_si256(pass2lo); + const __m256i wpass2hi = _mm256_broadcastsi128_si256(pass2hi); + const __m256i wpass3lo = _mm256_broadcastsi128_si256(pass3lo); + const __m256i wpass3hi = _mm256_broadcastsi128_si256(pass3hi); + const __m256i wpass4lo = _mm256_broadcastsi128_si256(pass4lo); + const __m256i wpass4hi = _mm256_broadcastsi128_si256(pass4hi); + const __m256i wpass8lo = _mm256_broadcastsi128_si256(pass8lo); + const __m256i wpass8hi = _mm256_broadcastsi128_si256(pass8hi); + const __m256i wpass12lo = _mm256_broadcastsi128_si256(pass12lo); + const __m256i wpass12hi = _mm256_broadcastsi128_si256(pass12hi); + const __m256i wfpenlo = _mm256_broadcastsi128_si256(fpenlo); + const __m256i wfpenhi = _mm256_broadcastsi128_si256(fpenhi); + const __m256i wepenlo = _mm256_broadcastsi128_si256(epenlo); + const __m256i wepenhi = _mm256_broadcastsi128_si256(epenhi); + const __m256i wcarryhi = _mm256_broadcastsi128_si256(carryhi); + const __m256i wk1lo = _mm256_broadcastsi128_si256(k1lo); + const __m256i wk1hi = _mm256_broadcastsi128_si256(k1hi); + const __m256i wrdgape = _mm256_set1_epi8((char)sc.readGapExtend()); + // Byte 15 of each lane is 0xff: shifted into row 0, it is row 0's diagonal + const __m256i wtop = _mm256_broadcastsi128_si256(_mm_slli_si128(_mm_cvtsi32_si128(0xff), 15)); + const __m256i wrow15 = _mm256_set1_epi8(15); + // Per pair of reference characters (A's, B's): the profile's two halves, + // k2lo, k2hi and kbc, A's in lane 0 and B's in lane 1 + __m256i pair[25][5]; + for(size_t a = 0; a < 5; a++) { + for(size_t b = 0; b < 5; b++) { + const __m128i va[5] = { _mm_load_si128(pvProf + 4 * a), _mm_load_si128(pvProf + 4 * a + 1), + k2lo[a], k2hi[a], kbc[a] }; + const __m128i vb[5] = { _mm_load_si128(pvProf + 4 * b), _mm_load_si128(pvProf + 4 * b + 1), + k2lo[b], k2hi[b], kbc[b] }; + for(size_t m = 0; m < 5; m++) { + pair[a * 5 + b][m] = _mm256_inserti128_si256(_mm256_castsi128_si256(va[m]), vb[m], 1); + } + } + } + const size_t n = (rff - rfi + W + 1) / 2; // columns per chain + const size_t sb = rff - n; // B's first column + // alignEE8V256Row32's scan (F within each half), for both chains + auto scan = [&](__m256i ylo, __m256i yhi, __m256i& outlo, __m256i& outhi) { + ylo = _mm256_max_epu8( + _mm256_max_epu8(ylo, _mm256_subs_epu8(_mm256_slli_si256(ylo, 1), wpass1lo)), + _mm256_max_epu8(_mm256_subs_epu8(_mm256_slli_si256(ylo, 2), wpass2lo), + _mm256_subs_epu8(_mm256_slli_si256(ylo, 3), wpass3lo))); + yhi = _mm256_max_epu8( + _mm256_max_epu8(yhi, _mm256_subs_epu8(_mm256_slli_si256(yhi, 1), wpass1hi)), + _mm256_max_epu8(_mm256_subs_epu8(_mm256_slli_si256(yhi, 2), wpass2hi), + _mm256_subs_epu8(_mm256_slli_si256(yhi, 3), wpass3hi))); + outlo = _mm256_max_epu8( + _mm256_max_epu8(ylo, _mm256_subs_epu8(_mm256_slli_si256(ylo, 4), wpass4lo)), + _mm256_max_epu8(_mm256_subs_epu8(_mm256_slli_si256(ylo, 8), wpass8lo), + _mm256_subs_epu8(_mm256_slli_si256(ylo, 12), wpass12lo))); + outhi = _mm256_max_epu8( + _mm256_max_epu8(yhi, _mm256_subs_epu8(_mm256_slli_si256(yhi, 4), wpass4hi)), + _mm256_max_epu8(_mm256_subs_epu8(_mm256_slli_si256(yhi, 8), wpass8hi), + _mm256_subs_epu8(_mm256_slli_si256(yhi, 12), wpass12hi))); + }; + // H0 and E of each chain's first column, from the staged initial column + __m256i h0lo, h0hi; + { + const size_t ra = (size_t)firsts5[(int)rf[rfi]]; + const size_t rb = (size_t)firsts5[(int)rf[sb]]; + const __m256i first = _mm256_broadcastsi128_si256(_mm_cvtsi32_si128(0xff)); + h0lo = _mm256_subs_epu8(first, pair[ra * 5 + rb][0]); + h0hi = _mm256_setzero_si256(); + } + __m256i elo = _mm256_setzero_si256(), ehi = _mm256_setzero_si256(); + __m256i hmaxlo = _mm256_setzero_si256(), hmaxhi = _mm256_setzero_si256(); + __m128i *pa = pvCol, *pb = pvCol + 2 * colstride * (sb - rfi); + __m256i ylo, yhi; + scan(h0lo, h0hi, ylo, yhi); + for(size_t k = 0; k < n; k++) { + // The next columns' reference characters. A's next column is at + // most rfi + n < rff; after B's last column, the values computed with + // its character are not used. + const size_t ib = sb + k + 1 < rff ? sb + k + 1 : sb + k; + const __m256i *t = pair[(size_t)firsts5[(int)rf[rfi + k + 1]] * 5 + + (size_t)firsts5[(int)rf[ib]]]; + const __m256i y1lo = _mm256_slli_si256(ylo, 1); + const __m256i y2lo = _mm256_slli_si256(ylo, 2); + const __m256i y1hi = _mm256_alignr_epi8(yhi, ylo, 15); + const __m256i y2hi = _mm256_alignr_epi8(yhi, ylo, 14); + const __m256i y15 = _mm256_shuffle_epi8(ylo, wrow15); + const __m256i glo = _mm256_max_epu8( + _mm256_max_epu8( + _mm256_subs_epu8(_mm256_alignr_epi8(h0lo, wtop, 15), t[0]), + _mm256_subs_epu8(elo, wrdgape)), + _mm256_subs_epu8(h0lo, wepenlo)); + const __m256i ghi = _mm256_max_epu8( + _mm256_max_epu8( + _mm256_subs_epu8(_mm256_alignr_epi8(h0hi, h0lo, 15), t[1]), + _mm256_subs_epu8(ehi, wrdgape)), + _mm256_subs_epu8(h0hi, wepenhi)); + const __m256i nlo = _mm256_max_epu8( + _mm256_max_epu8(glo, _mm256_subs_epu8(y1lo, wk1lo)), _mm256_subs_epu8(y2lo, t[2])); + const __m256i nhi = _mm256_max_epu8( + _mm256_max_epu8(ghi, _mm256_subs_epu8(y1hi, wk1hi)), + _mm256_max_epu8(_mm256_subs_epu8(y2hi, t[3]), _mm256_subs_epu8(y15, t[4]))); + // The next column's scan, ahead of this column's F, H and E + __m256i ynlo, ynhi; + scan(nlo, nhi, ynlo, ynhi); + const __m256i flo = _mm256_subs_epu8(y1lo, wfpenlo); + const __m256i fhi = _mm256_max_epu8( + _mm256_subs_epu8(y1hi, wfpenhi), _mm256_subs_epu8(y15, wcarryhi)); + const __m256i hlo = _mm256_max_epu8(h0lo, flo); + const __m256i hhi = _mm256_max_epu8(h0hi, fhi); + elo = _mm256_max_epu8(_mm256_subs_epu8(elo, wrdgape), _mm256_subs_epu8(hlo, wepenlo)); + ehi = _mm256_max_epu8(_mm256_subs_epu8(ehi, wrdgape), _mm256_subs_epu8(hhi, wepenhi)); + // Lane 0 to A's column, lane 1 to B's + _mm256_store_si256(reinterpret_cast<__m256i*>(pa + 2 * F), _mm256_permute2x128_si256(flo, fhi, 0x20)); + _mm256_store_si256(reinterpret_cast<__m256i*>(pb + 2 * F), _mm256_permute2x128_si256(flo, fhi, 0x31)); + _mm256_store_si256(reinterpret_cast<__m256i*>(pa + 2 * H), _mm256_permute2x128_si256(hlo, hhi, 0x20)); + _mm256_store_si256(reinterpret_cast<__m256i*>(pb + 2 * H), _mm256_permute2x128_si256(hlo, hhi, 0x31)); + pa += 2 * colstride; + pb += 2 * colstride; + _mm256_store_si256(reinterpret_cast<__m256i*>(pa + 2 * E), _mm256_permute2x128_si256(elo, ehi, 0x20)); + _mm256_store_si256(reinterpret_cast<__m256i*>(pb + 2 * E), _mm256_permute2x128_si256(elo, ehi, 0x31)); + hmaxlo = _mm256_max_epu8(hmaxlo, hlo); + hmaxhi = _mm256_max_epu8(hmaxhi, hhi); + h0lo = nlo; + h0hi = nhi; + ylo = ynlo; + yhi = ynhi; + } + uint8_t hmax[32]; + _mm_storeu_si128(reinterpret_cast<__m128i*>(hmax), _mm_max_epu8( + _mm256_castsi256_si128(hmaxlo), _mm256_extracti128_si256(hmaxlo, 1))); + _mm_storeu_si128(reinterpret_cast<__m128i*>(hmax + 16), _mm_max_epu8( + _mm256_castsi256_si128(hmaxhi), _mm256_extracti128_si256(hmaxhi, 1))); + return hmax[lastWordIdx]; +} +#endif + /** * Solve the current alignment problem using SSE instructions that operate on 16 * unsigned 8-bit values packed into a single 128-bit register. @@ -820,7 +1724,6 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseU8(int& flag, bool debug) { SSERegI vhdtmp = sse_setzero_siall(); #endif SSERegI vtmp = sse_setzero_siall(); - SSERegI vzero = sse_setzero_siall(); SSERegI vhilsw = sse_setzero_siall(); SSERegI vs0 = sse_setzero_siall(); @@ -886,11 +1789,51 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseU8(int& flag, bool debug) { SSERegI *pvELoad = d.mat_.evec(0, 0); SSERegI *pvEStore = d.mat_.evecUnsafe(0, 1); SSERegI *pvFStore = d.mat_.fvec(0, 0); - SSERegI *pvFTmp = NULL; - + assert_gt(sc_->gapbar, 0); size_t nfixup = 0; - + + // F (reference gap) values flow down a column, and in the striped layout + // the first pass over a column only carries them down within each lane. + // Rather than Farrar's lazy-F loop, which carries them one lane further + // per pass until nothing changes, the carries are propagated across all + // lanes at once with a prefix scan (max-plus, saturating). A carry passes + // through a lane at the cost of iter gap extensions, unless a row in the + // lane is inside the gap barrier, which sets F to 0 and so blocks it. + // vpass1 holds that cost for the lane below each lane; vpass2, 4, ... are + // its sums over 2, 4, ... consecutive lanes, for the scan's steps. The + // barrier vectors are the same for every reference character. + SSERegI vpass1, vpass2, vpass4, vpass8; +#if NBYTES_PER_REG == 32 + SSERegI vpass16; +#endif + { + const SSERegI *pvBar = d.profbuf_.ptr() + 1; + SSERegI vbar = sse_setzero_siall(); + for(size_t j = 0; j < iter; j++) { + vbar = sse_or_siall(vbar, pvBar[2 * j]); + } + size_t passPen = iter * (size_t)sc_->refGapExtend(); + if(passPen > 0xff) { + passPen = 0xff; + } + SSERegI vpass; + sse_fill_u8((int)passPen, vpass); + vpass = sse_or_siall(vpass, vbar); + vpass1 = sse_slli_siall(vpass, NBYTES_PER_WORD); + vpass2 = sse_adds_epu8(vpass1, sse_slli_siall(vpass1, 1 * NBYTES_PER_WORD)); + vpass4 = sse_adds_epu8(vpass2, sse_slli_siall(vpass2, 2 * NBYTES_PER_WORD)); + vpass8 = sse_adds_epu8(vpass4, sse_slli_siall(vpass4, 4 * NBYTES_PER_WORD)); +#if NBYTES_PER_REG == 32 + vpass16 = sse_adds_epu8(vpass8, sse_slli_siall(vpass8, 8 * NBYTES_PER_WORD)); +#endif + } + + // The final row of H vectors of the previous column, kept in a register + // from one column to the next; before the first column, the staged + // initial column (vlo). + SSERegI vhlast = vlo; + // Fill in the table as usual but instead of using the same gap-penalty // vector for each iteration of the inner loop, load words out of a // pre-calculated gap vector parallel to the query profile. The pre- @@ -906,6 +1849,77 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseU8(int& flag, bool debug) { colstop_ = rff_ - 1; lastsolcol_ = 0; +#if NBYTES_PER_REG == 32 + if(iter <= 4) { + // Up to 128 rows. The loop below spends most of each column on the + // chain of dependent instructions from one column to the next, much + // of it byte shifts across the two 128-bit halves (vperm2i128 then + // vpalignr); these fills compute the same cells with a shorter chain. + if(iter == 1) { + assert_eq(0, d.lastIter_); + // A fill of many columns (e.g. a mate-rescue window) runs as two + // chains at once, the second started this many columns early + const size_t warmup = alignEE8V256Row32Warmup(*sc_); + if((size_t)(rff_ - rfi_) >= warmup + 32) { + lrmax = alignEE8V256Row32Split( + reinterpret_cast(d.profbuf_.ptr()), + reinterpret_cast<__m128i*>(d.mat_.evec(0, 0)), colstride, rf_, + (size_t)rfi_, (size_t)rff_, *sc_, lastWordIdx, warmup); + } else { + lrmax = alignEE8V256Row32( + reinterpret_cast(d.profbuf_.ptr()), + reinterpret_cast<__m128i*>(d.mat_.evec(0, 0)), colstride, rf_, + (size_t)rfi_, (size_t)rff_, *sc_, lastWordIdx); + } + } else { + // A fill of many columns (e.g. a mate-rescue window) runs as two + // chains at once, the second started W columns early. Below about + // 3 W columns (3.4 W at ITER = 4) the warm-up costs more than the + // second chain gains. + const size_t ncol = (size_t)(rff_ - rfi_); + if(iter == 2) { + const size_t W = alignEE8V256ShortWarmup<2>(*sc_); + lrmax = ncol >= 4 * W ? + alignEE8V256ShortSplit<2>(d.profbuf_.ptr(), d.mat_.ptr(), colstride, + rf_, (size_t)rfi_, (size_t)rff_, *sc_, lastWordIdx, W) : + alignEE8V256Short<2>(d.profbuf_.ptr(), d.mat_.ptr(), colstride, + rf_, (size_t)rfi_, (size_t)rff_, *sc_, lastWordIdx); + } else if(iter == 3) { + const size_t W = alignEE8V256ShortWarmup<3>(*sc_); + lrmax = ncol >= 4 * W ? + alignEE8V256ShortSplit<3>(d.profbuf_.ptr(), d.mat_.ptr(), colstride, + rf_, (size_t)rfi_, (size_t)rff_, *sc_, lastWordIdx, W) : + alignEE8V256Short<3>(d.profbuf_.ptr(), d.mat_.ptr(), colstride, + rf_, (size_t)rfi_, (size_t)rff_, *sc_, lastWordIdx); + } else { + const size_t W = alignEE8V256ShortWarmup<4>(*sc_); + lrmax = ncol >= 4 * W ? + alignEE8V256ShortSplit<4>(d.profbuf_.ptr(), d.mat_.ptr(), colstride, + rf_, (size_t)rfi_, (size_t)rff_, *sc_, lastWordIdx, W) : + alignEE8V256Short<4>(d.profbuf_.ptr(), d.mat_.ptr(), colstride, + rf_, (size_t)rfi_, (size_t)rff_, *sc_, lastWordIdx); + } + } + nfixup += (rff_ - rfi_) * iter; + found = true; +#ifndef NDEBUG + for(size_t i = (size_t)rfi_; i < (size_t)rff_; i++) { + if((rand() & 15) == 0) { + for(size_t k = 0; k < dpRows(); k++) { + assert(cellOkEnd2EndU8( + d, + k, // row + i - rfi_, // col + (int)rf_[i], // reference mask + (int)(*rd_)[rdi_+k], // read char + (int)(*qu_)[rdi_+k], // read quality + *sc_)); // scoring scheme + } + } + } +#endif + } else +#endif for(size_t i = (size_t)rfi_; i < (size_t)rff_; i++) { assert(pvFStore == d.mat_.fvec(0, i - rfi_)); assert(pvHStore == d.mat_.hvec(0, i - rfi_)); @@ -919,10 +1933,9 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseU8(int& flag, bool debug) { // Set all cells to low value vf = sse_xor_siall(vf, vf); - // Load H vector from the final row of the previous column - vh = sse_load_siall(pvHLoad + colstride - ROWSTRIDE); + // H vector from the final row of the previous column // Shift 2 bytes down so that topmost (least sig) cell gets 0 - vh = sse_slli_siall(vh, NBYTES_PER_WORD); + vh = sse_slli_siall(vhlast, NBYTES_PER_WORD); // Fill topmost (least sig) cell with high value vh = sse_or_siall(vh, vhilsw); @@ -991,88 +2004,64 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseU8(int& flag, bool debug) { assert_all_lt(vf, vhi); vf = sse_max_epu8(vf, vtmp); } - pvScore -= (iter*2) - 1; // reset veto vector - vs1 = sse_load_siall(pvScore); - - // pvHStore, pvELoad, pvEStore have all rolled over to the next column - pvFTmp = pvFStore; - pvFStore -= colstride; // reset to start of column - vtmp = sse_load_siall(pvFStore); - - // vf from last row gets shifted down by one to overlay the first row - // rfgape has already been subtracted from it. + // vf holds, in each lane, the F value the first pass carries out of + // the lane's last row: max(F - rfgape, H - rfgapo). Lane L's first + // row receives the carry of lane L-1, or of any lane above it less the + // cost of passing through the lanes in between. vf becomes that + // carry for every lane: a prefix scan in log2(lanes) steps. vf = sse_slli_siall(vf, NBYTES_PER_WORD); - - vf = sse_subs_epu8(vf, vs1); // veto some ref gap extensions - vf = sse_max_epu8(vtmp, vf); - vtmp = sse_subs_epu8(vf, vtmp); - vtmp = sse_cmpeq_epi8(vtmp, vzero); - int cmp = sse_movemask_epi8(vtmp); - - // Load after computing cmp, so the result is ready by the time it is tested in while - pvHStore -= colstride; // reset to start of column - vh = sse_load_siall(pvHStore); - pvHLoad = pvHStore; // new pvHLoad = pvHStore - -#if 0 -#else - pvEStore -= colstride; // reset to start of column - ve = sse_load_siall(pvEStore); -#endif - - // If any element of vtmp is greater than H - gap-open... - j = 0; - while(cmp != SSE_MASK_ALL) { - // Store this vf - sse_store_siall(pvFStore, vf); - pvFStore += ROWSTRIDE; - - // Update vh w/r/t new vf - vh = sse_max_epu8(vh, vf); - - // Save vH values - sse_store_siall(pvHStore, vh); - pvHStore += ROWSTRIDE; - - // Update E in case it can be improved using our new vh -#if 0 -#else - vh = sse_subs_epu8(vh, rdgapo); - vh = sse_subs_epu8(vh, vs1); // veto some read gap opens - ve = sse_max_epu8(ve, vh); - sse_store_siall(pvEStore, ve); - pvEStore += ROWSTRIDE; + vtmp = sse_subs_epu8(sse_slli_siall(vf, 1 * NBYTES_PER_WORD), vpass1); + vf = sse_max_epu8(vf, vtmp); + vtmp = sse_subs_epu8(sse_slli_siall(vf, 2 * NBYTES_PER_WORD), vpass2); + vf = sse_max_epu8(vf, vtmp); + vtmp = sse_subs_epu8(sse_slli_siall(vf, 4 * NBYTES_PER_WORD), vpass4); + vf = sse_max_epu8(vf, vtmp); + vtmp = sse_subs_epu8(sse_slli_siall(vf, 8 * NBYTES_PER_WORD), vpass8); + vf = sse_max_epu8(vf, vtmp); +#if NBYTES_PER_REG == 32 + vtmp = sse_subs_epu8(sse_slli_siall(vf, 16 * NBYTES_PER_WORD), vpass16); + vf = sse_max_epu8(vf, vtmp); #endif - pvScore += 2; - - assert_lt(j, iter); - if(++j == iter) { - pvScore -= iter*2; - j = 0; - pvFStore -= colstride; - pvHStore -= colstride; - pvEStore -= colstride; - vf = sse_slli_siall(vf, NBYTES_PER_WORD); - } - vs1 = sse_load_siall(pvScore); - vtmp = sse_load_siall(pvFStore); // load next vf ASAP - // Update F with another gap extension - vf = sse_subs_epu8(vf, rfgape); - vf = sse_subs_epu8(vf, vs1); // veto some ref gap extensions - vf = sse_max_epu8(vtmp, vf); - vtmp = sse_subs_epu8(vf, vtmp); - vtmp = sse_cmpeq_epi8(vtmp, vzero); - cmp = sse_movemask_epi8(vtmp); - - // Load after computing cmp, so the result is afailable by the time it is tested - vh = sse_load_siall(pvHStore); // load next vh -#if 0 -#else - ve = sse_load_siall(pvEStore); // load next ve -#endif - - nfixup++; + // One more pass down the column extends each lane's carry through its + // rows and folds it into F, then H, then the next column's E: the + // values the lazy-F loop converges to, in exactly iter steps. The + // carry reaching row j has lost j gap extensions, or everything if a + // row up to j is inside the gap barrier; vpen accumulates that + // penalty (saturating) apart from the carry, so that each row's value + // is one subtraction from the carry rather than a chain of them. + { + const SSERegI *pvBar = pvScore - (iter*2) + 1; // gap barrier, row 0 + SSERegI *pvF = pvFStore - colstride; // this column + SSERegI *pvE = pvELoad; // next column + // The first column's pvHLoad started in the staging vectors, so + // take this column's H from pvHStore instead. + pvHLoad = pvHStore - colstride; + SSERegI *pvH = pvHLoad; // this column + SSERegI vpen = sse_setzero_siall(); + for(j = 0; j < iter; j++) { + vs1 = sse_load_siall(pvBar); + pvBar += 2; + vpen = sse_adds_epu8(vpen, vs1); // veto some ref gap extensions + vtmp = sse_load_siall(pvF); + vtmp = sse_max_epu8(vtmp, sse_subs_epu8(vf, vpen)); + sse_store_siall(pvF, vtmp); + pvF += ROWSTRIDE; + vh = sse_load_siall(pvH); + vh = sse_max_epu8(vh, vtmp); + sse_store_siall(pvH, vh); + pvH += ROWSTRIDE; + vhlast = vh; + // Update E in case it can be improved using our new vh + ve = sse_load_siall(pvE); + vh = sse_subs_epu8(vh, rdgapo); + vh = sse_subs_epu8(vh, vs1); // veto some read gap opens + ve = sse_max_epu8(ve, vh); + sse_store_siall(pvE, ve); + pvE += ROWSTRIDE; + vpen = sse_adds_epu8(vpen, rfgape); + } + nfixup += iter; } #ifndef NDEBUG @@ -1099,12 +2088,9 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseU8(int& flag, bool debug) { lrmax = lr; } - // pvELoad and pvHLoad are already where they need to be - - // Adjust the load and store vectors here. - pvHStore = pvHLoad + colstride; - pvEStore = pvELoad + colstride; - pvFStore = pvFTmp; + // pvHLoad (this column's H), pvELoad (next column's E), pvHStore and + // pvFStore (next column) and pvEStore (the column after) are already + // where the next column needs them. } // Update metrics @@ -1207,61 +2193,19 @@ bool SwAligner::gatherCellsNucleotidesEnd2EndSseU8(TAlScore best) { return !btncand_.empty(); } -#define MOVE_VEC_PTR_UP(vec, rowvec, rowelt) { \ - if(rowvec == 0) { \ - rowvec += d.mat_.nvecrow_; \ - vec += d.mat_.colstride_; \ - rowelt--; \ - } \ - rowvec--; \ - vec -= ROWSTRIDE; \ -} - -#define MOVE_VEC_PTR_LEFT(vec, rowvec, rowelt) { vec -= d.mat_.colstride_; } - -#define MOVE_VEC_PTR_UPLEFT(vec, rowvec, rowelt) { \ - MOVE_VEC_PTR_UP(vec, rowvec, rowelt); \ - MOVE_VEC_PTR_LEFT(vec, rowvec, rowelt); \ -} - -#define MOVE_ALL_LEFT() { \ - MOVE_VEC_PTR_LEFT(cur_vec, rowvec, rowelt); \ - MOVE_VEC_PTR_LEFT(left_vec, left_rowvec, left_rowelt); \ - MOVE_VEC_PTR_LEFT(up_vec, up_rowvec, up_rowelt); \ - MOVE_VEC_PTR_LEFT(upleft_vec, upleft_rowvec, upleft_rowelt); \ -} - -#define MOVE_ALL_UP() { \ - MOVE_VEC_PTR_UP(cur_vec, rowvec, rowelt); \ - MOVE_VEC_PTR_UP(left_vec, left_rowvec, left_rowelt); \ - MOVE_VEC_PTR_UP(up_vec, up_rowvec, up_rowelt); \ - MOVE_VEC_PTR_UP(upleft_vec, upleft_rowvec, upleft_rowelt); \ -} - -#define MOVE_ALL_UPLEFT() { \ - MOVE_VEC_PTR_UPLEFT(cur_vec, rowvec, rowelt); \ - MOVE_VEC_PTR_UPLEFT(left_vec, left_rowvec, left_rowelt); \ - MOVE_VEC_PTR_UPLEFT(up_vec, up_rowvec, up_rowelt); \ - MOVE_VEC_PTR_UPLEFT(upleft_vec, upleft_rowvec, upleft_rowelt); \ -} - -#define NEW_ROW_COL(row, col) { \ - rowelt = row / d.mat_.nvecrow_; \ - rowvec = row % d.mat_.nvecrow_; \ - eltvec = (col * d.mat_.colstride_) + (rowvec * ROWSTRIDE); \ - cur_vec = d.mat_.matbuf_.ptr() + eltvec; \ - left_vec = cur_vec; \ - left_rowelt = rowelt; \ - left_rowvec = rowvec; \ - MOVE_VEC_PTR_LEFT(left_vec, left_rowvec, left_rowelt); \ - up_vec = cur_vec; \ - up_rowelt = rowelt; \ - up_rowvec = rowvec; \ - MOVE_VEC_PTR_UP(up_vec, up_rowvec, up_rowelt); \ - upleft_vec = up_vec; \ - upleft_rowelt = up_rowelt; \ - upleft_rowvec = up_rowvec; \ - MOVE_VEC_PTR_LEFT(upleft_vec, upleft_rowvec, upleft_rowelt); \ +/* + * The backtrace addresses the striped matrix by one byte pointer per cell: + * btcur points at the current cell's E byte, element row / nvecrow_ of the + * E vector in vector row btrowvec = row % nvecrow_ of its column. The F and + * H bytes are one and two vectors further on (btF, btH), the cell to the + * left is one column (btLeft bytes) back, and the cell above is one vector + * row back (btUp bytes), or, from vector row 0, the previous element of the + * last vector row (btUpWrap bytes on). These are the moves the parent's + * MOVE_VEC_PTR_UP and MOVE_VEC_PTR_LEFT made on each of its four pointers. + */ +#define BT_SET_ROW_COL(row, col) { \ + btrowvec = (row) % btnvecrow; \ + btcur = btmatbuf + ((col) * btcolstride + btrowvec * ROWSTRIDE) * NBYTES_PER_REG + (row) / btnvecrow; \ } /** @@ -1294,6 +2238,37 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( SSEData& d = fw_ ? sseU8fw_ : sseU8rc_; SSEMetrics& met = extend_ ? sseU8ExtendMet_ : sseU8MateMet_; met.bt++; + // Loop invariants, kept in locals (the build has -fno-strict-aliasing, + // so the compiler would otherwise reload them after every store): the + // vector matrix's geometry in bytes (BT_SET_ROW_COL), the scoring + // parameters, the read, quality and reference, and the mask, reset and + // dirty-interval arrays (resetRow changes a row's contents, not these) + const size_t btnvecrow = d.mat_.nvecrow_; + const size_t btcolstride = d.mat_.colstride_; + uint8_t * const btmatbuf = (uint8_t*)d.mat_.matbuf_.ptr(); + const size_t btLeft = btcolstride * NBYTES_PER_REG; + const size_t btUp = ROWSTRIDE * NBYTES_PER_REG; + const size_t btUpWrap = (btcolstride - ROWSTRIDE) * NBYTES_PER_REG - 1; + const size_t btF = SSEMatrix::F * NBYTES_PER_REG; + const size_t btH = SSEMatrix::H * NBYTES_PER_REG; + assert_eq(0, SSEMatrix::E); + EList * const btmasks = d.mat_.masks_.ptr(); + const bool * const btreset = d.mat_.reset_.ptr(); + size_t * const btdirtylo = d.mat_.dirtylo_.ptr(); + size_t * const btdirtyhi = d.mat_.dirtyhi_.ptr(); + const size_t btnrow = d.mat_.nrow(); + const size_t gapbar = (size_t)sc_->gapbar; + const int rdgapo = sc_->readGapOpen(); + const int rdgape = sc_->readGapExtend(); + const int rfgapo = sc_->refGapOpen(); + const int rfgape = sc_->refGapExtend(); + const int64_t bonus = sc_->match(30); + const Scoring * const sc = sc_; + const char * const rdbuf = rd_->buf(); + const char * const qubuf = qu_->buf(); + const char * const rfbuf = rf_ + rfi_; + const size_t rdoff = rdi_; + size_t ncell = 0; // cells visited, added to met.btcell and nbts on return assert(!d.profbuf_.empty()); assert_lt(row, rd_->length()); btnstack_.clear(); // empty the backtrack stack @@ -1308,30 +2283,40 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( EList& ned = res.alres.ned(); assert(ned.empty()); assert_gt(dpRows(), row); - size_t trimEnd = dpRows() - row - 1; + size_t trimEnd = dpRows() - row - 1; size_t trimBeg = 0; size_t ct = SSEMatrix::H; // cell type - // Row and col in terms of where they fall in the SSE vector matrix - size_t rowelt, rowvec, eltvec; - size_t left_rowelt, up_rowelt, upleft_rowelt; - size_t left_rowvec, up_rowvec, upleft_rowvec; - SSERegI *cur_vec, *left_vec, *up_vec, *upleft_vec; - NEW_ROW_COL(row, col); + // The current cell in the SSE vector matrix (BT_SET_ROW_COL) + size_t btrowvec; + uint8_t *btcur; + BT_SET_ROW_COL(row, col); while((int)row >= 0) { - met.btcell++; - nbts++; - int readc = (*rd_)[rdi_ + row]; - int refm = (int)rf_[rfi_ + col]; - int readq = (*qu_)[row]; + ncell++; + int readc = rdbuf[rdoff + row]; + int refm = (int)rfbuf[col]; + int readq = qubuf[row]; + assert_eq(readc, (*rd_)[rdi_ + row]); + assert_eq(refm, (int)rf_[rfi_ + col]); + assert_eq(readq, (*qu_)[row]); assert_leq(col, origCol); + // The cell above; used only when row > 0 + const bool btwrap = (btrowvec == 0); + uint8_t * const btup = btwrap ? btcur + btUpWrap : btcur - btUp; + assert_eq(btcur[0], ((TCScore*)(d.mat_.matbuf_.ptr() + col * btcolstride + btrowvec * ROWSTRIDE))[row / btnvecrow]); + assert(row == 0 || btup[btH] == d.mat_.helt(row - 1, col)); // Get score in this cell bool empty = false; bool reportedThru, canMoveThru, branch = false; int cur = SSEMatrix::H; - if(!d.mat_.reset_[row]) { + if(!btreset[row]) { d.mat_.resetRow(row); } - reportedThru = d.mat_.reportedThrough(row, col); + // This cell's mask word (see SSEMatrix): bit 0 reported through, + // bit 1 H mask set and bits 2-6 the H mask, bit 7 E mask set and + // bits 8-9 the E mask, bit 10 F mask set and bits 11-12 the F mask + uint16_t& mk = btmasks[row][col]; + assert_eq(&mk, &d.mat_.masks_[row][col]); + reportedThru = (mk & (1 << 0)) != 0; canMoveThru = true; if(reportedThru) { canMoveThru = false; @@ -1339,10 +2324,10 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( empty = false; if(row > 0) { assert_gt(row, 0); - size_t rowFromEnd = d.mat_.nrow() - row - 1; + size_t rowFromEnd = btnrow - row - 1; bool gapsAllowed = true; - if(row < (size_t)sc_->gapbar || - rowFromEnd < (size_t)sc_->gapbar) + if(row < gapbar || + rowFromEnd < gapbar) { gapsAllowed = false; } @@ -1351,7 +2336,8 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( // Move to beginning of column/row if(ct == SSEMatrix::E) { // AKA rdgap assert_gt(col, 0); - TAlScore sc_cur = ((TCScore*)(cur_vec + SSEMatrix::E))[rowelt] + offsetsc; + const uint8_t *btleft = btcur - btLeft; + TAlScore sc_cur = ((TCScore)btcur[0]) + offsetsc; assert(gapsAllowed); // Currently in the E matrix; incoming transition must come from the // left. It's either a gap open from the H matrix or a gap extend from @@ -1359,45 +2345,33 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( // TODO: save and restore origMask as well as mask int origMask = 0, mask = 0; // Get H score of cell to the left - TAlScore sc_h_left = ((TCScore*)(left_vec + SSEMatrix::H))[left_rowelt] + offsetsc; - if(sc_h_left > floorsc && sc_h_left - sc_->readGapOpen() == sc_cur) { + TAlScore sc_h_left = ((TCScore)btleft[btH]) + offsetsc; + if(sc_h_left > floorsc && sc_h_left - rdgapo == sc_cur) { mask |= (1 << 0); } // Get E score of cell to the left - TAlScore sc_e_left = ((TCScore*)(left_vec + SSEMatrix::E))[left_rowelt] + offsetsc; - if(sc_e_left > floorsc && sc_e_left - sc_->readGapExtend() == sc_cur) { + TAlScore sc_e_left = ((TCScore)btleft[0]) + offsetsc; + if(sc_e_left > floorsc && sc_e_left - rdgape == sc_cur) { mask |= (1 << 1); } origMask = mask; assert(origMask > 0 || sc_cur <= sc_->match()); - if(d.mat_.isEMaskSet(row, col)) { - mask = (d.mat_.masks_[row][col] >> 8) & 3; + if((mk & (1 << 7)) != 0) { + mask = (mk >> 8) & 3; } if(mask == 3) { -#if 1 // Pick H -> E cell cur = SW_BT_OALL_READ_OPEN; - d.mat_.eMaskSet(row, col, 2); // might choose E later -#else - if(rnd.nextU2()) { - // Pick H -> E cell - cur = SW_BT_OALL_READ_OPEN; - d.mat_.eMaskSet(row, col, 2); // might choose E later - } else { - // Pick E -> E cell - cur = SW_BT_RDGAP_EXTEND; - d.mat_.eMaskSet(row, col, 1); // might choose H later - } -#endif + { mk &= ~(7 << 7); mk |= (1 << 7 | (2) << 8); } // might choose E later branch = true; } else if(mask == 2) { // I chose the E cell cur = SW_BT_RDGAP_EXTEND; - d.mat_.eMaskSet(row, col, 0); // done + { mk &= ~(7 << 7); mk |= (1 << 7 | (0) << 8); } // done } else if(mask == 1) { // I chose the H cell cur = SW_BT_OALL_READ_OPEN; - d.mat_.eMaskSet(row, col, 0); // done + { mk &= ~(7 << 7); mk |= (1 << 7 | (0) << 8); } // done } else { empty = true; // It's empty, so the only question left is whether we should be @@ -1410,52 +2384,40 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( } else if(ct == SSEMatrix::F) { // AKA rfgap assert_gt(row, 0); assert(gapsAllowed); - TAlScore sc_h_up = ((TCScore*)(up_vec + SSEMatrix::H))[up_rowelt] + offsetsc; - TAlScore sc_f_up = ((TCScore*)(up_vec + SSEMatrix::F))[up_rowelt] + offsetsc; - TAlScore sc_cur = ((TCScore*)(cur_vec + SSEMatrix::F))[rowelt] + offsetsc; + TAlScore sc_h_up = ((TCScore)btup[btH]) + offsetsc; + TAlScore sc_f_up = ((TCScore)btup[btF]) + offsetsc; + TAlScore sc_cur = ((TCScore)btcur[btF]) + offsetsc; // Currently in the F matrix; incoming transition must come from above. // It's either a gap open from the H matrix or a gap extend from the F // matrix. // TODO: save and restore origMask as well as mask int origMask = 0, mask = 0; // Get H score of cell above - if(sc_h_up > floorsc && sc_h_up - sc_->refGapOpen() == sc_cur) { + if(sc_h_up > floorsc && sc_h_up - rfgapo == sc_cur) { mask |= (1 << 0); } // Get F score of cell above - if(sc_f_up > floorsc && sc_f_up - sc_->refGapExtend() == sc_cur) { + if(sc_f_up > floorsc && sc_f_up - rfgape == sc_cur) { mask |= (1 << 1); } origMask = mask; assert(origMask > 0 || sc_cur <= sc_->match()); - if(d.mat_.isFMaskSet(row, col)) { - mask = (d.mat_.masks_[row][col] >> 11) & 3; + if((mk & (1 << 10)) != 0) { + mask = (mk >> 11) & 3; } if(mask == 3) { -#if 1 // I chose the H cell cur = SW_BT_OALL_REF_OPEN; - d.mat_.fMaskSet(row, col, 2); // might choose E later -#else - if(rnd.nextU2()) { - // I chose the H cell - cur = SW_BT_OALL_REF_OPEN; - d.mat_.fMaskSet(row, col, 2); // might choose E later - } else { - // I chose the F cell - cur = SW_BT_RFGAP_EXTEND; - d.mat_.fMaskSet(row, col, 1); // might choose E later - } -#endif + { mk &= ~(7 << 10); mk |= (1 << 10 | (2) << 11); } // might choose E later branch = true; } else if(mask == 2) { // I chose the F cell cur = SW_BT_RFGAP_EXTEND; - d.mat_.fMaskSet(row, col, 0); // done + { mk &= ~(7 << 10); mk |= (1 << 10 | (0) << 11); } // done } else if(mask == 1) { // I chose the H cell cur = SW_BT_OALL_REF_OPEN; - d.mat_.fMaskSet(row, col, 0); // done + { mk &= ~(7 << 10); mk |= (1 << 10 | (0) << 11); } // done } else { empty = true; // It's empty, so the only question left is whether we should be @@ -1467,26 +2429,26 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( assert(!empty || !canMoveThru); } else { assert_eq(SSEMatrix::H, ct); - TAlScore sc_cur = ((TCScore*)(cur_vec + SSEMatrix::H))[rowelt] + offsetsc; - TAlScore sc_f_up = ((TCScore*)(up_vec + SSEMatrix::F))[up_rowelt] + offsetsc; - TAlScore sc_h_up = ((TCScore*)(up_vec + SSEMatrix::H))[up_rowelt] + offsetsc; - TAlScore sc_h_left = col > 0 ? (((TCScore*)(left_vec + SSEMatrix::H))[left_rowelt] + offsetsc) : floorsc; - TAlScore sc_e_left = col > 0 ? (((TCScore*)(left_vec + SSEMatrix::E))[left_rowelt] + offsetsc) : floorsc; - TAlScore sc_h_upleft = col > 0 ? (((TCScore*)(upleft_vec + SSEMatrix::H))[upleft_rowelt] + offsetsc) : floorsc; - TAlScore sc_diag = sc_->score(readc, refm, readq - 33); + TAlScore sc_cur = ((TCScore)btcur[btH]) + offsetsc; + TAlScore sc_f_up = ((TCScore)btup[btF]) + offsetsc; + TAlScore sc_h_up = ((TCScore)btup[btH]) + offsetsc; + TAlScore sc_h_left = col > 0 ? (((TCScore)(btcur - btLeft)[btH]) + offsetsc) : floorsc; + TAlScore sc_e_left = col > 0 ? (((TCScore)(btcur - btLeft)[0]) + offsetsc) : floorsc; + TAlScore sc_h_upleft = col > 0 ? (((TCScore)(btup - btLeft)[btH]) + offsetsc) : floorsc; + TAlScore sc_diag = sc->score(readc, refm, readq - 33); // TODO: save and restore origMask as well as mask int origMask = 0, mask = 0; if(gapsAllowed) { - if(sc_h_up > floorsc && sc_cur == sc_h_up - sc_->refGapOpen()) { + if(sc_h_up > floorsc && sc_cur == sc_h_up - rfgapo) { mask |= (1 << 0); } - if(sc_h_left > floorsc && sc_cur == sc_h_left - sc_->readGapOpen()) { + if(sc_h_left > floorsc && sc_cur == sc_h_left - rdgapo) { mask |= (1 << 1); } - if(sc_f_up > floorsc && sc_cur == sc_f_up - sc_->refGapExtend()) { + if(sc_f_up > floorsc && sc_cur == sc_f_up - rfgape) { mask |= (1 << 2); } - if(sc_e_left > floorsc && sc_cur == sc_e_left - sc_->readGapExtend()) { + if(sc_e_left > floorsc && sc_cur == sc_e_left - rdgape) { mask |= (1 << 3); } } @@ -1495,50 +2457,26 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( } origMask = mask; assert(origMask > 0 || sc_cur <= sc_->match()); - if(d.mat_.isHMaskSet(row, col)) { - mask = (d.mat_.masks_[row][col] >> 2) & 31; + if((mk & (1 << 1)) != 0) { + mask = (mk >> 2) & 31; } assert(gapsAllowed || mask == (1 << 4) || mask == 0); - int opts = alts5[mask]; - int select = -1; - if(opts == 1) { - select = firsts5[mask]; - assert_geq(mask, 0); - d.mat_.hMaskSet(row, col, 0); - } else if(opts > 1) { -#if 1 - if( (mask & 16) != 0) { - select = 4; // H diag - } else if((mask & 1) != 0) { - select = 0; // H up - } else if((mask & 4) != 0) { - select = 2; // F up - } else if((mask & 2) != 0) { - select = 1; // H left - } else if((mask & 8) != 0) { - select = 3; // E left - } -#else - select = randFromMask(rnd, mask); -#endif - assert_geq(mask, 0); - mask &= ~(1 << select); - assert(gapsAllowed || mask == (1 << 4) || mask == 0); - d.mat_.hMaskSet(row, col, mask); - branch = true; - } else { /* No way to backtrack! */ } - if(select != -1) { - if(select == 4) { - cur = SW_BT_OALL_DIAG; - } else if(select == 0) { - cur = SW_BT_OALL_REF_OPEN; - } else if(select == 1) { - cur = SW_BT_OALL_READ_OPEN; - } else if(select == 2) { - cur = SW_BT_RFGAP_EXTEND; - } else { - assert_eq(3, select) - cur = SW_BT_RDGAP_EXTEND; + if(mask != 0) { + // The move and the mask left behind, from the table + // (see btHSelect) + int sel = btHSelect[mask]; + cur = sel & 7; + int rest = sel >> 3; + assert_eq(rest, mask & ~btHSelectBit(mask)); + assert_eq(rest != 0, alts5[mask] > 1); + assert_eq(cur, btHSelectBit(mask) == 16 ? (int)SW_BT_OALL_DIAG : + btHSelectBit(mask) == 1 ? (int)SW_BT_OALL_REF_OPEN : + btHSelectBit(mask) == 2 ? (int)SW_BT_OALL_READ_OPEN : + btHSelectBit(mask) == 4 ? (int)SW_BT_RFGAP_EXTEND : + (int)SW_BT_RDGAP_EXTEND); + { mk &= ~(31 << 1); mk |= (1 << 1 | rest << 2); } + if(rest != 0) { + branch = true; } } else { empty = true; @@ -1553,7 +2491,11 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( } } //cerr << "reportedThrough rejected (" << row << ", " << col << ")" << endl; - d.mat_.setReportedThrough(row, col); + // Mark the cell reported through, and record it in the row's dirty + // interval (SSEMatrix::markDirty) + mk |= (1 << 0); + btdirtylo[row] = (col < btdirtylo[row]) ? col : btdirtylo[row]; + btdirtyhi[row] = (col >= btdirtyhi[row]) ? col + 1 : btdirtyhi[row]; assert_eq(gaps, Edit::numGaps(ned)); assert_leq(gaps, rdgap_ + rfgap_); // Cell was involved in a previously-reported alignment? @@ -1574,10 +2516,12 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( ct = btnstack_.back().ct; btnstack_.pop_back(); assert(!sc_->monotone || score.score() >= escore); - NEW_ROW_COL(row, col); + BT_SET_ROW_COL(row, col); continue; } else { // No branch points to revisit; just give up + met.btcell += ncell; + nbts += ncell; res.reset(); met.btfail++; // DP backtraces failed return false; @@ -1620,8 +2564,9 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( // it and add a nucleotide mismatch. case SW_BT_OALL_DIAG: { assert_gt(row, 0); assert_gt(col, 0); - int readC = (*rd_)[row]; - int refNmask = (int)rf_[rfi_+col]; + int readC = rdbuf[row]; + int refNmask = (int)rfbuf[col]; + assert_eq(readC, (*rd_)[row]); assert_gt(refNmask, 0); int m = matchesEx(readC, refNmask); ct = SSEMatrix::H; @@ -1639,7 +2584,6 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( assert(!sc_->monotone || score.score() >= escore); } else { // Reward a match - int64_t bonus = sc_->match(30); score.score_ += bonus; assert(!sc_->monotone || score.score() >= escore); } @@ -1647,7 +2591,8 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( score.ns_++; } row--; col--; - MOVE_ALL_UPLEFT(); + btcur = btup - btLeft; + btrowvec = btwrap ? btnvecrow - 1 : btrowvec - 1; assert(VALID_AL_SCORE(score)); break; } @@ -1658,7 +2603,7 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( Edit e( (int)row, '-', - "ACGTN"[(int)(*rd_)[row]], + "ACGTN"[(int)rdbuf[row]], EDIT_TYPE_REF_GAP); assert(e.repOk()); assert(ned.empty() || ned.back().pos >= row); @@ -1667,13 +2612,14 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( assert_geq((int)(rdf_-rdi_-row-1), sc_->gapbar-1); row--; ct = SSEMatrix::H; - int pen = sc_->refGapOpen(); + int pen = rfgapo; score.score_ -= pen; assert(!sc_->monotone || score.score() >= minsc_); gaps++; refGaps++; assert_eq(gaps, Edit::numGaps(ned)); assert_leq(gaps, rdgap_ + rfgap_); - MOVE_ALL_UP(); + btcur = btup; + btrowvec = btwrap ? btnvecrow - 1 : btrowvec - 1; break; } // Move up. Add an edit encoding the ref gap. @@ -1683,7 +2629,7 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( Edit e( (int)row, '-', - "ACGTN"[(int)(*rd_)[row]], + "ACGTN"[(int)rdbuf[row]], EDIT_TYPE_REF_GAP); assert(e.repOk()); assert(ned.empty() || ned.back().pos >= row); @@ -1692,13 +2638,14 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( assert_geq((int)(rdf_-rdi_-row-1), sc_->gapbar-1); row--; ct = SSEMatrix::F; - int pen = sc_->refGapExtend(); + int pen = rfgape; score.score_ -= pen; assert(!sc_->monotone || score.score() >= minsc_); gaps++; refGaps++; assert_eq(gaps, Edit::numGaps(ned)); assert_leq(gaps, rdgap_ + rfgap_); - MOVE_ALL_UP(); + btcur = btup; + btrowvec = btwrap ? btnvecrow - 1 : btrowvec - 1; break; } case SW_BT_OALL_READ_OPEN: @@ -1706,7 +2653,7 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( assert_gt(col, 0); Edit e( (int)row+1, - mask2dna[(int)rf_[rfi_+col]], + mask2dna[(int)rfbuf[col]], '-', EDIT_TYPE_READ_GAP); assert(e.repOk()); @@ -1716,13 +2663,13 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( assert_geq((int)(rdf_-rdi_-row-1), sc_->gapbar-1); col--; ct = SSEMatrix::H; - int pen = sc_->readGapOpen(); + int pen = rdgapo; score.score_ -= pen; assert(!sc_->monotone || score.score() >= minsc_); gaps++; readGaps++; assert_eq(gaps, Edit::numGaps(ned)); assert_leq(gaps, rdgap_ + rfgap_); - MOVE_ALL_LEFT(); + btcur -= btLeft; break; } case SW_BT_RDGAP_EXTEND: @@ -1730,7 +2677,7 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( assert_gt(col, 1); Edit e( (int)row+1, - mask2dna[(int)rf_[rfi_+col]], + mask2dna[(int)rfbuf[col]], '-', EDIT_TYPE_READ_GAP); assert(e.repOk()); @@ -1740,18 +2687,20 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( assert_geq((int)(rdf_-rdi_-row-1), sc_->gapbar-1); col--; ct = SSEMatrix::E; - int pen = sc_->readGapExtend(); + int pen = rdgape; score.score_ -= pen; assert(!sc_->monotone || score.score() >= minsc_); gaps++; readGaps++; assert_eq(gaps, Edit::numGaps(ned)); assert_leq(gaps, rdgap_ + rfgap_); - MOVE_ALL_LEFT(); + btcur -= btLeft; break; } default: throw 1; } } // while((int)row > 0) + met.btcell += ncell; + nbts += ncell; assert_eq(0, trimBeg); assert_eq(0, trimEnd); assert_geq(col, 0); diff --git a/aligner_swsse_loc_u8.cpp b/aligner_swsse_loc_u8.cpp index d5c8a461..886eae68 100644 --- a/aligner_swsse_loc_u8.cpp +++ b/aligner_swsse_loc_u8.cpp @@ -95,67 +95,103 @@ void SwAligner::buildQueryProfileLocalSseU8(bool fw) { d.lastIter_ = d.lastWord_ = 0; d.qprofStride_ = d.gbarStride_ = 2; d.bias_ = 0; - // Calculate bias - for(size_t refc = 0; refc < ALPHA_SIZE; refc++) { - for(size_t i = 0; i < len; i++) { - int readc = (*rd)[i]; - int readq = (*qu)[i]; - int sc = sc_->score(readc, (int)(1 << refc), readq - 33); - if(sc < 0 && sc < d.bias_) { - d.bias_ = sc; - } - } + // The scorer's qualProf table holds sc_->score(readc, 1 << refc, + // readq - 33) for the five reference characters at once, as bytes, with + // the smallest score and the largest penalty and bonus among them. + const char *rdbuf = rd->buf(); + const char *qubuf = qu->buf(); + // Calculate bias: minus the most negative score, or 0 + int minsc = 0; + for(size_t j = 0; j < len; j++) { + const int readc = rdbuf[j]; + const int readq = qubuf[j]; + assert_range(0, 4, readc); + assert_range(0, 255, readq - 33); + minsc = std::min(minsc, sc_->qualProf[readc][readq - 33].locMin); } - assert_leq(d.bias_, 0); - d.bias_ = -d.bias_; - // For each reference character A, C, G, T, N ... - for(size_t refc = 0; refc < ALPHA_SIZE; refc++) { - // For each segment ... - for(size_t i = 0; i < seglen; i++) { - size_t j = i; - uint8_t *qprofWords = - reinterpret_cast(d.profbuf_.ptr() + (refc * seglen * 2) + (i * 2)); - uint8_t *gbarWords = - reinterpret_cast(d.profbuf_.ptr() + (refc * seglen * 2) + (i * 2) + 1); - // For each sub-word (byte) ... - for(size_t k = 0; k < NWORDS_PER_REG; k++) { - int sc = 0; - *gbarWords = 0; - if(j < len) { - int readc = (*rd)[j]; - int readq = (*qu)[j]; - sc = sc_->score(readc, (int)(1 << refc), readq - 33); - assert_range(0, 255, sc + d.bias_); - size_t j_from_end = len - j - 1; - if(j < (size_t)sc_->gapbar || - j_from_end < (size_t)sc_->gapbar) - { - // Inside the gap barrier - *gbarWords = 0xff; - } - } - if(refc == 0 && j == len-1) { - // Remember which 128-bit word and which smaller word has - // the final row - d.lastIter_ = i; - d.lastWord_ = k; + d.bias_ = -minsc; + assert_geq(d.bias_, 0); + const int bias = d.bias_; + if(len > 0) { + // Remember which 128-bit word and which smaller word has the final + // row: read position j is in segment j % seglen, word j / seglen + d.lastIter_ = (len - 1) % seglen; + d.lastWord_ = (len - 1) / seglen; + } + // For each reference character A, C, G, T, N there are seglen segments, + // each a query-profile register followed by a gap-barrier register. + // Word k of segment i holds read position j = k * seglen + i. + SSERegI *pvec = d.profbuf_.ptr(); + uint8_t *prof = reinterpret_cast(pvec); + const size_t segBytes = 2 * NBYTES_PER_REG; // one segment's two registers + const size_t refBytes = seglen * segBytes; // one reference character's segments + // The gap barrier depends on the read position alone: 0xff where j < len + // and j or len - j - 1 is below gapbar, 0 elsewhere. Zero it, then mark + // the barrier positions at both ends. + SSERegI vzero = sse_setzero_siall(); + for(size_t s = 0; s < seglen * ALPHA_SIZE; s++) { + sse_store_siall(pvec + 2 * s + 1, vzero); + } + { + const size_t gapbar = (size_t)sc_->gapbar; + const size_t nbar = std::min(len, gapbar); + // j < gapbar for j in [0, nbar); len - j - 1 < gapbar for j in + // [len - nbar, len) + const size_t ranges[2][2] = { + { 0, nbar }, + { std::max(nbar, len - nbar), len } }; + for(size_t r = 0; r < 2; r++) { + if(ranges[r][0] >= ranges[r][1]) { + continue; + } + size_t i = ranges[r][0] % seglen, k = ranges[r][0] / seglen; + for(size_t j = ranges[r][0]; j < ranges[r][1]; j++) { + uint8_t *g = prof + i * segBytes + NBYTES_PER_REG + k; + for(size_t refc = 0; refc < ALPHA_SIZE; refc++) { + g[refc * refBytes] = 0xff; } - if(sc < 0) { - if((size_t)(-sc) > d.maxPen_) { - d.maxPen_ = (size_t)(-sc); - } - } else { - if((size_t)sc > d.maxBonus_) { - d.maxBonus_ = (size_t)sc; - } + if(++i == seglen) { + i = 0; + k++; } - *qprofWords = (uint8_t)(sc + d.bias_); - gbarWords++; - qprofWords++; - j += seglen; // update offset into query } } } + // Query profile: (uint8_t)(score + bias), adding the bias to each of the + // five bytes modulo 256 without carries between them; the bias alone (a + // score of 0) past the read's end. + const uint64_t hi = 0x8080808080808080ULL; + const uint64_t biasw = (uint64_t)(uint8_t)bias * 0x0101010101010101ULL; + size_t maxPen = 0, maxBonus = 0; + for(size_t i = 0; i < seglen; i++) { + uint8_t *p = prof + i * segBytes; + size_t j = i, k = 0; + for(; j < len; j += seglen, k++) { + const int readc = rdbuf[j]; + const int readq = qubuf[j]; + const Scoring::QualProf& e = sc_->qualProf[readc][readq - 33]; + const uint64_t w = (((e.loc & ~hi) + (biasw & ~hi)) ^ ((e.loc ^ biasw) & hi)); + assert_eq((uint8_t)w, (uint8_t)(sc_->score(readc, 1, readq - 33) + bias)); + assert_eq((uint8_t)(w >> 8), (uint8_t)(sc_->score(readc, 2, readq - 33) + bias)); + assert_eq((uint8_t)(w >> 16), (uint8_t)(sc_->score(readc, 4, readq - 33) + bias)); + assert_eq((uint8_t)(w >> 24), (uint8_t)(sc_->score(readc, 8, readq - 33) + bias)); + assert_eq((uint8_t)(w >> 32), (uint8_t)(sc_->score(readc, 16, readq - 33) + bias)); + maxPen = std::max(maxPen, (size_t)e.locPen); + maxBonus = std::max(maxBonus, (size_t)e.locBonus); + p[k] = (uint8_t)w; + p[k + refBytes] = (uint8_t)(w >> 8); + p[k + 2 * refBytes] = (uint8_t)(w >> 16); + p[k + 3 * refBytes] = (uint8_t)(w >> 24); + p[k + 4 * refBytes] = (uint8_t)(w >> 32); + } + for(; k < NWORDS_PER_REG; k++) { + for(size_t refc = 0; refc < ALPHA_SIZE; refc++) { + p[k + refc * refBytes] = (uint8_t)bias; + } + } + } + d.maxPen_ = maxPen; + d.maxBonus_ = maxBonus; } #ifndef NDEBUG @@ -920,6 +956,329 @@ TAlScore SwAligner::alignGatherLoc8(int& flag, bool debug) { return (TAlScore)score; } +#if NBYTES_PER_REG == 32 +/** + * The column loop of alignNucleotidesLocalSseU8 for a column of ITER = 2 to 4 + * 256-bit vectors (33 to 128 rows). It computes the general loop's saturating + * u8 recurrence (see the comments there) and stores the same E, F, H and + * column-maximum vectors in the same places, and stops at the same column. + * The general loop's speed is set by the chain of dependent instructions from + * one column to the next: H from the last vector row through the diagonal + * shift, the first pass's F through every row, and Farrar's lazy-F loop, which + * carries F one lane further per pass and whose exit is a data-dependent + * branch. Here the chain is a prefix scan with in-half shifts and a few + * instructions, and every value passed between columns stays in a register. + * This is alignEE8V256Short (aligner_swsse_ee_u8.cpp) with local mode's + * diagonal: H0[j] = max(diag + profile - bias, E[j]), saturating at both ends, + * and 0 shifted into row 0. + * + * Row j of lane L is read row L * ITER + j. For each column: + * - within a lane, F flows from row j to row j + 1 at the cost of rfgapo + * from H0 or of the barrier and rfgape from F (rfgapo >= rfgape); HL and + * FL are H and F with these carries only; + * - the carry c into each lane from the lanes above is the prefix scan of + * the lanes' carry-outs, done within each 128-bit half and then across + * the halves once; + * - each row takes c less the cost from the top of its lane (vpen): F = + * max(FL, that), H = max(HL, that), and the next column's E = max(E - + * rdgape, H - rdgapo - barrier). + * The next column's carry-out, the scan's input, is max_j(H0'[j] - cpen[j]). + * Every operation is monotone, so it is also the max of terms in HL and E, + * which are ready while the scan runs, and of terms in c and in c shifted + * down a lane, each taking its least cost (zc, wc, kb). The chain from one + * column to the next is then the scan and one subtraction. + * + * sat(sat(x - a) - b) = sat(x - sat(a + b)), so costs may be summed before + * they are subtracted. The diagonal adds the profile, a gain where the read + * matches: a term in c through it is sat(sat(sat(c - vpen) + P) - bias - + * cpen). While sat(c - vpen) + P < 255 that is sat(c - (vpen + cpen + bias - + * P)), and the caller makes vpen + cpen + bias - P >= 0 (the largest bonus is + * at most rfgapo) and computes cpen + bias - P without saturating (rfgapo + + * (ITER-1) * rfgape + bias <= 255). Where sat(c - vpen) + P >= 255, the next + * column's H0 reaches 255 - bias, so that column saturates in both loops, + * which return at it with the same flag. Its own cells may then differ; the + * caller does not use the matrix of a saturated fill. + * + * Returns -2 if a column saturated, else 0 with the maximum of H over the + * filled columns in vmax. + */ +template +static int alignLoc8V256Short( + const __m256i *prof, // query profile: per reference character, ITER (score, barrier) pairs + __m256i *mat, // matrix: column 0, row 0, E (SSEMatrix layout) + size_t colstride, // vectors per matrix column + const char *rf, // reference + size_t rfi, // first column + size_t rff, // last column (excl.) + const Scoring& sc, // gap penalties + int bias, // added to every profile score + TAlScore minsc, // minimum score of a valid alignment + TAlScore matchsc, // score of a match, for the bail-out test + size_t& colstop, // out: columns filled (as colstop_) + size_t& lastsolcol, // out: last column reaching minsc (as lastsolcol_) + __m256i& vmax) // out: maximum of H over the filled columns +{ + const size_t E = SSEMatrix::E, F = SSEMatrix::F, H = SSEMatrix::H, TMP = SSEMatrix::TMP; + const __m256i zero = _mm256_setzero_si256(); + const __m256i vrfgapo = _mm256_set1_epi8((char)sc.refGapOpen()); + const __m256i vrfgape = _mm256_set1_epi8((char)sc.refGapExtend()); + const __m256i vrdgapo = _mm256_set1_epi8((char)sc.readGapOpen()); + const __m256i vrdgape = _mm256_set1_epi8((char)sc.readGapExtend()); + const __m256i vbias = _mm256_set1_epi8((char)bias); + // Gap barrier rows (the same for every reference character), and the costs + // derived from them, per vector row j: + // bpe: F carried from row j to row j + 1 (barrier, then rfgape); + // cpen: H0 at row j to the lane's carry-out (rfgapo, then bpe below); + // vpen: the carry into the lane to row j (as the general loop's vpen); + // epen: H to the next column's E (rdgapo, then barrier); + // below: 0xff where a row below row j in the lane is in the barrier. + __m256i bar[ITER], bpe[ITER], cpen[ITER], vpen[ITER], epen[ITER], below[ITER]; + __m256i vbar = zero; + for(size_t j = 0; j < ITER; j++) { + bar[j] = _mm256_load_si256(prof + 2 * j + 1); + bpe[j] = _mm256_adds_epu8(bar[j], vrfgape); + epen[j] = _mm256_adds_epu8(vrdgapo, bar[j]); + vpen[j] = j == 0 ? bar[0] : _mm256_adds_epu8(_mm256_adds_epu8(vpen[j-1], vrfgape), bar[j]); + vbar = _mm256_or_si256(vbar, bar[j]); + } + cpen[ITER-1] = vrfgapo; + below[ITER-1] = zero; + for(size_t j = ITER-1; j > 0; j--) { + cpen[j-1] = _mm256_adds_epu8(cpen[j], bpe[j]); + below[j-1] = _mm256_or_si256(below[j], bar[j]); + } + // The scan's costs: passing lane L whole is pass(L); qN(L) is the cost of + // passing the N lanes that end at L, within a half. chi(L) is the cost of + // passing lanes 16 to L - 1, for the carry from lane 15 into lane L > 16. + size_t passPen = ITER * (size_t)sc.refGapExtend(); + if(passPen > 0xff) { + passPen = 0xff; + } + const __m256i q1 = _mm256_or_si256(_mm256_set1_epi8((char)passPen), vbar); + const __m256i q2 = _mm256_adds_epu8(q1, _mm256_slli_si256(q1, 1)); + const __m256i q3 = _mm256_adds_epu8(q2, _mm256_slli_si256(q1, 2)); + const __m256i q4 = _mm256_adds_epu8(q2, _mm256_slli_si256(q2, 2)); + const __m256i q8 = _mm256_adds_epu8(q4, _mm256_slli_si256(q4, 4)); + const __m256i q12 = _mm256_adds_epu8(q8, _mm256_slli_si256(q4, 8)); + const __m256i q16 = _mm256_adds_epu8(q8, _mm256_slli_si256(q8, 8)); + const __m256i chi = _mm256_slli_si256(q16, 1); + // Selects byte 15 of the upper half (lane 15, once moved there) into + // every byte of the upper half, and zero into the lower half + const __m256i vidx15 = _mm256_set_epi64x( + 0x0f0f0f0f0f0f0f0fLL, 0x0f0f0f0f0f0f0f0fLL, + (long long)0x8080808080808080ULL, (long long)0x8080808080808080ULL); + // Costs to the next column's carry-out, max_j(H0'[j] - cpen[j]), where + // H0'[j] = max(diag[j] + profile'[j] - bias, E'[j]), E'[j] = max(E[j] - + // rdgape, H[j] - epen[j]), H[j] = max(HL[j], c - vpen[j]) and diag[j] = + // H[j - 1] or, for j = 0, H[ITER - 1] shifted down a lane (0 into lane 0): + // rc[j]: from E[j] (rdgape, cpen); ec[j]: from H[j] (epen, cpen); + // bc[j]: from diag[j] after the profile is added (bias, cpen); + // zc[r]: from c, the least over its paths, for reference character r; + // wc[r]: from c shifted down a lane; kb[r]: from lane 15 of the scan, + // carried into the upper half, for both (c and c shifted are taken from + // the scan's values directly). + // Through the diagonal, a path's cost is vpen + cpen + bias - profile. + // cpen + bias - profile is computed from the scalar cpen of a lane without + // a barrier below (the caller keeps it at most 255), as 0xff where there + // is one. + __m256i rc[ITER], ec[ITER], bc[ITER], zc[5], wc[5], kb[5]; + // chi for c shifted down a lane: lane 16 takes lane 14's value, with no + // carry from lane 15 + const __m256i chish = _mm256_or_si256(_mm256_slli_si256(chi, 1), + _mm256_set_epi64x(0, 0xff, 0, 0)); + const __m256i vpenSh = _mm256_alignr_epi8(vpen[ITER-1], + _mm256_permute2x128_si256(vpen[ITER-1], vpen[ITER-1], 0x08), 15); + __m256i cbias[ITER]; + for(size_t j = 0; j < ITER; j++) { + rc[j] = _mm256_adds_epu8(vrdgape, cpen[j]); + ec[j] = _mm256_adds_epu8(epen[j], cpen[j]); + bc[j] = _mm256_adds_epu8(vbias, cpen[j]); + cbias[j] = _mm256_set1_epi8((char)(sc.refGapOpen() + + (int)(ITER - 1 - j) * sc.refGapExtend() + bias)); + } + for(size_t r = 0; r < 5; r++) { + const __m256i *pr = prof + r * ITER * 2; + __m256i z = _mm256_adds_epu8(vpen[0], ec[0]); + for(size_t j = 1; j < ITER; j++) { + const __m256i cps = _mm256_or_si256( + _mm256_subs_epu8(cbias[j], _mm256_load_si256(pr + 2 * j)), below[j]); + z = _mm256_min_epu8(z, _mm256_adds_epu8(vpen[j-1], cps)); + z = _mm256_min_epu8(z, _mm256_adds_epu8(vpen[j], ec[j])); + } + zc[r] = z; + const __m256i cps0 = _mm256_or_si256( + _mm256_subs_epu8(cbias[0], _mm256_load_si256(pr)), below[0]); + wc[r] = _mm256_adds_epu8(vpenSh, cps0); + kb[r] = _mm256_min_epu8(_mm256_adds_epu8(chi, zc[r]), _mm256_adds_epu8(chish, wc[r])); + } + // lpen[j]: H0 at row j to F of the lane's last row (its barrier included) + __m256i lpen[ITER-1]; + lpen[ITER-2] = vrfgapo; + for(size_t j = ITER-2; j > 0; j--) { + lpen[j-1] = _mm256_adds_epu8(lpen[j], bpe[j]); + } + for(size_t j = 0; j + 1 < ITER; j++) { + lpen[j] = _mm256_adds_epu8(lpen[j], bar[ITER-1]); + } + // H0 and E of the column, and its carry-out: for the first, from the + // staged initial column (zero), so every diagonal is 0 + __m256i h0[ITER], ecur[ITER], co = zero; + { + const __m256i *pvS = prof + (size_t)firsts5[(int)rf[rfi]] * ITER * 2; + for(size_t j = 0; j < ITER; j++) { + h0[j] = _mm256_subs_epu8(_mm256_load_si256(pvS + 2 * j), vbias); + ecur[j] = zero; + co = _mm256_max_epu8(co, _mm256_subs_epu8(h0[j], cpen[j])); + } + } + __m256i vm = zero; + size_t ls = 0; + colstop = rff - rfi; + __m256i *pvCol = mat; + for(size_t i = rfi; i < rff; i++) { + // The next column's reference character. (After the last column, + // the values computed with it are not used.) + const size_t rn = (size_t)firsts5[(int)rf[i + 1 < rff ? i + 1 : i]]; + const __m256i *pvS = prof + rn * ITER * 2; + // The scan within each half, four lanes per step: over the 4 lanes + // ending at each lane, then over 16 + __m256i y = co; + y = _mm256_max_epu8( + _mm256_max_epu8(y, _mm256_subs_epu8(_mm256_slli_si256(y, 1), q1)), + _mm256_max_epu8(_mm256_subs_epu8(_mm256_slli_si256(y, 2), q2), + _mm256_subs_epu8(_mm256_slli_si256(y, 3), q3))); + y = _mm256_max_epu8( + _mm256_max_epu8(y, _mm256_subs_epu8(_mm256_slli_si256(y, 4), q4)), + _mm256_max_epu8(_mm256_subs_epu8(_mm256_slli_si256(y, 8), q8), + _mm256_subs_epu8(_mm256_slli_si256(y, 12), q12))); + // The carry into each lane is the scan's value one lane up (y15), and + // into the upper half also lane 15's (yc), carried down + const __m256i py = _mm256_permute2x128_si256(y, y, 0x08); + const __m256i y15 = _mm256_alignr_epi8(y, py, 15); + const __m256i y14 = _mm256_alignr_epi8(y, py, 14); + const __m256i yc = _mm256_shuffle_epi8(py, vidx15); + const __m256i c = _mm256_max_epu8(y15, _mm256_subs_epu8(yc, chi)); + // The next column's carry-out: its terms in c and in c shifted down a + // lane (from y14 and yc) + const __m256i cc = _mm256_max_epu8( + _mm256_subs_epu8(y14, wc[rn]), _mm256_subs_epu8(yc, kb[rn])); + // c shifted down a lane (0 into lane 0), for the next column's + // diagonal + const __m256i csh = _mm256_max_epu8(y14, _mm256_subs_epu8(yc, chish)); + // F and H with the carries from within the lane only + __m256i fl[ITER], hl[ITER]; + fl[0] = zero; + hl[0] = h0[0]; + { + __m256i vf = _mm256_subs_epu8(h0[0], vrfgapo); + for(size_t j = 1; j < ITER; j++) { + // (into the last row of ITER > 2, vf has the barrier, from lpen) + fl[j] = j + 1 == ITER && ITER > 2 ? vf : _mm256_subs_epu8(vf, bar[j]); + hl[j] = _mm256_max_epu8(h0[j], fl[j]); + if(j + 2 < ITER) { + vf = _mm256_max_epu8( + _mm256_subs_epu8(vf, bpe[j]), _mm256_subs_epu8(h0[j], vrfgapo)); + } else if(j + 2 == ITER) { + // into the last row, as a maximum over the rows above + __m256i fz = _mm256_subs_epu8(h0[0], lpen[0]); + for(size_t k = 1; k < ITER-1; k++) { + fz = _mm256_max_epu8(fz, _mm256_subs_epu8(h0[k], lpen[k])); + } + vf = fz; + } + } + } + // ... and its other terms, the latest (from the last row of HL) last. + // hlsh: HL's last row shifted down a lane (0 into lane 0) + __m256i hlsh; + { + __m256i a = _mm256_max_epu8( + _mm256_subs_epu8(ecur[0], rc[0]), _mm256_subs_epu8(ecur[1], rc[1])); + for(size_t j = 2; j < ITER; j++) { + a = _mm256_max_epu8(a, _mm256_subs_epu8(ecur[j], rc[j])); + } + for(size_t j = 0; j + 1 < ITER; j++) { + a = _mm256_max_epu8(a, _mm256_max_epu8( + _mm256_subs_epu8(hl[j], ec[j]), + _mm256_subs_epu8(_mm256_adds_epu8(hl[j], _mm256_load_si256(pvS + 2 * (j+1))), bc[j+1]))); + } + hlsh = _mm256_alignr_epi8(hl[ITER-1], + _mm256_permute2x128_si256(hl[ITER-1], hl[ITER-1], 0x08), 15); + a = _mm256_max_epu8(a, _mm256_max_epu8( + _mm256_subs_epu8(hl[ITER-1], ec[ITER-1]), + _mm256_subs_epu8(_mm256_adds_epu8(hlsh, _mm256_load_si256(pvS)), bc[0]))); + co = _mm256_max_epu8(_mm256_max_epu8(a, _mm256_subs_epu8(y15, zc[rn])), cc); + } + // F, H and the next column's E with the carry from the lanes above + __m256i hcur[ITER]; + __m256i colmax = zero; + __m256i *pv = pvCol; + for(size_t j = 0; j < ITER; j++) { + const __m256i s = _mm256_subs_epu8(c, vpen[j]); + const __m256i f = j == 0 ? s : _mm256_max_epu8(fl[j], s); + const __m256i h = _mm256_max_epu8(hl[j], s); + const __m256i e = _mm256_max_epu8( + _mm256_subs_epu8(ecur[j], vrdgape), _mm256_subs_epu8(h, epen[j])); + _mm256_store_si256(pv + F, f); + _mm256_store_si256(pv + H, h); + _mm256_store_si256(pv + colstride + E, e); + pv += ROWSTRIDE; + colmax = _mm256_max_epu8(colmax, h); + hcur[j] = h; + ecur[j] = e; + } + // The column's maximum, its saturation and bail-out tests, as the + // general loop's + _mm256_store_si256(pvCol + TMP, colmax); + vm = _mm256_max_epu8(vm, colmax); + { + // The largest byte of colmax (as sse_max_score_u8): 255 less the + // smallest byte of its complement, taken by phminposuw over the + // minima of byte pairs + int score; + { + const __m128i m = _mm_max_epu8(_mm256_castsi256_si128(colmax), + _mm256_extracti128_si256(colmax, 1)); + const __m128i n = _mm_xor_si128(m, _mm_set1_epi8((char)0xff)); + const __m128i p = _mm_min_epu8(n, _mm_srli_epi16(n, 8)); + score = 255 - (_mm_cvtsi128_si32(_mm_minpos_epu16(p)) & 0xffff); + } + if(score + bias >= 255) { + lastsolcol = ls; + vmax = vm; + return -2; + } + if(score < minsc) { + size_t ncolleft = rff - i - 1; + if(score + (TAlScore)ncolleft * matchsc < minsc) { + colstop = (i+1) - rfi; + break; + } + } else { + ls = i - rfi; + } + } + pvCol += colstride; + // H0 of the next column. Its diagonal is row j - 1 of this column, + // or for j = 0 the last vector row shifted down a lane, with 0 in + // lane 0. That is max(HL, c - vpen) shifted, taken from hlsh and csh + // so that the shift is not after c. + { + const __m256i d0 = _mm256_max_epu8(hlsh, _mm256_subs_epu8(csh, vpenSh)); + h0[0] = _mm256_max_epu8(_mm256_subs_epu8( + _mm256_adds_epu8(d0, _mm256_load_si256(pvS)), vbias), ecur[0]); + } + for(size_t j = 1; j < ITER; j++) { + h0[j] = _mm256_max_epu8(_mm256_subs_epu8( + _mm256_adds_epu8(hcur[j-1], _mm256_load_si256(pvS + 2 * j)), vbias), ecur[j]); + } + } + lastsolcol = ls; + vmax = vm; + return 0; +} +#endif + /** * Solve the current alignment problem using SSE instructions that operate on 16 * unsigned 8-bit values packed into a single 128-bit register. @@ -1049,6 +1408,60 @@ TAlScore SwAligner::alignNucleotidesLocalSseU8(int& flag, bool debug) { colstop_ = rff_ - rfi_; lastsolcol_ = 0; +#if NBYTES_PER_REG == 32 + // 33 to 128 rows. The loop below spends most of each column on the chain + // of dependent instructions from one column to the next and on the lazy-F + // loop; alignLoc8V256Short computes the same cells with a shorter chain. + // It needs every path cost through the diagonal to be non-negative (the + // largest match bonus at most rfgapo) and the costs it derives from rfgapo + // and rfgape not to saturate. + if(iter >= 2 && iter <= 4 && + d.maxBonus_ <= (size_t)sc_->refGapOpen() && + sc_->refGapExtend() <= sc_->refGapOpen() && + sc_->refGapOpen() + (int)(iter - 1) * sc_->refGapExtend() + d.bias_ <= 255) + { + size_t colstop = 0, lastsolcol = 0; + int ret; + if(iter == 2) { + ret = alignLoc8V256Short<2>(d.profbuf_.ptr(), d.mat_.ptr(), colstride, + rf_, (size_t)rfi_, (size_t)rff_, *sc_, d.bias_, minsc_, matchsc, + colstop, lastsolcol, vmax); + } else if(iter == 3) { + ret = alignLoc8V256Short<3>(d.profbuf_.ptr(), d.mat_.ptr(), colstride, + rf_, (size_t)rfi_, (size_t)rff_, *sc_, d.bias_, minsc_, matchsc, + colstop, lastsolcol, vmax); + } else { + ret = alignLoc8V256Short<4>(d.profbuf_.ptr(), d.mat_.ptr(), colstride, + rf_, (size_t)rfi_, (size_t)rff_, *sc_, d.bias_, minsc_, matchsc, + colstop, lastsolcol, vmax); + } + lastsolcol_ = lastsolcol; + if(ret == -2) { + flag = -2; // saturated + if(!debug) met.dpsat++; + return MIN_I64; + } + colstop_ = colstop; + // The scan does the work of one fix-up pass per column + nfixup += colstop_ * iter; +#ifndef NDEBUG + for(size_t i = (size_t)rfi_; i < (size_t)rfi_ + colstop_; i++) { + if((rand() & 15) == 0) { + for(size_t k = 0; k < dpRows(); k++) { + assert(cellOkLocalU8( + d, + k, // row + i - rfi_, // col + (int)rf_[i], // reference mask + (int)(*rd_)[rdi_+k], // read char + (int)(*qu_)[rdi_+k], // read quality + *sc_)); // scoring scheme + } + } + } +#endif + } else +#endif for(size_t i = (size_t)rfi_; i < (size_t)rff_; i++) { assert(pvFStore == d.mat_.fvec(0, i - rfi_)); assert(pvHStore == d.mat_.hvec(0, i - rfi_)); @@ -1406,153 +1819,166 @@ bool SwAligner::gatherCellsNucleotidesLocalSseU8(TAlScore best) { assert_geq(minsc_, 0); assert_gt(bonus, 0); size_t minrow = (size_t)(((minsc_ + bonus - 1) / bonus) - 1); - for(size_t j = 0; j < ncol; j++) { - // Establish the range of rows where a backtrace from the cell in this - // row/col is close enough to one of the core diagonals that it could - // conceivably count - size_t nrow_lo = MIN_SIZE_T; - size_t nrow_hi = nrow; - // First, check if there is a cell in this column with a score - // above the score threshold - int score = 0; - sse_max_score_u8(*d.mat_.tmpvec(0, j), score); + // A candidate is a cell in a row from minrow on whose H is at least + // minsc_, whose read character matches the reference character, and + // whose successor on the diagonal (down and to the right) does not match. + // No H byte can reach a minsc_ above 255, and no row can be a candidate + // when minrow is past the last one. Otherwise the tests are done on a + // whole vector of H at a time, and only the lanes that pass all of them + // are visited, in the order of a scan by column, then vector row, then + // lane. + if(minsc_ <= 0xff && minrow < nrow) { + SSERegI vminsc; + SSERegI vzero = sse_setzero_siall(); + sse_fill_u8((int)minsc_, vminsc); + // For each vector row i, two vectors with one byte per lane k, i.e. + // per row i + k*iter: the row's read character as a mask (1 << c), + // and the next row's. Rows before minrow, and padding rows past the + // read, get 0 in both, which matches no reference character. The + // last row has no successor and gets 0 in the second. + d.vecbuf_.resizeNoCopy(2 * iter); + uint8_t *rdmask = reinterpret_cast(d.vecbuf_.ptr()); + for(size_t i = 0; i < iter; i++) { + uint8_t *rdc = rdmask + (2 * i) * NWORDS_PER_REG; // this row + uint8_t *rdn = rdc + NWORDS_PER_REG; // next row + for(size_t k = 0; k < NWORDS_PER_REG; k++) { + const size_t rdoff = i + k * iter; + rdc[k] = rdn[k] = 0; + if(rdoff >= minrow && rdoff < nrow) { + assert_lt((int)(*rd_)[rdoff], 8); + rdc[k] = (uint8_t)(1 << (*rd_)[rdoff]); + if(rdoff < dpRows()-1) { + assert_lt((int)(*rd_)[rdoff+1], 8); + rdn[k] = (uint8_t)(1 << (*rd_)[rdoff+1]); + } + } + } + } + for(size_t j = 0; j < ncol; j++) { + // First, check if there is a cell in this column with a score + // above the score threshold: the column maxima are in the first + // TMP vector + SSERegI vcolmax = sse_load_siall(d.mat_.tmpvec(0, j)); + int colpass = sse_movemask_epi8( + sse_cmpeq_epi8(sse_max_epu8(vcolmax, vminsc), vcolmax)); #ifndef NDEBUG - { - // Start in upper vector row and move down - TAlScore max = 0; + { + int score = 0; + sse_max_score_u8(vcolmax, score); + // Start in upper vector row and move down + TAlScore max = 0; + SSERegI *pvH = d.mat_.hvec(0, j); + for(size_t i = 0; i < iter; i++) { + for(size_t k = 0; k < NWORDS_PER_REG; k++) { + TAlScore sc = (TAlScore)((TCScore*)pvH)[k]; + if(sc > max) { + max = sc; + } + } + pvH += ROWSTRIDE; + } + assert_eq(max, score); + assert_eq(colpass != 0, (TAlScore)score >= minsc_); + } +#endif + if(colpass == 0) { + // Scores in column aren't good enough + continue; + } + // This column's reference character and the next one's (the + // reference has one character past the rectangle), as masks + assert_range(0, 16, (int)rf_[j + rfi_]); + assert_range(0, 16, (int)rf_[j + rfi_ + 1]); + SSERegI vref, vrefSucc; + sse_fill_u8((uint8_t)rf_[j + rfi_], vref); + sse_fill_u8((uint8_t)rf_[j + rfi_ + 1], vrefSucc); + // Get pointer to first cell in column to examine: SSERegI *pvH = d.mat_.hvec(0, j); + const SSERegI *pvRd = d.vecbuf_.ptr(); + // Start in upper vector row and move down for(size_t i = 0; i < iter; i++) { - for(size_t k = 0; k < NWORDS_PER_REG; k++) { + SSERegI vh = sse_load_siall(pvH); + // Lanes whose H is at least minsc_ ... + uint32_t lanes = (uint32_t)sse_movemask_epi8( + sse_cmpeq_epi8(sse_max_epu8(vh, vminsc), vh)); + // ... whose read character matches the reference character ... + lanes &= ~(uint32_t)sse_movemask_epi8( + sse_cmpeq_epi8(sse_and_siall(vref, pvRd[0]), vzero)); + // ... and whose diagonal successor does not match + lanes &= (uint32_t)sse_movemask_epi8( + sse_cmpeq_epi8(sse_and_siall(vrefSucc, pvRd[1]), vzero)); + while(lanes != 0) { + const size_t k = (size_t)__builtin_ctz(lanes); + lanes &= lanes - 1; + const size_t rdoff = i + k * iter; + assert_geq(rdoff, minrow); + assert_lt(rdoff, nrow); TAlScore sc = (TAlScore)((TCScore*)pvH)[k]; - if(sc > max) { - max = sc; - } + assert_leq(sc, best); + assert_geq(sc, minsc_); + // Yes, this is legit + met.gathsol++; + btncand_.expand(); + btncand_.back().init(rdoff, j, sc); } pvH += ROWSTRIDE; + pvRd += 2; } - assert_eq(max, score); } -#endif - if((TAlScore)score < minsc_) { - // Scores in column aren't good enough - continue; - } - // Get pointer to first cell in column to examine: - SSERegI *pvHorig = d.mat_.hvec(0, j); - SSERegI *pvH = pvHorig; - // Get pointer to the vector in the following column that corresponds - // to the cells diagonally down and to the right from the cells in pvH - SSERegI *pvHSucc = (j < ncol-1) ? d.mat_.hvec(0, j+1) : NULL; - // Start in upper vector row and move down - for(size_t i = 0; i < iter; i++) { - if(pvHSucc != NULL) { - pvHSucc += ROWSTRIDE; - if(i == iter-1) { - pvHSucc = d.mat_.hvec(0, j+1); - } - } - // Which elements of this vector are exhaustively scored? - size_t rdoff = i; - for(size_t k = 0; k < NWORDS_PER_REG; k++) { - // Is this row, col one that we can potential backtrace from? - // I.e. are we close enough to a core diagonal? - if(rdoff >= nrow_lo && rdoff < nrow_hi) { - // This cell has been exhaustively scored - if(rdoff >= minrow) { - // ... and it could potentially score high enough - TAlScore sc = (TAlScore)((TCScore*)pvH)[k]; - assert_leq(sc, best); - if(sc >= minsc_) { - // This is a potential solution - bool matchSucc = false; - int readc = (*rd_)[rdoff]; - int refc = rf_[j + rfi_]; - bool match = ((refc & (1 << readc)) != 0); - if(rdoff < dpRows()-1) { - int readcSucc = (*rd_)[rdoff+1]; - int refcSucc = rf_[j + rfi_ + 1]; - assert_range(0, 16, refcSucc); - matchSucc = ((refcSucc & (1 << readcSucc)) != 0); - } - if(match && !matchSucc) { - // Yes, this is legit - met.gathsol++; - btncand_.expand(); - btncand_.back().init(rdoff, j, sc); - } - } + } +#ifndef NDEBUG + { + // The same candidates, in the same order, by a scalar scan of every + // cell + size_t n = 0; + for(size_t j = 0; j < ncol; j++) { + SSERegI *pvH = d.mat_.hvec(0, j); + for(size_t i = 0; i < iter; i++) { + for(size_t k = 0; k < NWORDS_PER_REG; k++) { + size_t rdoff = i + k * iter; + if(rdoff >= nrow) { + break; + } + TAlScore sc = (TAlScore)((TCScore*)pvH)[k]; + if(rdoff < minrow || sc < minsc_) { + continue; + } + bool match = ((rf_[j + rfi_] & (1 << (*rd_)[rdoff])) != 0); + bool matchSucc = rdoff < dpRows()-1 && + ((rf_[j + rfi_ + 1] & (1 << (*rd_)[rdoff+1])) != 0); + if(match && !matchSucc) { + assert_lt(n, btncand_.size()); + assert_eq(rdoff, btncand_[n].row); + assert_eq(j, btncand_[n].col); + assert_eq(sc, btncand_[n].score); + n++; } - } else { - // Already saw every element in the vector that's been - // exhaustively scored - break; } - rdoff += iter; + pvH += ROWSTRIDE; } - pvH += ROWSTRIDE; } + assert_eq(n, btncand_.size()); } +#endif if(!btncand_.empty()) { d.mat_.initMasks(); } return !btncand_.empty(); } -#define MOVE_VEC_PTR_UP(vec, rowvec, rowelt) { \ - if(rowvec == 0) { \ - rowvec += d.mat_.nvecrow_; \ - vec += d.mat_.colstride_; \ - rowelt--; \ - } \ - rowvec--; \ - vec -= ROWSTRIDE; \ -} - -#define MOVE_VEC_PTR_LEFT(vec, rowvec, rowelt) { vec -= d.mat_.colstride_; } - -#define MOVE_VEC_PTR_UPLEFT(vec, rowvec, rowelt) { \ - MOVE_VEC_PTR_UP(vec, rowvec, rowelt); \ - MOVE_VEC_PTR_LEFT(vec, rowvec, rowelt); \ -} - -#define MOVE_ALL_LEFT() { \ - MOVE_VEC_PTR_LEFT(cur_vec, rowvec, rowelt); \ - MOVE_VEC_PTR_LEFT(left_vec, left_rowvec, left_rowelt); \ - MOVE_VEC_PTR_LEFT(up_vec, up_rowvec, up_rowelt); \ - MOVE_VEC_PTR_LEFT(upleft_vec, upleft_rowvec, upleft_rowelt); \ -} - -#define MOVE_ALL_UP() { \ - MOVE_VEC_PTR_UP(cur_vec, rowvec, rowelt); \ - MOVE_VEC_PTR_UP(left_vec, left_rowvec, left_rowelt); \ - MOVE_VEC_PTR_UP(up_vec, up_rowvec, up_rowelt); \ - MOVE_VEC_PTR_UP(upleft_vec, upleft_rowvec, upleft_rowelt); \ -} - -#define MOVE_ALL_UPLEFT() { \ - MOVE_VEC_PTR_UPLEFT(cur_vec, rowvec, rowelt); \ - MOVE_VEC_PTR_UPLEFT(left_vec, left_rowvec, left_rowelt); \ - MOVE_VEC_PTR_UPLEFT(up_vec, up_rowvec, up_rowelt); \ - MOVE_VEC_PTR_UPLEFT(upleft_vec, upleft_rowvec, upleft_rowelt); \ -} - -#define NEW_ROW_COL(row, col) { \ - rowelt = row / d.mat_.nvecrow_; \ - rowvec = row % d.mat_.nvecrow_; \ - eltvec = (col * d.mat_.colstride_) + (rowvec * ROWSTRIDE); \ - cur_vec = d.mat_.matbuf_.ptr() + eltvec; \ - left_vec = cur_vec; \ - left_rowelt = rowelt; \ - left_rowvec = rowvec; \ - MOVE_VEC_PTR_LEFT(left_vec, left_rowvec, left_rowelt); \ - up_vec = cur_vec; \ - up_rowelt = rowelt; \ - up_rowvec = rowvec; \ - MOVE_VEC_PTR_UP(up_vec, up_rowvec, up_rowelt); \ - upleft_vec = up_vec; \ - upleft_rowelt = up_rowelt; \ - upleft_rowvec = up_rowvec; \ - MOVE_VEC_PTR_LEFT(upleft_vec, upleft_rowvec, upleft_rowelt); \ +/* + * The backtrace addresses the striped matrix by one byte pointer per cell: + * btcur points at the current cell's E byte, element row / nvecrow_ of the + * E vector in vector row btrowvec = row % nvecrow_ of its column. The F and + * H bytes are one and two vectors further on (btF, btH), the cell to the + * left is one column (btLeft bytes) back, and the cell above is one vector + * row back (btUp bytes), or, from vector row 0, the previous element of the + * last vector row (btUpWrap bytes on). These are the moves the parent's + * MOVE_VEC_PTR_UP and MOVE_VEC_PTR_LEFT made on each of its four pointers. + */ +#define BT_SET_ROW_COL(row, col) { \ + btrowvec = (row) % btnvecrow; \ + btcur = btmatbuf + ((col) * btcolstride + btrowvec * ROWSTRIDE) * NBYTES_PER_REG + (row) / btnvecrow; \ } /** @@ -1599,9 +2025,46 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( SSEData& d = fw_ ? sseU8fw_ : sseU8rc_; SSEMetrics& met = extend_ ? sseU8ExtendMet_ : sseU8MateMet_; met.bt++; + // Loop invariants, kept in locals (the build has -fno-strict-aliasing, + // so the compiler would otherwise reload them after every store): the + // vector matrix's geometry in bytes (BT_SET_ROW_COL), the scoring + // parameters, the read, quality and reference, and the mask, reset and + // dirty-interval arrays (resetRow changes a row's contents, not these) + const size_t btnvecrow = d.mat_.nvecrow_; + const size_t btcolstride = d.mat_.colstride_; + uint8_t * const btmatbuf = (uint8_t*)d.mat_.matbuf_.ptr(); + const size_t btLeft = btcolstride * NBYTES_PER_REG; + const size_t btUp = ROWSTRIDE * NBYTES_PER_REG; + const size_t btUpWrap = (btcolstride - ROWSTRIDE) * NBYTES_PER_REG - 1; + const size_t btF = SSEMatrix::F * NBYTES_PER_REG; + const size_t btH = SSEMatrix::H * NBYTES_PER_REG; + assert_eq(0, SSEMatrix::E); + EList * const btmasks = d.mat_.masks_.ptr(); + const bool * const btreset = d.mat_.reset_.ptr(); + size_t * const btdirtylo = d.mat_.dirtylo_.ptr(); + size_t * const btdirtyhi = d.mat_.dirtyhi_.ptr(); + const size_t btnrow = d.mat_.nrow(); + const size_t gapbar = (size_t)sc_->gapbar; + const int rdgapo = sc_->readGapOpen(); + const int rdgape = sc_->readGapExtend(); + const int rfgapo = sc_->refGapOpen(); + const int rfgape = sc_->refGapExtend(); + const int64_t bonus = sc_->match(30); + const Scoring * const sc = sc_; + const char * const rdbuf = rd_->buf(); + const char * const qubuf = qu_->buf(); + const char * const rfbuf = rf_ + rfi_; + const size_t rdoff = rdi_; + size_t ncell = 0; // cells visited, added to met.btcell and nbts on return + // Branch points the parent code pushed on its backtrack stack (btnstack_). + // A frame's cell is marked reported through before the frame is pushed, + // so a popped frame re-enters a reported cell, which pops the next frame: + // once a walk reaches a cell it cannot move through, it fails after one + // more cell visit per frame, and no frame ever yields a second path. + // Only their number is kept. + size_t nframes = 0; assert(!d.profbuf_.empty()); assert_lt(row, rd_->length()); - btnstack_.clear(); // empty the backtrack stack btcells_.clear(); // empty the cells-so-far list AlnScore score; score.score_ = 0; score.gaps_ = score.ns_ = 0; @@ -1614,26 +2077,41 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( size_t trimEnd = dpRows() - row - 1; size_t trimBeg = 0; size_t ct = SSEMatrix::H; // cell type - // Row and col in terms of where they fall in the SSE vector matrix - size_t rowelt, rowvec, eltvec; - size_t left_rowelt, up_rowelt, upleft_rowelt; - size_t left_rowvec, up_rowvec, upleft_rowvec; - SSERegI *cur_vec, *left_vec, *up_vec, *upleft_vec; - NEW_ROW_COL(row, col); + // The current cell in the SSE vector matrix (BT_SET_ROW_COL) + size_t btrowvec; + uint8_t *btcur; + BT_SET_ROW_COL(row, col); while((int)row >= 0) { - met.btcell++; - nbts++; - int readc = (*rd_)[rdi_ + row]; - int refm = (int)rf_[rfi_ + col]; - int readq = (*qu_)[row]; + ncell++; + int readc = rdbuf[rdoff + row]; + int refm = (int)rfbuf[col]; + int readq = qubuf[row]; + assert_eq(readc, (*rd_)[rdi_ + row]); + assert_eq(refm, (int)rf_[rfi_ + col]); + assert_eq(readq, (*qu_)[row]); assert_leq(col, origCol); + // The cell above; used only when row > 0 + const bool btwrap = (btrowvec == 0); + uint8_t * const btup = btwrap ? btcur + btUpWrap : btcur - btUp; + assert_eq(btcur[0], ((TCScore*)(d.mat_.matbuf_.ptr() + col * btcolstride + btrowvec * ROWSTRIDE))[row / btnvecrow]); + assert(row == 0 || btup[btH] == d.mat_.helt(row - 1, col)); // Get score in this cell - bool empty = false, reportedThru, canMoveThru, branch = false; + bool empty = false; + bool reportedThru, canMoveThru, branch = false; int cur = SSEMatrix::H; - if(!d.mat_.reset_[row]) { + if(!btreset[row]) { d.mat_.resetRow(row); } - reportedThru = d.mat_.reportedThrough(row, col); + // This cell's mask word (see SSEMatrix). Bit 0, reported through, + // is the only bit any code reads. This backtrace writes a word only + // when it enters a cell whose bit 0 is clear, and then writes 1, so + // a word is 0 or 1. The parent code also stored the remaining H, E + // and F masks in bits 1-12 and read them back only when bit 0 was + // clear, where its words were 0 too. + uint16_t& mk = btmasks[row][col]; + assert_eq(&mk, &d.mat_.masks_[row][col]); + reportedThru = (mk & (1 << 0)) != 0; + assert(reportedThru || mk == 0); canMoveThru = true; if(reportedThru) { canMoveThru = false; @@ -1641,10 +2119,10 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( empty = false; if(row > 0) { assert_gt(row, 0); - size_t rowFromEnd = d.mat_.nrow() - row - 1; + size_t rowFromEnd = btnrow - row - 1; bool gapsAllowed = true; - if(row < (size_t)sc_->gapbar || - rowFromEnd < (size_t)sc_->gapbar) + if(row < gapbar || + rowFromEnd < gapbar) { gapsAllowed = false; } @@ -1653,7 +2131,8 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( // Move to beginning of column/row if(ct == SSEMatrix::E) { // AKA rdgap assert_gt(col, 0); - TAlScore sc_cur = ((TCScore*)(cur_vec + SSEMatrix::E))[rowelt] + offsetsc; + const uint8_t *btleft = btcur - btLeft; + TAlScore sc_cur = ((TCScore)btcur[0]) + offsetsc; assert(gapsAllowed); // Currently in the E matrix; incoming transition must come from the // left. It's either a gap open from the H matrix or a gap extend from @@ -1661,45 +2140,27 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( // TODO: save and restore origMask as well as mask int origMask = 0, mask = 0; // Get H score of cell to the left - TAlScore sc_h_left = ((TCScore*)(left_vec + SSEMatrix::H))[left_rowelt] + offsetsc; - if(sc_h_left > 0 && sc_h_left - sc_->readGapOpen() == sc_cur) { + TAlScore sc_h_left = ((TCScore)btleft[btH]) + offsetsc; + if(sc_h_left > 0 && sc_h_left - rdgapo == sc_cur) { mask |= (1 << 0); } // Get E score of cell to the left - TAlScore sc_e_left = ((TCScore*)(left_vec + SSEMatrix::E))[left_rowelt] + offsetsc; - if(sc_e_left > 0 && sc_e_left - sc_->readGapExtend() == sc_cur) { + TAlScore sc_e_left = ((TCScore)btleft[0]) + offsetsc; + if(sc_e_left > 0 && sc_e_left - rdgape == sc_cur) { mask |= (1 << 1); } origMask = mask; assert(origMask > 0 || sc_cur <= sc_->match()); - if(d.mat_.isEMaskSet(row, col)) { - mask = (d.mat_.masks_[row][col] >> 8) & 3; - } if(mask == 3) { -#if 1 // Pick H -> E cell cur = SW_BT_OALL_READ_OPEN; - d.mat_.eMaskSet(row, col, 2); // might choose E later -#else - if(rnd.nextU2()) { - // Pick H -> E cell - cur = SW_BT_OALL_READ_OPEN; - d.mat_.eMaskSet(row, col, 2); // might choose E later - } else { - // Pick E -> E cell - cur = SW_BT_RDGAP_EXTEND; - d.mat_.eMaskSet(row, col, 1); // might choose H later - } -#endif branch = true; } else if(mask == 2) { // I chose the E cell cur = SW_BT_RDGAP_EXTEND; - d.mat_.eMaskSet(row, col, 0); // done } else if(mask == 1) { // I chose the H cell cur = SW_BT_OALL_READ_OPEN; - d.mat_.eMaskSet(row, col, 0); // done } else { empty = true; // It's empty, so the only question left is whether we should be @@ -1712,52 +2173,34 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( } else if(ct == SSEMatrix::F) { // AKA rfgap assert_gt(row, 0); assert(gapsAllowed); - TAlScore sc_h_up = ((TCScore*)(up_vec + SSEMatrix::H))[up_rowelt] + offsetsc; - TAlScore sc_f_up = ((TCScore*)(up_vec + SSEMatrix::F))[up_rowelt] + offsetsc; - TAlScore sc_cur = ((TCScore*)(cur_vec + SSEMatrix::F))[rowelt] + offsetsc; + TAlScore sc_h_up = ((TCScore)btup[btH]) + offsetsc; + TAlScore sc_f_up = ((TCScore)btup[btF]) + offsetsc; + TAlScore sc_cur = ((TCScore)btcur[btF]) + offsetsc; // Currently in the F matrix; incoming transition must come from above. // It's either a gap open from the H matrix or a gap extend from the F // matrix. // TODO: save and restore origMask as well as mask int origMask = 0, mask = 0; // Get H score of cell above - if(sc_h_up > floorsc && sc_h_up - sc_->refGapOpen() == sc_cur) { + if(sc_h_up > floorsc && sc_h_up - rfgapo == sc_cur) { mask |= (1 << 0); } // Get F score of cell above - if(sc_f_up > floorsc && sc_f_up - sc_->refGapExtend() == sc_cur) { + if(sc_f_up > floorsc && sc_f_up - rfgape == sc_cur) { mask |= (1 << 1); } origMask = mask; assert(origMask > 0 || sc_cur <= sc_->match()); - if(d.mat_.isFMaskSet(row, col)) { - mask = (d.mat_.masks_[row][col] >> 11) & 3; - } if(mask == 3) { -#if 1 // I chose the H cell cur = SW_BT_OALL_REF_OPEN; - d.mat_.fMaskSet(row, col, 2); // might choose E later -#else - if(rnd.nextU2()) { - // I chose the H cell - cur = SW_BT_OALL_REF_OPEN; - d.mat_.fMaskSet(row, col, 2); // might choose E later - } else { - // I chose the F cell - cur = SW_BT_RFGAP_EXTEND; - d.mat_.fMaskSet(row, col, 1); // might choose E later - } -#endif branch = true; } else if(mask == 2) { // I chose the F cell cur = SW_BT_RFGAP_EXTEND; - d.mat_.fMaskSet(row, col, 0); // done } else if(mask == 1) { // I chose the H cell cur = SW_BT_OALL_REF_OPEN; - d.mat_.fMaskSet(row, col, 0); // done } else { empty = true; // It's empty, so the only question left is whether we should be @@ -1769,26 +2212,26 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( assert(!empty || !canMoveThru); } else { assert_eq(SSEMatrix::H, ct); - TAlScore sc_cur = ((TCScore*)(cur_vec + SSEMatrix::H))[rowelt] + offsetsc; - TAlScore sc_f_up = ((TCScore*)(up_vec + SSEMatrix::F))[up_rowelt] + offsetsc; - TAlScore sc_h_up = ((TCScore*)(up_vec + SSEMatrix::H))[up_rowelt] + offsetsc; - TAlScore sc_h_left = col > 0 ? (((TCScore*)(left_vec + SSEMatrix::H))[left_rowelt] + offsetsc) : floorsc; - TAlScore sc_e_left = col > 0 ? (((TCScore*)(left_vec + SSEMatrix::E))[left_rowelt] + offsetsc) : floorsc; - TAlScore sc_h_upleft = col > 0 ? (((TCScore*)(upleft_vec + SSEMatrix::H))[upleft_rowelt] + offsetsc) : floorsc; - TAlScore sc_diag = sc_->score(readc, refm, readq - 33); + TAlScore sc_cur = ((TCScore)btcur[btH]) + offsetsc; + TAlScore sc_f_up = ((TCScore)btup[btF]) + offsetsc; + TAlScore sc_h_up = ((TCScore)btup[btH]) + offsetsc; + TAlScore sc_h_left = col > 0 ? (((TCScore)(btcur - btLeft)[btH]) + offsetsc) : floorsc; + TAlScore sc_e_left = col > 0 ? (((TCScore)(btcur - btLeft)[0]) + offsetsc) : floorsc; + TAlScore sc_h_upleft = col > 0 ? (((TCScore)(btup - btLeft)[btH]) + offsetsc) : floorsc; + TAlScore sc_diag = sc->score(readc, refm, readq - 33); // TODO: save and restore origMask as well as mask int origMask = 0, mask = 0; if(gapsAllowed) { - if(sc_h_up > floorsc && sc_cur == sc_h_up - sc_->refGapOpen()) { + if(sc_h_up > floorsc && sc_cur == sc_h_up - rfgapo) { mask |= (1 << 0); } - if(sc_h_left > floorsc && sc_cur == sc_h_left - sc_->readGapOpen()) { + if(sc_h_left > floorsc && sc_cur == sc_h_left - rdgapo) { mask |= (1 << 1); } - if(sc_f_up > floorsc && sc_cur == sc_f_up - sc_->refGapExtend()) { + if(sc_f_up > floorsc && sc_cur == sc_f_up - rfgape) { mask |= (1 << 2); } - if(sc_e_left > floorsc && sc_cur == sc_e_left - sc_->readGapExtend()) { + if(sc_e_left > floorsc && sc_cur == sc_e_left - rdgape) { mask |= (1 << 3); } } @@ -1797,50 +2240,22 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( } origMask = mask; assert(origMask > 0 || sc_cur <= sc_->match()); - if(d.mat_.isHMaskSet(row, col)) { - mask = (d.mat_.masks_[row][col] >> 2) & 31; - } assert(gapsAllowed || mask == (1 << 4) || mask == 0); - int opts = alts5[mask]; - int select = -1; - if(opts == 1) { - select = firsts5[mask]; - assert_geq(mask, 0); - d.mat_.hMaskSet(row, col, 0); - } else if(opts > 1) { -#if 1 - if( (mask & 16) != 0) { - select = 4; // H diag - } else if((mask & 1) != 0) { - select = 0; // H up - } else if((mask & 4) != 0) { - select = 2; // F up - } else if((mask & 2) != 0) { - select = 1; // H left - } else if((mask & 8) != 0) { - select = 3; // E left - } -#else - select = randFromMask(rnd, mask); -#endif - assert_geq(mask, 0); - mask &= ~(1 << select); - assert(gapsAllowed || mask == (1 << 4) || mask == 0); - d.mat_.hMaskSet(row, col, mask); - branch = true; - } else { /* No way to backtrack! */ } - if(select != -1) { - if(select == 4) { - cur = SW_BT_OALL_DIAG; - } else if(select == 0) { - cur = SW_BT_OALL_REF_OPEN; - } else if(select == 1) { - cur = SW_BT_OALL_READ_OPEN; - } else if(select == 2) { - cur = SW_BT_RFGAP_EXTEND; - } else { - assert_eq(3, select) - cur = SW_BT_RDGAP_EXTEND; + if(mask != 0) { + // The move and the mask left behind, from the table + // (see btHSelect) + int sel = btHSelect[mask]; + cur = sel & 7; + int rest = sel >> 3; + assert_eq(rest, mask & ~btHSelectBit(mask)); + assert_eq(rest != 0, alts5[mask] > 1); + assert_eq(cur, btHSelectBit(mask) == 16 ? (int)SW_BT_OALL_DIAG : + btHSelectBit(mask) == 1 ? (int)SW_BT_OALL_REF_OPEN : + btHSelectBit(mask) == 2 ? (int)SW_BT_OALL_READ_OPEN : + btHSelectBit(mask) == 4 ? (int)SW_BT_RFGAP_EXTEND : + (int)SW_BT_RDGAP_EXTEND); + if(rest != 0) { + branch = true; } } else { empty = true; @@ -1854,35 +2269,31 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( assert(!empty || !canMoveThru || ct == SSEMatrix::H); } } - d.mat_.setReportedThrough(row, col); + //cerr << "reportedThrough rejected (" << row << ", " << col << ")" << endl; + if(!reportedThru) { + // Mark the cell reported through, and record it in the row's + // dirty interval (SSEMatrix::markDirty). A cell already + // reported through was marked in this fill, after its row was + // reset, so its word is 1 and its column is in the interval. + mk = (1 << 0); + btdirtylo[row] = (col < btdirtylo[row]) ? col : btdirtylo[row]; + btdirtyhi[row] = (col >= btdirtyhi[row]) ? col + 1 : btdirtyhi[row]; + } + assert(d.mat_.reportedThrough(row, col)); assert_eq(gaps, Edit::numGaps(ned)); assert_leq(gaps, rdgap_ + rfgap_); // Cell was involved in a previously-reported alignment? if(!canMoveThru) { - if(!btnstack_.empty()) { - // Remove all the cells from list back to and including the - // cell where the branch occurred - btcells_.resize(btnstack_.back().celsz); - // Pop record off the top of the stack - ned.resize(btnstack_.back().nedsz); - //aed.resize(btnstack_.back().aedsz); - row = btnstack_.back().row; - col = btnstack_.back().col; - gaps = btnstack_.back().gaps; - readGaps = btnstack_.back().readGaps; - refGaps = btnstack_.back().refGaps; - score = btnstack_.back().score; - ct = btnstack_.back().ct; - btnstack_.pop_back(); - assert(!sc_->monotone || score.score() >= escore); - NEW_ROW_COL(row, col); - continue; - } else { - // No branch points to revisit; just give up - res.reset(); - met.btfail++; // DP backtraces failed - return false; - } + // Give up. The parent code first popped each frame of its + // backtrack stack and re-entered the frame's cell, which it had + // marked reported through before pushing the frame, so each pop + // counted one more cell and led to the next pop. + ncell += nframes; + met.btcell += ncell; + nbts += ncell; + res.reset(); + met.btfail++; // DP backtraces failed + return false; } assert(!reportedThru); assert(!sc_->monotone || score.score() >= minsc_); @@ -1897,19 +2308,8 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( break; } if(branch) { - // Add a frame to the backtrack stack - btnstack_.expand(); - btnstack_.back().init( - ned.size(), - 0, // aed.size() - btcells_.size(), - row, - col, - gaps, - readGaps, - refGaps, - score, - (int)ct); + // The parent code added a frame to its backtrack stack here + nframes++; } btcells_.expand(); btcells_.back().first = row; @@ -1920,8 +2320,9 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( // it and add a nucleotide mismatch. case SW_BT_OALL_DIAG: { assert_gt(row, 0); assert_gt(col, 0); - int readC = (*rd_)[row]; - int refNmask = (int)rf_[rfi_+col]; + int readC = rdbuf[row]; + int refNmask = (int)rfbuf[col]; + assert_eq(readC, (*rd_)[row]); assert_gt(refNmask, 0); int m = matchesEx(readC, refNmask); ct = SSEMatrix::H; @@ -1939,7 +2340,6 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( assert(!sc_->monotone || score.score() >= escore); } else { // Reward a match - int64_t bonus = sc_->match(30); score.score_ += bonus; assert(!sc_->monotone || score.score() >= escore); } @@ -1947,7 +2347,8 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( score.ns_++; } row--; col--; - MOVE_ALL_UPLEFT(); + btcur = btup - btLeft; + btrowvec = btwrap ? btnvecrow - 1 : btrowvec - 1; assert(VALID_AL_SCORE(score)); break; } @@ -1958,7 +2359,7 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( Edit e( (int)row, '-', - "ACGTN"[(int)(*rd_)[row]], + "ACGTN"[(int)rdbuf[row]], EDIT_TYPE_REF_GAP); assert(e.repOk()); assert(ned.empty() || ned.back().pos >= row); @@ -1967,13 +2368,14 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( assert_geq((int)(rdf_-rdi_-row-1), sc_->gapbar-1); row--; ct = SSEMatrix::H; - int pen = sc_->refGapOpen(); + int pen = rfgapo; score.score_ -= pen; assert(!sc_->monotone || score.score() >= minsc_); gaps++; refGaps++; assert_eq(gaps, Edit::numGaps(ned)); assert_leq(gaps, rdgap_ + rfgap_); - MOVE_ALL_UP(); + btcur = btup; + btrowvec = btwrap ? btnvecrow - 1 : btrowvec - 1; break; } // Move up. Add an edit encoding the ref gap. @@ -1983,7 +2385,7 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( Edit e( (int)row, '-', - "ACGTN"[(int)(*rd_)[row]], + "ACGTN"[(int)rdbuf[row]], EDIT_TYPE_REF_GAP); assert(e.repOk()); assert(ned.empty() || ned.back().pos >= row); @@ -1992,13 +2394,14 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( assert_geq((int)(rdf_-rdi_-row-1), sc_->gapbar-1); row--; ct = SSEMatrix::F; - int pen = sc_->refGapExtend(); + int pen = rfgape; score.score_ -= pen; assert(!sc_->monotone || score.score() >= minsc_); gaps++; refGaps++; assert_eq(gaps, Edit::numGaps(ned)); assert_leq(gaps, rdgap_ + rfgap_); - MOVE_ALL_UP(); + btcur = btup; + btrowvec = btwrap ? btnvecrow - 1 : btrowvec - 1; break; } case SW_BT_OALL_READ_OPEN: @@ -2006,7 +2409,7 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( assert_gt(col, 0); Edit e( (int)row+1, - mask2dna[(int)rf_[rfi_+col]], + mask2dna[(int)rfbuf[col]], '-', EDIT_TYPE_READ_GAP); assert(e.repOk()); @@ -2016,13 +2419,13 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( assert_geq((int)(rdf_-rdi_-row-1), sc_->gapbar-1); col--; ct = SSEMatrix::H; - int pen = sc_->readGapOpen(); + int pen = rdgapo; score.score_ -= pen; assert(!sc_->monotone || score.score() >= minsc_); gaps++; readGaps++; assert_eq(gaps, Edit::numGaps(ned)); assert_leq(gaps, rdgap_ + rfgap_); - MOVE_ALL_LEFT(); + btcur -= btLeft; break; } case SW_BT_RDGAP_EXTEND: @@ -2030,7 +2433,7 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( assert_gt(col, 1); Edit e( (int)row+1, - mask2dna[(int)rf_[rfi_+col]], + mask2dna[(int)rfbuf[col]], '-', EDIT_TYPE_READ_GAP); assert(e.repOk()); @@ -2040,18 +2443,20 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( assert_geq((int)(rdf_-rdi_-row-1), sc_->gapbar-1); col--; ct = SSEMatrix::E; - int pen = sc_->readGapExtend(); + int pen = rdgape; score.score_ -= pen; assert(!sc_->monotone || score.score() >= minsc_); gaps++; readGaps++; assert_eq(gaps, Edit::numGaps(ned)); assert_leq(gaps, rdgap_ + rfgap_); - MOVE_ALL_LEFT(); + btcur -= btLeft; break; } default: throw 1; } } // while((int)row > 0) + met.btcell += ncell; + nbts += ncell; assert_geq(col, 0); assert_eq(SSEMatrix::H, ct); // The number of cells in the backtracs should equal the number of read diff --git a/aln_sink.cpp b/aln_sink.cpp index 541f44e2..cb48eee1 100644 --- a/aln_sink.cpp +++ b/aln_sink.cpp @@ -1913,6 +1913,7 @@ void AlnSinkSam::appendMate( return; } char buf[1024]; + size_t nbuf = 0; // length of the number itoa10 last wrote into buf int offAdj = 0; // QNAME samc_.printReadName(o, rd.name, flags.partOfPair()); @@ -1947,8 +1948,8 @@ void AlnSinkSam::appendMate( // Failed to align fl |= SAM_FLAG_UNMAPPED; } - itoa10(fl, buf); - o.append(buf); + nbuf = (size_t)(itoa10(fl, buf) - buf); + o.append(buf, nbuf); o.append('\t'); // RNAME if(rs != NULL) { @@ -1970,16 +1971,16 @@ void AlnSinkSam::appendMate( // Note: POS is *after* soft clipping. I.e. POS points to the // upstream-most character *involved in the clipped alignment*. if(rs != NULL) { - itoa10(rs->refoff()+1+offAdj, buf); - o.append(buf); + nbuf = (size_t)(itoa10(rs->refoff()+1+offAdj, buf) - buf); + o.append(buf, nbuf); o.append('\t'); } else { if(summ.orefid() != -1) { // Opposite mate aligned but this one didn't - print the opposite // mate's RNAME and POS as is customary assert(flags.partOfPair()); - itoa10(summ.orefoff()+1+offAdj, buf); - o.append(buf); + nbuf = (size_t)(itoa10(summ.orefoff()+1+offAdj, buf) - buf); + o.append(buf, nbuf); } else { // No alignment o.append('0'); @@ -2020,29 +2021,29 @@ void AlnSinkSam::appendMate( // PNEXT if(rs != NULL && flags.partOfPair()) { if(rso != NULL) { - itoa10(rso->refoff()+1, buf); - o.append(buf); + nbuf = (size_t)(itoa10(rso->refoff()+1, buf) - buf); + o.append(buf, nbuf); o.append('\t'); } else { // The convenstion is that if this mate aligns but the opposite // doesn't, we print this mate's offset here - itoa10(rs->refoff()+1, buf); - o.append(buf); + nbuf = (size_t)(itoa10(rs->refoff()+1, buf) - buf); + o.append(buf, nbuf); o.append('\t'); } } else if(summ.orefid() != -1) { // The convention if this mate fails to align but the other doesn't is // to copy the mate's details into here - itoa10(summ.orefoff()+1, buf); - o.append(buf); + nbuf = (size_t)(itoa10(summ.orefoff()+1, buf) - buf); + o.append(buf, nbuf); o.append('\t'); } else { o.append("0\t"); } // ISIZE if(rs != NULL && rs->isFraglenSet()) { - itoa10(rs->fragmentLength(), buf); - o.append(buf); + nbuf = (size_t)(itoa10(rs->fragmentLength(), buf) - buf); + o.append(buf, nbuf); o.append('\t'); } else { // No fragment @@ -2056,10 +2057,11 @@ void AlnSinkSam::appendMate( if(rd.patFw.length() == 0) { o.append('*'); } else { + // Same bytes as o.append(rd.patFw.toZBuf()), written directly if(rs == NULL || rs->fw()) { - o.append(rd.patFw.toZBuf()); + o.appendXForm(rd.patFw.buf(), rd.patFw.length(), "ACGTN"); } else { - o.append(rd.patRc.toZBuf()); + o.appendXForm(rd.patRc.buf(), rd.patRc.length(), "ACGTN"); } } } diff --git a/bowtie_main.cpp b/bowtie_main.cpp index aaecb963..e0e33588 100644 --- a/bowtie_main.cpp +++ b/bowtie_main.cpp @@ -26,6 +26,10 @@ #ifdef ENABLE_x86_64_v3 #include +#ifdef _MSC_VER +#include +#include +#endif #endif using namespace std; @@ -35,38 +39,65 @@ extern "C" { } #ifdef ENABLE_x86_64_v3 -// taken from https://attractivechaos.wordpress.com/2017/09/04/on-cpu-dispatch/ -#define SIMD_SSE 0x1 -#define SIMD_SSE2 0x2 -#define SIMD_SSE3 0x4 -#define SIMD_SSE4_1 0x8 -#define SIMD_SSE4_2 0x10 -#define SIMD_AVX 0x20 -#define SIMD_AVX2 0x40 -#define SIMD_AVX512F 0x80 +// The -v256 binaries are compiled with -march=x86-64-v3: x86-64-v2 (SSE3, +// SSSE3, SSE4.1, SSE4.2, POPCNT, CMPXCHG16B, LAHF/SAHF) plus AVX, AVX2, BMI1, +// BMI2, F16C, FMA, LZCNT, MOVBE and XSAVE. The OS must also save the XMM and +// YMM registers (OSXSAVE, then XCR0 bits 1 and 2), as __builtin_cpu_supports +// ("x86-64-v3") requires; that builtin is not available on every compiler. +static const unsigned X86_64_V3_LEAF1_ECX = // CPUID leaf 1, ECX + (1u << 0) | (1u << 9) | (1u << 12) | (1u << 13) | // SSE3, SSSE3, FMA, CMPXCHG16B + (1u << 19) | (1u << 20) | (1u << 22) | (1u << 23) | // SSE4.1, SSE4.2, MOVBE, POPCNT + (1u << 26) | (1u << 27) | (1u << 28) | (1u << 29); // XSAVE, OSXSAVE, AVX, F16C +static const unsigned X86_64_V3_LEAF7_EBX = // CPUID leaf 7 sub-leaf 0, EBX + (1u << 3) | (1u << 5) | (1u << 8); // BMI1, AVX2, BMI2 +static const unsigned X86_64_V3_EXT1_ECX = // CPUID leaf 0x80000001, ECX + (1u << 0) | (1u << 5); // LAHF/SAHF, LZCNT + +// A leaf the CPU does not report is passed as 0. +static bool x86_64_v3_from_cpuid(unsigned leaf1_ecx, unsigned leaf7_ebx, + unsigned ext1_ecx, unsigned long long xcr0) { + return (leaf1_ecx & X86_64_V3_LEAF1_ECX) == X86_64_V3_LEAF1_ECX && + (leaf7_ebx & X86_64_V3_LEAF7_EBX) == X86_64_V3_LEAF7_EBX && + (ext1_ecx & X86_64_V3_EXT1_ECX) == X86_64_V3_EXT1_ECX && + (xcr0 & 6) == 6; +} + +static void x86_cpuid(unsigned leaf, unsigned subleaf, unsigned regs[4]) { +#ifdef _MSC_VER + int r[4]; + __cpuidex(r, (int)leaf, (int)subleaf); + for (int i = 0; i < 4; i++) regs[i] = (unsigned)r[i]; +#else + asm volatile("cpuid" : "=a" (regs[0]), "=b" (regs[1]), "=c" (regs[2]), "=d" (regs[3]) + : "a" (leaf), "c" (subleaf)); +#endif +} -unsigned x86_simd(void) { - unsigned eax, ebx, ecx, edx, flag = 0; +static bool x86_64_v3_supported(void) { + unsigned r[4]; + x86_cpuid(0, 0, r); + const unsigned max_leaf = r[0]; + x86_cpuid(0x80000000u, 0, r); + const unsigned max_ext_leaf = r[0]; + unsigned leaf1_ecx = 0, leaf7_ebx = 0, ext1_ecx = 0; + unsigned long long xcr0 = 0; + if (max_leaf >= 1) { x86_cpuid(1, 0, r); leaf1_ecx = r[2]; } + if (max_leaf >= 7) { x86_cpuid(7, 0, r); leaf7_ebx = r[1]; } + if (max_ext_leaf >= 0x80000001u) { x86_cpuid(0x80000001u, 0, r); ext1_ecx = r[2]; } + if ((leaf1_ecx >> 27) & 1) { // XGETBV faults unless the OS set OSXSAVE #ifdef _MSC_VER - int cpuid[4]; - __cpuid(cpuid, 1); - eax = cpuid[0], ebx = cpuid[1], ecx = cpuid[2], edx = cpuid[3]; + xcr0 = _xgetbv(0); #else - asm volatile("cpuid" : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) : "a" (1)); + unsigned lo, hi; + asm volatile("xgetbv" : "=a" (lo), "=d" (hi) : "c" (0)); + xcr0 = ((unsigned long long)hi << 32) | lo; #endif - if (edx>>25&1) flag |= SIMD_SSE; - if (edx>>26&1) flag |= SIMD_SSE2; - if (ecx>>0 &1) flag |= SIMD_SSE3; - if (ecx>>19&1) flag |= SIMD_SSE4_1; - if (ecx>>20&1) flag |= SIMD_SSE4_2; - if (ecx>>28&1) flag |= SIMD_AVX; - if (ebx>>5 &1) flag |= SIMD_AVX2; - if (ebx>>16&1) flag |= SIMD_AVX512F; - return flag; + } + return x86_64_v3_from_cpuid(leaf1_ecx, leaf7_ebx, ext1_ecx, xcr0); } void check_x86_64_v3(int argc, const char **argv) { - if ((x86_simd() & SIMD_AVX2) && (argc<126)) { + if (x86_64_v3_supported() && (argc<126)) { const char* new_argv[128]; // should always be enough, but above check enforces it, too const char * org_path = argv[0]; // Append -v256 to the original path @@ -76,10 +107,12 @@ void check_x86_64_v3(int argc, const char **argv) { strncpy(new_path+fn_len,"-v256",15); for (int i=1; i<=argc; i++) new_argv[i] = argv[i]; // all but first the same, also copy final NULL - new_argv[0] = new_path; + // keep argv[0], so --version and the @PG line name the binary that was run, + // whichever variant does the work + new_argv[0] = org_path; // now replace the executable with new variant // assuming the executable exists... execvp will gracefully fail else, which is OK - execvp(new_argv[0], (char *const *)new_argv); + execvp(new_path, (char *const *)new_argv); // we should never get out of the above call fprintf(stderr,"[WARNING] Failed to launch x86-64-v3 version, staying with default\n"); } diff --git a/bt2_idx.cpp b/bt2_idx.cpp index f6b44a9a..b0877ed5 100644 --- a/bt2_idx.cpp +++ b/bt2_idx.cpp @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include "bt2_idx.h" using namespace std; @@ -48,8 +50,10 @@ string gLastIOErrMsg; /** * Take an offset into the joined text and translate it into the * reference of the index it falls on, the offset into the reference, - * and the length of the reference. Use a binary search through the - * sorted list of reference fragment ranges t + * and the length of the reference. The fragment holding the offset + * comes from the bucket table built by buildFragTab() when there is + * one, else from a binary search through the sorted list of reference + * fragment ranges. */ void Ebwt::joinedToTextOff( TIndexOffU qlen, @@ -61,68 +65,144 @@ void Ebwt::joinedToTextOff( bool& straddled) const { assert(rstarts() != NULL); // must have loaded rstarts - TIndexOffU top = 0; - TIndexOffU bot = _nFrag; // 1 greater than largest addressable element - TIndexOffU elt = OFF_MASK; - // Begin binary search - while(true) { - ASSERT_ONLY(TIndexOffU oldelt = elt); - elt = top + ((bot - top) >> 1); - assert_neq(oldelt, elt); // must have made progress - TIndexOffU lower = rstarts()[elt*3]; - TIndexOffU upper; - if(elt == _nFrag-1) { - upper = _eh._len; - } else { - upper = rstarts()[((elt+1)*3)]; + TIndexOffU elt; + if(off < _fragTabLen) { + // The fragment holding 'off' is the last one starting at or + // before 'off'. It lies between the fragments holding the first + // offsets of off's bucket and of the next bucket, which are + // usually the same fragment. + size_t b = (size_t)(off >> _fragShift); + elt = _fragTab[b]; + TIndexOffU hi = _fragTab[b+1]; + while(elt < hi) { + TIndexOffU mid = elt + ((hi - elt + 1) >> 1); + if(rstarts()[mid*3] <= off) { + elt = mid; + } else { + hi = mid - 1; + } } - assert_gt(upper, lower); - TIndexOffU fraglen = upper - lower; - if(lower <= off) { - if(upper > off) { // not last element, but it's within - // off is in this range; check if it falls off - if(off + qlen > upper) { - straddled = true; - if(rejectStraddle) { - // it falls off; signal no-go and return - tidx = OFF_MASK; - assert_lt(elt, _nFrag-1); - return; - } - } - // This is the correct text idx whether the index is - // forward or reverse - tidx = rstarts()[(elt*3)+1]; - assert_lt(tidx, this->_nPat); - assert_leq(fraglen, this->plen()[tidx]); - // it doesn't fall off; now calculate textoff. - // Initially it's the number of characters that precede - // the alignment in the fragment - TIndexOffU fragoff = off - rstarts()[(elt*3)]; - if(!this->fw_) { - fragoff = fraglen - fragoff - 1; - fragoff -= (qlen-1); + } else { + TIndexOffU top = 0; + TIndexOffU bot = _nFrag; // 1 greater than largest addressable element + elt = OFF_MASK; + // Begin binary search + while(true) { + ASSERT_ONLY(TIndexOffU oldelt = elt); + elt = top + ((bot - top) >> 1); + assert_neq(oldelt, elt); // must have made progress + TIndexOffU lower = rstarts()[elt*3]; + TIndexOffU upper; + if(elt == _nFrag-1) { + upper = _eh._len; + } else { + upper = rstarts()[((elt+1)*3)]; + } + assert_gt(upper, lower); + if(lower <= off) { + if(upper > off) { // not last element, but it's within + break; // done with binary search + } else { + // 'off' belongs somewhere in the region between elt + // and bot + top = elt; } - // Add the alignment's offset into the fragment - // ('fragoff') to the fragment's offset within the text - textoff = fragoff + rstarts()[(elt*3)+2]; - assert_lt(textoff, this->plen()[tidx]); - break; // done with binary search } else { - // 'off' belongs somewhere in the region between elt - // and bot - top = elt; + // 'off' belongs somewhere in the region between top and + // elt + bot = elt; } - } else { - // 'off' belongs somewhere in the region between top and - // elt - bot = elt; + // continue with binary search + } + } + TIndexOffU lower = rstarts()[elt*3]; + TIndexOffU upper; + if(elt == _nFrag-1) { + upper = _eh._len; + } else { + upper = rstarts()[((elt+1)*3)]; + } + assert_leq(lower, off); + assert_gt(upper, off); + TIndexOffU fraglen = upper - lower; + // off is in this range; check if it falls off + if(off + qlen > upper) { + straddled = true; + if(rejectStraddle) { + // it falls off; signal no-go and return + tidx = OFF_MASK; + assert_lt(elt, _nFrag-1); + return; } - // continue with binary search } + // This is the correct text idx whether the index is + // forward or reverse + tidx = rstarts()[(elt*3)+1]; + assert_lt(tidx, this->_nPat); + assert_leq(fraglen, this->plen()[tidx]); + // it doesn't fall off; now calculate textoff. + // Initially it's the number of characters that precede + // the alignment in the fragment + TIndexOffU fragoff = off - rstarts()[(elt*3)]; + if(!this->fw_) { + fragoff = fraglen - fragoff - 1; + fragoff -= (qlen-1); + } + // Add the alignment's offset into the fragment + // ('fragoff') to the fragment's offset within the text + textoff = fragoff + rstarts()[(elt*3)+2]; + assert_lt(textoff, this->plen()[tidx]); tlen = this->plen()[tidx]; } +/** + * Build the table joinedToTextOff() uses in place of its binary search, + * from the loaded rstarts and the joined length 'len': for each bucket of + * 2^_fragShift joined offsets, the index of the fragment holding the + * bucket's first offset, then the last fragment's index. There are at + * most 16 buckets per fragment and at most 2^15 in all (hg38: 23,265 + * buckets of 128 kbp). The table is left empty (_fragTabLen = 0), and + * the binary search runs, unless the fragments start at 0, their starts + * strictly increase and the last one starts before 'len'. + */ +void Ebwt::buildFragTab(TIndexOffU len) { + _fragTab.clear(); + _fragTabLen = 0; + _fragShift = 0; + const TIndexOffU* rs = rstarts(); + if(rs == NULL || _nFrag == 0 || len == 0 || + (uint64_t)_nFrag > (uint64_t)std::numeric_limits::max()) + { + return; + } + if(rs[0] != 0 || rs[(_nFrag-1)*3] >= len) { + return; + } + for(TIndexOffU i = 1; i < _nFrag; i++) { + if(rs[i*3] <= rs[(i-1)*3]) { + return; + } + } + const uint64_t maxBuckets = std::min((uint64_t)_nFrag * 16, 1u << 15); + int shift = 0; + while((uint64_t)((len - 1) >> shift) + 1 > maxBuckets) { + shift++; + } + size_t nb = (size_t)((len - 1) >> shift) + 1; + _fragTab.resize(nb + 1); + TIndexOffU e = 0; + for(size_t b = 0; b < nb; b++) { + TIndexOffU first = (TIndexOffU)b << shift; + while(e + 1 < _nFrag && rs[(e+1)*3] <= first) { + e++; + } + _fragTab[b] = (uint32_t)e; + } + _fragTab[nb] = (uint32_t)(_nFrag - 1); + _fragShift = shift; + _fragTabLen = len; +} + /** * Walk 'steps' steps to the left and return the row arrived at. If we * walk through the dollar sign, return max value. diff --git a/bt2_idx.h b/bt2_idx.h index 4f5944bc..f4ac55a9 100644 --- a/bt2_idx.h +++ b/bt2_idx.h @@ -76,6 +76,92 @@ static const uint64_t c_table[4] = { 0x0000000000000000 }; +/** + * BT2_COUNT_WORDS: Ebwt::countUpTo() and Ebwt::countUpToEx() count the + * characters before a row in its side over every 64-bit word of the side, + * each word masked to the characters that precede the row, instead of in + * loops whose trip counts depend on the row (and are mispredicted). Used + * where popcount is an instruction and the side's bytes can be read as + * little-endian words; elsewhere (e.g. the SSE2-only x86 build) the loops + * run as before. + */ +#if defined(__POPCNT__) && defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define BT2_COUNT_WORDS 1 +#else +#define BT2_COUNT_WORDS 0 +#endif + +#if BT2_COUNT_WORDS +/// Bytes of a side that hold characters (4 per byte), and 64-bit words +static const int BT2_SIDE_BWT_BYTES = 12 * OFF_SIZE; +static const int BT2_SIDE_WORDS = BT2_SIDE_BWT_BYTES / 8; + +#define BT2_R8(x) x, x, x, x, x, x, x, x +#define BT2_R48(x) BT2_R8(x), BT2_R8(x), BT2_R8(x), BT2_R8(x), BT2_R8(x), BT2_R8(x) +#if OFF_SIZE == 4 +#define BT2_RSB(x) BT2_R48(x) +#else +#define BT2_RSB(x) BT2_R48(x), BT2_R48(x) +#endif +/** + * Row bp: BT2_SIDE_BWT_BYTES bytes 0xff, then a byte with its low bp + * bit-pairs set, then zeros. Read from byte BT2_SIDE_BWT_BYTES - by on, + * as little-endian words, it masks the first 4 * by + bp characters of a + * side; see bt2PairMasks(). + */ +alignas(64) static const uint8_t bt2PairMaskTab[4][2 * BT2_SIDE_BWT_BYTES + 8] = { + { BT2_RSB(0xff), 0x00, BT2_RSB(0x00) }, + { BT2_RSB(0xff), 0x03, BT2_RSB(0x00) }, + { BT2_RSB(0xff), 0x0f, BT2_RSB(0x00) }, + { BT2_RSB(0xff), 0x3f, BT2_RSB(0x00) } +}; +#undef BT2_R8 +#undef BT2_R48 +#undef BT2_RSB + +/** + * Masks of the first n characters of a side (0 <= n < 48 * OFF_SIZE): word + * w of the side ANDed with bt2LoadWord(bt2PairMasks(n), w). + */ +static inline const uint8_t* bt2PairMasks(int n) { + return &bt2PairMaskTab[n & 3][BT2_SIDE_BWT_BYTES - (n >> 2)]; +} + +/// Word w (little-endian) from p +static inline uint64_t bt2LoadWord(const uint8_t* p, int w) { + uint64_t x; + memcpy(&x, p + 8 * w, 8); + return x; +} + +/** + * Occurrences of each character among the first n characters of a side: + * pd = #C + #G + 2 #T (set bits), pl = #C + #T (set low bits of the + * bit-pairs), pt = #T. Every word of the side is read and masked. + */ +static inline void bt2CountWordsEx( + const uint8_t* side, + int n, + TIndexOffU& pd, + TIndexOffU& pl, + TIndexOffU& pt) +{ + const uint8_t* m = bt2PairMasks(n); + pd = pl = pt = 0; + // Two words at a time: the low-bit and T planes use only even bits, so + // a word's plane and the next word's shifted left by one share a popcount + for(int w = 0; w < BT2_SIDE_WORDS; w += 2) { + const uint64_t dx = bt2LoadWord(side, w) & bt2LoadWord(m, w); + const uint64_t dy = bt2LoadWord(side, w + 1) & bt2LoadWord(m, w + 1); + const uint64_t lx = dx & 0x5555555555555555ull; + const uint64_t ly = dy & 0x5555555555555555ull; + pd += (TIndexOffU)(__builtin_popcountll(dx) + __builtin_popcountll(dy)); + pl += (TIndexOffU)__builtin_popcountll(lx | (ly << 1)); + pt += (TIndexOffU)__builtin_popcountll((lx & (dx >> 1)) | ((ly & (dy >> 1)) << 1)); + } +} +#endif + #ifndef VMSG_NL #define VMSG_NL(...) \ if(this->verbose()) { \ @@ -299,6 +385,34 @@ static inline int64_t fileSize(const char* name) { return static_cast(f.tellg() - begin_pos); } +/** + * Prefetch the cache lines of the side at locus 'loc' of the BWT at 'eb': + * the lines SideLocus::initFromRow(row, ep, eb, true) prefetches. Lane loops + * set the loci of a round's walks with prefetch=false and then issue the + * round's prefetches together with this, so that their page walks overlap. + * A macro, so that the prefetches stay in the loop that issues them: GCC + * deletes calls to a void helper that only reads memory. + */ +#define SIDE_LOCUS_PREFETCH(loc, eb) { \ + const uint8_t* slp_ = (eb) + (loc)._sideByteOff; \ + __builtin_prefetch(slp_); \ + __builtin_prefetch(slp_ + 64); /* 64 byte cache lines */ \ + if(OFF_SIZE > 4) __builtin_prefetch(slp_ + 2*64); \ +} + +/** + * Prefetch the lines that setting loci tloc and bloc from a top/bot pair with + * prefetches would have prefetched: tloc's side, and bloc's when it is valid + * and in another side (SideLocus::initFromTopBot prefetches bot's side only + * then). + */ +#define SIDE_LOCI_PREFETCH(tloc, bloc, eb) { \ + SIDE_LOCUS_PREFETCH(tloc, eb); \ + if((bloc).valid() && (bloc)._sideByteOff != (tloc)._sideByteOff) { \ + SIDE_LOCUS_PREFETCH(bloc, eb); \ + } \ +} + /** * Encapsulates a location in the bwt text in terms of the side it * occurs in and its offset within the side. @@ -1324,7 +1438,7 @@ class Ebwt { /** * Load this Ebwt into memory by reading it in from the _in1 and - * _in2 streams. + * _in2 streams; with a loader, call loader->finish() before use. */ void loadIntoMemory( int color, @@ -1333,7 +1447,7 @@ class Ebwt { bool loadFtab, bool loadRstarts, bool loadNames, - bool verbose) + bool verbose, IndexLoader* loader = NULL) { readIntoMemory( color, // expect index to be colorspace? @@ -1345,7 +1459,7 @@ class Ebwt { NULL, // params false, // mmSweep loadNames, // loadNames - verbose); // startVerbose + verbose, loader); // startVerbose; BWT, SA sample in background } /** @@ -1357,6 +1471,8 @@ class Ebwt { _ftab.free(); _eftab.free(); _rstarts.free(); + _fragTab.clear(); + _fragTabLen = 0; _offs.free(); // might not be under control of APtrWrap _ebwt.free(); // might not be under control of APtrWrap // Keep plen; it's small and the client may want to seq it @@ -1714,7 +1830,7 @@ class Ebwt { template void buildToDisk(InorderBlockwiseSA& sa, const TStr& s, ostream& out1, ostream& out2, ostream* saOut, ostream* bwtOut); // I/O - void readIntoMemory(int color, int needEntireRev, bool loadSASamp, bool loadFtab, bool loadRstarts, bool justHeader, EbwtParams *params, bool mmSweep, bool loadNames, bool startVerbose); + void readIntoMemory(int color, int needEntireRev, bool loadSASamp, bool loadFtab, bool loadRstarts, bool justHeader, EbwtParams *params, bool mmSweep, bool loadNames, bool startVerbose, IndexLoader* loader = NULL); void writeFromMemory(bool justHeader, ostream& out1, ostream& out2) const; void writeFromMemory(bool justHeader, const string& out1, const string& out2) const; @@ -1726,6 +1842,7 @@ class Ebwt { // Searching and reporting void joinedToTextOff(TIndexOffU qlen, TIndexOffU off, TIndexOffU& tidx, TIndexOffU& textoff, TIndexOffU& tlen, bool rejectStraddle, bool& straddled) const; + void buildFragTab(TIndexOffU len); #define WITHIN_BWT_LEN(x) \ assert_leq(x[0], this->_eh._sideBwtLen); \ @@ -1931,6 +2048,23 @@ class Ebwt { * */ inline TIndexOffU countUpTo(const SideLocus& l, int c) const { // @double-check +#if BT2_COUNT_WORDS + // Every word of the side, masked to the characters before the row + const uint8_t *side = l.side(this->ebwt()); + const uint64_t pat = c_table[c]; + const int n = (int)l._charOff; + const uint8_t *m = bt2PairMasks(n); + TIndexOffU cCnt = 0; + for(int w = 0; w < BT2_SIDE_WORDS; w += 2) { + const uint64_t x = bt2LoadWord(side, w) ^ pat; + const uint64_t y = bt2LoadWord(side, w + 1) ^ pat; + // even bits only, so two words share a popcount + cCnt += (TIndexOffU)__builtin_popcountll( + (x & (x >> 1) & 0x5555555555555555ull & bt2LoadWord(m, w)) | + ((y & (y >> 1) & 0x5555555555555555ull & bt2LoadWord(m, w + 1)) << 1)); + } + return cCnt; +#else // Count occurrences of c in each 64-bit (using bit trickery); // Someday countInU64() and pop() functions should be // vectorized/SSE-ized in case that helps. @@ -1962,6 +2096,7 @@ class Ebwt { cCnt += cCntLUT_4[(int)l._bp][c][side[i]]; } return cCnt; +#endif } /** @@ -2036,6 +2171,17 @@ class Ebwt { * mostly eliminate the cache misses. */ inline void countUpToEx(const SideLocus& l, TIndexOffU* arrs) const { +#if BT2_COUNT_WORDS + // Every word of the side, masked to the characters before the row: + // #T = pt, #C = pl - pt, #G = pd - pl - pt, #A = n - pd + pt + const int n = (int)l._charOff; + TIndexOffU pd, pl, pt; + bt2CountWordsEx(l.side(this->ebwt()), n, pd, pl, pt); + arrs[0] += (TIndexOffU)n - pd + pt; + arrs[1] += pl - pt; + arrs[2] += pd - pl - pt; + arrs[3] += pt; +#else int i = 0; // Count occurrences of each nucleotide in each 64-bit word using // bit trickery; note: this seems does not seem to lend a @@ -2077,6 +2223,7 @@ class Ebwt { arrs[2] += cCntLUT_4[(int)l._bp][2][side[i]]; arrs[3] += cCntLUT_4[(int)l._bp][3][side[i]]; } +#endif } #ifndef NDEBUG @@ -2525,6 +2672,11 @@ class Ebwt { TIndexOffU _nFrag; /// number of fragments APtrWrap _plen; APtrWrap _rstarts; // starting offset of fragments / text indexes + // Fragment by joined-offset bucket, for joinedToTextOff(); see + // buildFragTab(). No table when _fragTabLen is 0. + EList _fragTab; + TIndexOffU _fragTabLen = 0; + int _fragShift = 0; // _fchr, _ftab and _eftab are expected to be relatively small // (usually < 1MB, perhaps a few MB if _fchr is particularly large // - like, say, 11). For this reason, we don't bother with writing diff --git a/bt2_io.cpp b/bt2_io.cpp index b571f934..450ec024 100644 --- a/bt2_io.cpp +++ b/bt2_io.cpp @@ -46,7 +46,7 @@ void Ebwt::readIntoMemory( EbwtParams *params, bool mmSweep, bool loadNames, - bool startVerbose) + bool startVerbose, IndexLoader* loader) { #ifdef BOWTIE_MM char *mmFile[] = { NULL, NULL }; @@ -347,14 +347,22 @@ void Ebwt::readIntoMemory( } } else { try { - _ebwt.init(new uint8_t[eh->_ebwtTotLen], eh->_ebwtTotLen, true); + _ebwt.initAligned(eh->_ebwtTotLen, MM_INDEX_ALIGN); } catch(bad_alloc& e) { cerr << "Out of memory allocating the ebwt[] array for the Bowtie index. Please try" << endl << "again on a computer with more memory." << endl; throw 1; } } - if(shmemLeader) { + const off_t ebwtPos = (loader != NULL) ? ftello(_in1) : (off_t)-1; + if(shmemLeader && !useShmem_ && !_switchEndian && loader != NULL && ebwtPos >= 0 && + loader->tryDefer(IndexLoader::LOOP_EBWT, _in1Str, (uint64_t)ebwtPos, + this->ebwt(), eh->_ebwtTotLen)) + { + // Read on a helper thread (the caller's loader->finish() + // waits for it); the rest of the file is read from here on + fseeko(_in1, eh->_ebwtTotLen, SEEK_CUR); + } else if(shmemLeader) { // Read ebwt from primary stream uint64_t bytesLeft = eh->_ebwtTotLen; char *pebwt = (char*)this->ebwt(); @@ -426,7 +434,7 @@ void Ebwt::readIntoMemory( fseeko(_in1, eh->_ftabLen*OFF_SIZE, SEEK_CUR); #endif } else { - _ftab.init(new TIndexOffU[eh->_ftabLen], eh->_ftabLen, true); + _ftab.initAligned(eh->_ftabLen, MM_INDEX_ALIGN); if(_switchEndian) { for(TIndexOffU i = 0; i < eh->_ftabLen; i++) this->ftab()[i] = readU(_in1, _switchEndian); @@ -456,7 +464,7 @@ void Ebwt::readIntoMemory( fseeko(_in1, eh->_eftabLen*OFF_SIZE, SEEK_CUR); #endif } else { - _eftab.init(new TIndexOffU[eh->_eftabLen], eh->_eftabLen, true); + _eftab.initAligned(eh->_eftabLen, MM_INDEX_ALIGN); if(_switchEndian) { for(TIndexOffU i = 0; i < eh->_eftabLen; i++) this->eftab()[i] = readU(_in1, _switchEndian); @@ -524,7 +532,7 @@ void Ebwt::readIntoMemory( if(!useShmem_) { // Allocate offs_ try { - _offs.init(new TIndexOffU[offsLenSampled], offsLenSampled, true); + _offs.initAligned(offsLenSampled, MM_INDEX_ALIGN); } catch(bad_alloc& e) { cerr << "Out of memory allocating the offs[] array for the Bowtie index." << endl << "Please try again on a computer with more memory." << endl; @@ -542,6 +550,7 @@ void Ebwt::readIntoMemory( if(_overrideOffRate < 32) { if(shmemLeader) { // Allocate offs (big allocation) + off_t offsPos = -1; if(_switchEndian || offRateDiff > 0) { assert(!_useMm); const TIndexOffU blockMaxSz = (2 * 1024 * 1024); // 2 MB block size @@ -578,6 +587,14 @@ void Ebwt::readIntoMemory( bytesRead += offsSz; fseeko(_in2, offsSz, SEEK_CUR); #endif + } else if(!useShmem_ && loader != NULL && + (offsPos = ftello(_in2)) >= 0 && + loader->tryDefer(IndexLoader::LOOP_OFFS, _in2Str, + (uint64_t)offsPos, this->offs(), offsSz)) + { + // Read on a helper thread (the caller's + // loader->finish() waits for it) + fseeko(_in2, offsSz, SEEK_CUR); } else { // Workaround for small-index mode where MM_READ may // not be able to handle read amounts greater than 2^32 @@ -614,6 +631,7 @@ void Ebwt::readIntoMemory( } this->postReadInit(*eh); // Initialize fields of Ebwt not read from file + this->buildFragTab(this->_eh._len); // for joinedToTextOff() if(_verbose || startVerbose) print(cerr, *eh); // The fact that _ebwt and friends actually point to something diff --git a/bt2_search.cpp b/bt2_search.cpp index b079d2ec..6d5b6488 100644 --- a/bt2_search.cpp +++ b/bt2_search.cpp @@ -3231,6 +3231,19 @@ static void multiseedSearchWorker(void *vp) { PerReadMetrics prm; + // Exact end-to-end sweeps of the reads of the current batch: two + // jobs (mates) per read, indexed by the read's place in the batch + std::vector sweepJobs(2 * readsPerBatch); + std::vector sweepHave(2 * readsPerBatch, 0); // job swept? + std::vector sweepList; + sweepList.reserve(2 * readsPerBatch); + // 1-mismatch end-to-end searches of the mates swept together, + // indexed as sweepJobs + std::vector mmJobs(2 * readsPerBatch); + std::vector mmHave(2 * readsPerBatch, 0); // job searched? + std::vector mmList; + mmList.reserve(2 * readsPerBatch); + // Used by thread with threadid == 1 to measure time elapsed time_t iTime = time(0); @@ -3257,6 +3270,8 @@ static void multiseedSearchWorker(void *vp) { PatternSourceReadAhead psrah(readahead_factory); PatternSourcePerThread* const ps = psrah.ptr(); bool firstPS = true; + std::fill(sweepHave.begin(), sweepHave.end(), (uint8_t)0); + std::fill(mmHave.begin(), mmHave.end(), (uint8_t)0); do { pair ret = firstPS ? psrah.readResult() : // nextReadPair was already called in the psrah constructor @@ -3509,24 +3524,135 @@ static void multiseedSearchWorker(void *vp) { // Find end-to-end exact alignments for each read if(doExactUpFront) { + // The sweeps of this read's mates and of the reads + // after it in this batch run together, the first time + // one of them is needed (exactSweepBatch); each mate + // then gets the results its own exactSweep call gives, + // at the same point as before. A sweep depends only on + // its read, its strands and the index, and the + // conditions for sweeping a mate are evaluated here + // for every read, as before. + bool sweep[2] = { false, false }; + for(size_t mate = 0; mate < (paired ? 2:1); mate++) { + sweep[mate] = !(!filt[mate] || done[mate] || msinkwrap.state().doneWithMate(mate == 0)); + } + const size_t slot = ps->bufIndex(); + bool sweepNow = false; + for(size_t mate = 0; mate < 2; mate++) { + const ExactSweepJob& job = sweepJobs[2 * slot + mate]; + if(sweep[mate] && !(sweepHave[2 * slot + mate] && + job.read == rds[mate] && job.nofw == nofw[mate] && job.norc == norc[mate])) + { + sweepNow = true; + } + } + if(sweepNow) { + const size_t nahead = ps->parseAhead(readsPerBatch); + if(sweepJobs.size() < 2 * (slot + nahead + 1)) { + sweepJobs.resize(2 * (slot + nahead + 1)); + sweepHave.resize(2 * (slot + nahead + 1), 0); + mmJobs.resize(2 * (slot + nahead + 1)); + mmHave.resize(2 * (slot + nahead + 1), 0); + } + sweepList.clear(); + for(size_t mate = 0; mate < 2; mate++) { + if(sweep[mate]) { + ExactSweepJob& job = sweepJobs[2 * slot + mate]; + job.init(rds[mate], nofw[mate], norc[mate]); + sweepList.push_back(&job); + } + } + // The reads after this one in the batch whose mates + // may be swept: in range of -s/-u, long enough, and + // not removed by --qc-filter. A mate swept here + // that turns out not to be needed is never used. + for(size_t k = 1; k <= nahead; k++) { + const size_t aslot = slot + k; + if(!ps->aheadOk(k) || sweepHave[2 * aslot] || sweepHave[2 * aslot + 1]) { + continue; + } + const TReadId ardid = ps->rdid(k); + if(ardid < skipReads || ardid >= qUpto) { + continue; + } + const Read* ards[2] = { &ps->read_a(k), &ps->read_b(k) }; + const bool apaired = !ards[1]->empty(); + const bool anofw[2] = { + apaired ? (gMate1fw ? gNofw : gNorc) : gNofw, + apaired ? (gMate2fw ? gNofw : gNorc) : gNofw }; + const bool anorc[2] = { + apaired ? (gMate1fw ? gNorc : gNofw) : gNorc, + apaired ? (gMate2fw ? gNorc : gNofw) : gNorc }; + for(size_t mate = 0; mate < (apaired ? 2:1); mate++) { + const size_t alen = ards[mate]->length(); + if(alen < 2 || alen <= (size_t)multiseedMms || + (qcFilter && ards[mate]->filter == '0')) + { + continue; + } + ExactSweepJob& job = sweepJobs[2 * aslot + mate]; + job.init(ards[mate], anofw[mate], anorc[mate]); + sweepList.push_back(&job); + sweepHave[2 * aslot + mate] = 1; + } + } + al.exactSweepBatch( + ebwtFw, // index + sc, // scoring scheme + 2, // max # edits we care about + true, // report 0mm hits + sweepList.data(), // reads to sweep + sweepList.size()); // # reads + // The offset walks of the rows of the exact hits found, together + sd.specBatch(ebwtFw, sweepList.data(), sweepList.size()); + // The 1-mismatch searches of the mates just swept, + // together. Whether a mate's search runs, and on + // which strands, depends on its edit bounds, known + // now, and on whether the mate is done by then, + // which is not; a search done here is used only if + // the 1-mismatch step below searches the same read + // on the same strands, and any other search runs + // there as before. Mates with more than one exact + // hit row are left out: extending their exact hits + // usually finishes them (always for unpaired reads + // in the timed sets), and their searches are long. + if(do1mmUpFront && !seedSumm) { + mmList.clear(); + for(size_t j = 0; j < sweepList.size(); j++) { + const ExactSweepJob& sj = *sweepList[j]; + const bool yfw = sj.mine[0] <= 1 && !sj.nofw; + const bool yrc = sj.mine[1] <= 1 && !sj.norc; + const size_t mi = (size_t)(sweepList[j] - sweepJobs.data()); + mmHave[mi] = 0; + if((!yfw && !yrc) || sj.nelt > 1) { + continue; + } + OneMmJob& mj = mmJobs[mi]; + mj.init(sj.read, !yfw, !yrc); + mmList.push_back(&mj); + mmHave[mi] = 1; + } + al.oneMmSearchBatch( + &ebwtFw, // BWT index + ebwtBw, // BWT' index + sc, // scoring scheme + false, // do exact match + true, // do 1mm + mmList.data(), // reads to search + mmList.size()); // # reads + } + } for(size_t matei = 0; matei < (paired ? 2:1); matei++) { size_t mate = matemap[matei]; - if(!filt[mate] || done[mate] || msinkwrap.state().doneWithMate(mate == 0)) { + if(!sweep[mate]) { continue; } swmSeed.exatts++; - nelt[mate] = al.exactSweep( - ebwtFw, // index - *rds[mate], // read - sc, // scoring scheme - nofw[mate], // nofw? - norc[mate], // norc? - 2, // max # edits we care about + nelt[mate] = al.exactSweepApply( + sweepJobs[2 * slot + mate], minedfw[mate], // minimum # edits for fw mate minedrc[mate], // minimum # edits for rc mate - true, // report 0mm hits - shs[mate], // put end-to-end results here - sdm); // metrics + shs[mate]); // put end-to-end results here size_t bestmin = min(minedfw[mate], minedrc[mate]); if(bestmin == 0) { sdm.bestmin0++; @@ -3537,6 +3663,7 @@ static void multiseedSearchWorker(void *vp) { sdm.bestmin2++; } } + sweepHave[2 * slot] = sweepHave[2 * slot + 1] = 0; matemap[0] = 0; matemap[1] = 1; if(nelt[0] > 0 && nelt[1] > 0 && nelt[0] > nelt[1]) { // Do the mate with fewer exact hits first @@ -3710,22 +3837,41 @@ static void multiseedSearchWorker(void *vp) { if(yfw || yrc) { // Clear out the exact hits swmSeed.mm1atts++; - al.oneMmSearch( - &ebwtFw, // BWT index - ebwtBw, // BWT' index - *rds[mate], // read - sc, // scoring scheme - minsc[mate], // minimum score - !yfw, // don't align forward read - !yrc, // don't align revcomp read - localAlign, // must be legal local alns? - false, // do exact match - true, // do 1mm - shs[mate], // seed hits (hits installed here) - sdm); // metrics + // The search may have run already, with the + // exact sweeps of this batch (above) + const size_t mi = 2 * ps->bufIndex() + mate; + if(mi < mmHave.size() && mmHave[mi] && + mmJobs[mi].read == rds[mate] && + mmJobs[mi].nofw == !yfw && mmJobs[mi].norc == !yrc) + { + al.oneMmSearchApply( + mmJobs[mi], // the search + &ebwtFw, // BWT index + sc, // scoring scheme + minsc[mate], // minimum score + localAlign, // must be legal local alns? + shs[mate]); // seed hits (hits installed here) + } else { + al.oneMmSearch( + &ebwtFw, // BWT index + ebwtBw, // BWT' index + *rds[mate], // read + sc, // scoring scheme + minsc[mate], // minimum score + !yfw, // don't align forward read + !yrc, // don't align revcomp read + localAlign, // must be legal local alns? + false, // do exact match + true, // do 1mm + shs[mate], // seed hits (hits installed here) + sdm); // metrics + } nelt[mate] = shs[mate].num1mmE2eHits(); } } + if(2 * ps->bufIndex() + 1 < mmHave.size()) { + mmHave[2 * ps->bufIndex()] = mmHave[2 * ps->bufIndex() + 1] = 0; + } // Possibly reorder the mates matemap[0] = 0; matemap[1] = 1; if(nelt[0] > 0 && nelt[1] > 0 && nelt[0] > nelt[1]) { @@ -4789,8 +4935,22 @@ static void multiseedSearch( multiseed_ebwtBw = ebwtBw; multiseed_sc = ≻ multiseed_metricsOfb = metricsOfb; + // The bit-packed reference, the forward BWT and SA sample and the + // mirror BWT are read in 64 MB pieces on up to INDEX_LOAD_THREADS helper + // threads (no more than -p) while the rest of the files is read here in + // the usual order; loader.finish() below reads pieces here too, then + // waits for the helpers, before any worker starts. Serial as before at + // -p 1, with --mm or --shmem, and when load messages or timings print. + const int INDEX_LOAD_THREADS = 4; + const bool loadInBackground = nthreads > 1 && !useMm && !useShmem && + !gVerbose && !startVerbose && !timing && !sanityCheck; + unique_ptr refs; + // Declared after refs: if loading throws, the loader's destructor + // waits for the helpers before refs and the Ebwts are freed + IndexLoader loader(loadInBackground ? std::min(nthreads, INDEX_LOAD_THREADS) : 0); + IndexLoader* bgLoader = loadInBackground ? &loader : NULL; Timer *_t = new Timer(cerr, "Time loading reference: ", timing); - unique_ptr refs( + refs.reset( new BitPairReference( adjIdxBase, false, @@ -4802,7 +4962,8 @@ static void multiseedSearch( useShmem, mmSweep, gVerbose, - startVerbose) + startVerbose, + bgLoader) ); delete _t; if(!refs->loaded()) throw 1; @@ -4840,7 +5001,8 @@ static void multiseedSearch( true, // load ftab (in forward index) true, // load rstarts (in forward index) !noRefNames, // load names? - startVerbose); + startVerbose, + bgLoader); // BWT and SA sample in the background } if(multiseedMms > 0 || do1mmUpFront) { // Load the other half of the index into memory @@ -4855,8 +5017,12 @@ static void multiseedSearch( true, // yes, need ftab in reverse index false, // don't load rstarts in reverse index !noRefNames, // load names? - startVerbose); + startVerbose, + bgLoader); // BWT in the background } + // Every index array is in memory after this; a failed read prints the + // serial load's message and throws 1 + loader.finish(); // Start the metrics thread diff --git a/ds.h b/ds.h index 60f0c217..e0027158 100644 --- a/ds.h +++ b/ds.h @@ -252,6 +252,7 @@ class APtrWrap { void init(T* p, size_t sz, bool freeable = true) { assert(p_ == NULL); p_ = p; + base_ = p; sz_ = sz; freeable_ = freeable; #ifdef USE_MEM_TALLY @@ -263,10 +264,25 @@ class APtrWrap { #endif } + /** + * Allocate sz elements with new[] and wrap them at an address that + * is a multiple of 'align' bytes, a power of two that sizeof(T) + * divides. The block is over-allocated by 'align' bytes, and free() + * deletes the block new[] returned. Throws bad_alloc as new[] does. + */ + void initAligned(size_t sz, size_t align) { + assert_eq(0, align & (align - 1)); + assert_eq(0, align % sizeof(T)); + T* base = new T[sz + align / sizeof(T)]; + const uintptr_t a = align; + init((T*)(((uintptr_t)base + a - 1) & ~(a - 1)), sz, true); + base_ = base; + } + void free() { if(p_ != NULL) { if(freeable_) { - delete[] p_; + delete[] base_; #ifdef USE_MEM_TALLY gMemTally.del(cat_, sizeof(T) * sz_); #endif @@ -281,6 +297,7 @@ class APtrWrap { private: int cat_; T *p_; + T *base_; // what new[] returned; p_ unless initAligned() moved p_ up bool freeable_; size_t sz_; }; @@ -3243,6 +3260,42 @@ class PList { return true; } + /** + * Add up to 'num' objects, all equal to 'o', to the list, and return + * how many were added. This has the effect of calling add(p, o) up to + * 'num' times and stopping at the first call that fails: pages are + * taken from the pool one at a time, when the first object that needs + * one is added, and the return value is less than 'num' only when the + * pool ran out. The list and the pool are then in the state that + * add(p, o) leaves them in when it fails. The objects are stored a + * page-sized run at a time rather than one call per object. + */ + size_t addFillUpTo(Pool& p, size_t num, const T& o) { + assert(repOk()); + if(num == 0) return 0; + if(pages_.size() == 0 && expand(p) == NULL) { + return 0; + } + size_t added = 0; + while(added < num) { + if(cur_ == PLIST_PER_PAGE) { + if(curPage_ == pages_.size()-1 && expand(p) == NULL) { + return added; + } + cur_ = 0; + curPage_++; + } + assert_lt(curPage_, pages_.size()); + assert_lt(cur_, PLIST_PER_PAGE); + size_t n = std::min(num - added, PLIST_PER_PAGE - cur_); + std::fill_n(pages_[curPage_] + cur_, n, o); + cur_ += n; + added += n; + } + assert(repOk()); + return added; + } + /** * Free all pages associated with the list. */ @@ -3298,6 +3351,19 @@ class PList { return pages_[page][elt]; } + /** + * Return a pointer to the ith element if the elements i through + * i+n-1 lie on one page, i.e. are contiguous in memory; else NULL. + */ + inline T* contiguous(size_t i, size_t n) { + assert_gt(n, 0); + assert_leq(i + n, size()); + if(i / PLIST_PER_PAGE != (i + n - 1) / PLIST_PER_PAGE) { + return NULL; + } + return &get(i); + } + /** * Get the most recently added element. */ @@ -3450,6 +3516,15 @@ class EListSlice { return list_->get(i + i_); } + /** + * Return a pointer to the slice's first element (the elements of an + * EList are contiguous), or NULL if the slice is empty. + */ + inline T* contiguous() { + if(len_ == 0) return NULL; + return &list_->get(i_); + } + /** * Return a reference to the ith element. */ @@ -3582,6 +3657,15 @@ class PListSlice { return list_->get(i+i_); } + /** + * Return a pointer to the slice's first element if its elements are + * contiguous in memory, else NULL. + */ + inline T* contiguous() { + if(len_ == 0) return NULL; + return list_->contiguous(i_, len_); + } + /** * Return a reference to the ith element. */ diff --git a/group_walk.h b/group_walk.h index 867ece68..7f5fcc05 100644 --- a/group_walk.h +++ b/group_walk.h @@ -425,6 +425,67 @@ class GWHit { size_t nrep_; }; +// kSpecMask[r]: the low r bit-pairs of a 64-bit word +static const uint64_t kSpecMask[33] = { + 0x0ull, 0x3ull, 0xfull, 0x3full, 0xffull, 0x3ffull, 0xfffull, 0x3fffull, + 0xffffull, 0x3ffffull, 0xfffffull, 0x3fffffull, 0xffffffull, 0x3ffffffull, + 0xfffffffull, 0x3fffffffull, 0xffffffffull, 0x3ffffffffull, 0xfffffffffull, + 0x3fffffffffull, 0xffffffffffull, 0x3ffffffffffull, 0xfffffffffffull, + 0x3fffffffffffull, 0xffffffffffffull, 0x3ffffffffffffull, 0xfffffffffffffull, + 0x3fffffffffffffull, 0xffffffffffffffull, 0x3ffffffffffffffull, + 0xfffffffffffffffull, 0x3fffffffffffffffull, 0xffffffffffffffffull }; + +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +// GWState::advance() steps a range of several rows a 64-bit word of the BWT +// (32 characters) at a time, reading the side's bytes as little-endian words +#define GW_RANGE_WORDS 1 +#else +#define GW_RANGE_WORDS 0 +#endif + +/** + * Bit 2k of the result is set iff the kth character (bit-pair) of 'dw' is + * 'c' (as countInU64() counts them). + */ +static inline uint64_t gwCharBits(uint64_t dw, int c) { + const uint64_t x0 = dw ^ c_table[c]; + return x0 & (x0 >> 1) & 0x5555555555555555ull; +} + +/** + * Call f(dw, rm, j, lo) for each 64-bit word of the BWT holding the last + * characters of 'n' consecutive rows, the first of which is character 'k' of + * the side at 'side': dw is the word, rm the bits (as gwCharBits() sets them) + * of the characters that belong to the rows, j the index (from 0) of the row + * of the first of them, and lo its character position in the word. Sides + * hold 48*OFF_SIZE characters, as SideLocus::initFromRow() assumes. + */ +template +static inline void gwRangeWords( + const uint8_t* side, + int k, + TIndexOffU n, + size_t sideSz, + F f) +{ + const int CPS = 48 * OFF_SIZE; + TIndexOffU j = 0; + while(j < n) { + const int lo = k & 31; + const TIndexOffU left = n - j; + const int take = (TIndexOffU)(32 - lo) < left ? 32 - lo : (int)left; + uint64_t dw; + memcpy(&dw, side + 8 * (k >> 5), 8); + f(dw, (kSpecMask[lo + take] ^ kSpecMask[lo]) & 0x5555555555555555ull, j, lo); + j += take; + k += take; + if(k == CPS) { + k = 0; + side += sideSz; + } + } +} + /** * Encapsulates the progress made along a particular path from the original * range. @@ -503,7 +564,9 @@ class GWState { TIndexOffU trimBegin = 0, trimEnd = 0; bool empty = true; // assume all resolved until proven otherwise // Commit new information, if any, to the PListSlide. Also, - // trim and check if we're done. + // trim and check if we're done. (initScan() does this loop's work + // when it can, faster.) + if(!initScan(ebwt, sa, hit, range, reportList, ret, trimBegin, trimEnd, empty, met)) for(size_t i = mapi_; i < map_.size(); i++) { bool resolved = (off(i, sa) != OFF_MASK); if(!resolved) { @@ -712,6 +775,162 @@ class GWState { return ret; } + /** + * What the loop of init() over the elements does, for when nothing is + * reported as it is resolved (reportList false), mapi_ is 0 and the + * range's offset slots are contiguous in memory: the same offsets + * resolved, forward-map entries written, counts and trims, with the + * lists' pointers and the index's fields held in locals (the build's + * -fno-strict-aliasing otherwise reloads them after every store). Only + * the rows whose offsets tryOffset() knows are visited to resolve; the + * elements are independent, so the order of the writes does not matter. + * Returns false, having done nothing, when those conditions do not hold. + */ + bool initScan( + const Ebwt& ebwt, // forward Bowtie index + SARangeWithOffs& sa, // SA range with offsets + GWHit& hit, // Corresponding hit structure + TIndexOffU range, // range being inited + bool reportList, // report resolutions, adding to 'res' list? + pair& ret, + TIndexOffU& trimBegin, + TIndexOffU& trimEnd, + bool& empty, + WalkMetrics& met) // update these metrics + { + if(reportList || mapi_ != 0 || map_.empty()) { + return false; + } + TIndexOffU *offs = sa.offs.contiguous(); + if(offs == NULL) { + return false; + } + const TIndexOffU *mp = map_.ptr(); + const size_t sz = map_.size(); + std::pair *fmap = hit.fmap.ptr(); + const TIndexOffU zOff = ebwt._zOff; + const TIndexOffU offMask = ebwt.eh()._offMask; + const int offRate = ebwt.eh()._offRate; + const TIndexOffU *saOffs = ebwt.offs(); + const TIndexOffU tp = top; + const TIndexOffU stp = (TIndexOffU)step; + TIndexOff nres = 0, nleft = 0; + TIndexOffU tb = 0, te = 0; + bool emp = true; + if(sz < 32) { + // Few rows: one pass, as init()'s loop + for(size_t i = 0; i < sz; i++) { + const TIndexOffU e = mp[i]; + assert_lt(e, sa.offs.size()); + TIndexOffU o = offs[e]; + if(o == OFF_MASK) { + // Elt not resolved yet; try to resolve it now, as + // Ebwt::tryOffset() does + const TIndexOffU bwrow = (TIndexOffU)(TIndexOff)(tp + i); + assert_eq(bwrow, ebwt.walkLeft(sa.topf + e, step)); + if(bwrow == zOff || (bwrow & offMask) == bwrow) { + const TIndexOffU toff = (bwrow == zOff) ? 0 : saOffs[bwrow >> offRate]; + assert_eq(toff, ebwt.tryOffset(bwrow)); + if(toff != OFF_MASK) { + o = toff + stp; + assert_eq(o, ebwt.getOffset(sa.topf + e)); + offs[e] = o; + nres++; + } + } + } + if(o != OFF_MASK) { + // Offset resolved + if(emp) { + tb++; + } else { + te++; + } + } else { + // Offset not yet resolved + nleft++; + te = 0; + emp = false; + fmap[e].first = range; + fmap[e].second = (TIndexOffU)i; + } + } + } else { + const uint64_t tend = (uint64_t)tp + sz; + // Resolve the unresolved elements whose rows' offsets are + // known, as Ebwt::tryOffset() finds them: the '$' row's, and + // those of the rows that are multiples of 2^offRate (sampled) + const uint64_t per = (uint64_t)1 << offRate; + assert_eq((TIndexOffU)~(per - 1), offMask); + for(uint64_t r = ((uint64_t)tp + per - 1) & ~(per - 1); r < tend; r += per) { + if((TIndexOffU)r == zOff) { + continue; + } + const TIndexOffU e = mp[r - tp]; + if(offs[e] == OFF_MASK) { + const TIndexOffU toff = saOffs[r >> offRate]; + assert_eq(toff, ebwt.tryOffset((TIndexOffU)r)); + if(toff != OFF_MASK) { + offs[e] = toff + stp; + assert_eq(offs[e], ebwt.getOffset(sa.topf + e)); + nres++; + } + } + } + if((uint64_t)zOff >= tp && (uint64_t)zOff < tend) { + const TIndexOffU e = mp[zOff - tp]; + if(offs[e] == OFF_MASK) { + offs[e] = 0 + stp; + nres++; + } + } + // Point the forward map at each element still unresolved; trim + // the resolved ones before the first and after the last of + // those. The unresolved are listed a block of rows at a time + // without branching on them (which rows are resolved follows no + // pattern, so such branches are mispredicted). + size_t first = 0, last = 0; + const size_t BLK = 256; + TIndexOffU blk[BLK]; + for(size_t i0 = 0; i0 < sz; i0 += BLK) { + const size_t i1 = (sz - i0 < BLK) ? sz : i0 + BLK; + size_t k = 0; + for(size_t i = i0; i < i1; i++) { + blk[k] = (TIndexOffU)i; + k += (offs[mp[i]] == OFF_MASK) ? 1 : 0; + } + if(k > 0) { + if(nleft == 0) { + first = blk[0]; + } + last = blk[k-1]; + for(size_t j = 0; j < k; j++) { + const TIndexOffU i = blk[j]; + assert_eq(tp + i, ebwt.walkLeft(sa.topf + mp[i], step)); + std::pair& f = fmap[mp[i]]; + f.first = range; + f.second = i; + } + nleft += (TIndexOff)k; + } + } + if(nleft > 0) { + tb = (TIndexOffU)first; + te = (TIndexOffU)(sz - 1 - last); + emp = false; + } else { + tb = (TIndexOffU)sz; + } + } + met.resolves += (uint64_t)nres; + ret.first += nres; + ret.second += nleft; + trimBegin = tb; + trimEnd = te; + empty = emp; + return true; + } + #ifndef NDEBUG /** * Check if this GWP is internally consistent. @@ -885,15 +1104,19 @@ class GWState { if(bloc.valid()) { // Still multiple elements being tracked assert_lt(top+1, bot); - TIndexOffU upto[4], in[4]; - upto[0] = in[0] = upto[1] = in[1] = - upto[2] = in[2] = upto[3] = in[3] = 0; assert_eq(bot, bloc.toBWRow()); met.bwops++; prm.nExFmops++; // Assert that there's not a dollar sign in the middle of // this range assert(bot <= ebwt._zOff || top > ebwt._zOff); +#if GW_RANGE_WORDS + (void)gws; + stepRange(ebwt, ref, sa, hit, reportList, res, st, met, ret); +#else + TIndexOffU upto[4], in[4]; + upto[0] = in[0] = upto[1] = in[1] = + upto[2] = in[2] = upto[3] = in[3] = 0; ebwt.mapLFRange(tloc, bloc, bot-top, upto, in, gws.masks); #ifndef NDEBUG for(int i = 0; i < 4; i++) { @@ -986,6 +1209,7 @@ class GWState { //assert(repOkMapRepeats()); //assert(repOkMapInclusive(hit, range)); assert_eq(bot-top, map_.size()); +#endif } else { // Down to one element assert_eq(bot, top+1); @@ -1023,6 +1247,289 @@ class GWState { return ret; } +#if GW_RANGE_WORDS + /** + * For the 'n' rows starting at the row at locus 'l': 'upto' gets, for + * each character, the row the first of them that ends in the character + * moves to with an LF step, and 'in' how many end in the character; + * what Ebwt::mapLFRange() puts in its cntsUpto and cntsIn. Counted a + * 64-bit word of the side at a time. + */ + static void countRange( + const Ebwt& ebwt, + const SideLocus& l, + TIndexOffU n, + TIndexOffU* upto, + TIndexOffU* in) + { + assert_gt(n, 0); + const EbwtParams& eh = ebwt.eh(); + const uint8_t *side = l.side(ebwt.ebwt()); + const int co = (int)l._charOff; + const size_t sideSz = (size_t)eh._sideSz; + // Occurrences of each character before the top row in its side, then + // as Ebwt::countBt2SideRange() completes them + upto[0] = upto[1] = upto[2] = upto[3] = 0; + const int NW = (48 * OFF_SIZE) / 32; + for(int w = 0; w < NW; w++) { + uint64_t dw; + memcpy(&dw, side + 8 * w, 8); + int r = co - 32 * w; + r = r < 0 ? 0 : r; + r = r > 32 ? 32 : r; + const uint64_t m = kSpecMask[r]; + for(int c = 0; c < 3; c++) { + upto[c] += (TIndexOffU)__builtin_popcountll(gwCharBits(dw, c) & m); + } + } + upto[3] = (TIndexOffU)co - upto[0] - upto[1] - upto[2]; + if(l._sideByteOff <= ebwt._zEbwtByteOff && + l._sideByteOff + l._by >= ebwt._zEbwtByteOff) + { + // '$' is represented as an 'A' (as in Ebwt::countBt2SideRange()) + if((l._sideByteOff + l._by > ebwt._zEbwtByteOff) || + (l._sideByteOff + l._by == ebwt._zEbwtByteOff && l._bp > ebwt._zEbwtBpOff)) + { + upto[0]--; + } + } + const TIndexOffU *acgt = reinterpret_cast(side + eh._sideBwtSz); + for(int c = 0; c < 4; c++) { + upto[c] += endianizeU(acgt[c], ebwt._switchEndian) + ebwt.fchr()[c]; + } + // Occurrences of each character in the range + in[0] = in[1] = in[2] = in[3] = 0; + gwRangeWords(side, co, n, sideSz, + [&](uint64_t dw, uint64_t rm, TIndexOffU, int) { + for(int c = 0; c < 3; c++) { + in[c] += (TIndexOffU)__builtin_popcountll(gwCharBits(dw, c) & rm); + } + }); + in[3] = n - in[0] - in[1] - in[2]; + } + + /** + * The step of advance() for a range of several rows (bloc valid): what + * mapLFRange() and the loops over its masks do, a word of the BWT at a + * time. The rows whose last character is the first (lowest) character + * present stay in this state; those of each other character present, in + * increasing order, go to a new state appended to 'st' and initialized + * as soon as it is made, as advance() makes them. Each list keeps the + * order of the rows. top, bot, mapi_ and map_ are left as advance() + * leaves them before its final init(). The range never holds the '$' + * row (init() trims or splits it off). + */ + template + void stepRange( + const Ebwt& ebwt, // the forward Bowtie index, for stepping left + const BitPairReference& ref, // bitpair-encoded reference + SARangeWithOffs& sa, // SA range with offsets + GWHit& hit, // the associated GWHit object + bool reportList, // if true, "report" resolved offsets immediately by adding them to 'res' list + EList* res, // EList where resolved offsets should be appended + EList& st, // EList of GWStates for range being advanced + WalkMetrics& met, + pair& ret) + { + const TIndexOffU n = bot - top; + assert_gt(n, 1); + const uint8_t *side = tloc.side(ebwt.ebwt()); + const int co = (int)tloc._charOff; + const size_t sideSz = (size_t)ebwt.eh()._sideSz; + TIndexOffU upto[4], in[4]; + countRange(ebwt, tloc, n, upto, in); +#ifndef NDEBUG + { + TIndexOffU upto2[4] = { 0, 0, 0, 0 }, in2[4] = { 0, 0, 0, 0 }; + GroupWalkState gws2(GW_CAT); + SideLocus tl = tloc, bl = bloc; + ebwt.mapLFRange(tl, bl, n, upto2, in2, gws2.masks); + for(int c = 0; c < 4; c++) { + assert_eq(upto2[c], upto[c]); + assert_eq(in2[c], in[c]); + } + } +#endif + // cf: the first character present. The rows of each later one go + // to a new state, made and initialized in character order. + int cf = 0; + while(in[cf] == 0) cf++; + const TIndexOffU *src = map_.ptr() + mapi_; + for(int c = cf + 1; c < 4; c++) { + if(in[c] == 0) continue; + // NOTE: this can cause the underlying list to be expanded + // which in turn might leave 'st' pointing to bad memory. + st.expand(); + GWState& nst = st.back(); + nst.reset(); + nst.map_.resize(in[c]); + gather(side, co, n, sideSz, c, src, nst.map_.ptr()); + met.branches++; + pair rret = + nst.init( + ebwt, // forward Bowtie index + ref, // bitpair-encodede reference + sa, // SA range with offsets + st, // EList of all GWStates associated with original range + hit, // associated GWHit object + (TIndexOffU)st.size()-1, // range offset + reportList, // if true, report hits to 'res' list + res, // report hits here if reportList is true + upto[c], // BW top of new range + upto[c] + in[c], // BW bot of new range + step+1, // # steps taken to get to this new range + met); // update these metrics + ret.first += rret.first; + ret.second += rret.second; + } + // This state keeps the rows of character cf, moved to the front of + // map_ (each moves to the same or an earlier slot) + if(in[cf] < n) { + gather(side, co, n, sideSz, cf, src, map_.ptr()); + } else if(mapi_ > 0) { + memmove(map_.ptr(), src, n * sizeof(TIndexOffU)); + } + map_.resize(in[cf]); + mapi_ = 0; + assert(top != upto[cf] || bot != upto[cf] + in[cf]); + top = upto[cf]; + bot = top + in[cf]; + } + + /** + * Write the entries of 'src' of the rows (of the n starting at character + * 'k' of 'side') whose last character is 'c', in order, to 'dst'. 'dst' + * may be 'src' or before it in the same list. + */ + static void gather( + const uint8_t* side, + int k, + TIndexOffU n, + size_t sideSz, + int c, + const TIndexOffU* src, + TIndexOffU* dst) + { + gwRangeWords(side, k, n, sideSz, + [&](uint64_t dw, uint64_t rm, TIndexOffU j, int lo) { + // One row per set bit, in order (each read before any + // write that could reach it, as dst never passes src + j) + uint64_t b = gwCharBits(dw, c) & rm; + while(b != 0) { + const int p = __builtin_ctzll(b) >> 1; + *dst++ = src[j + (TIndexOffU)(p - lo)]; + b &= b - 1; + } + }); + } +#endif + + /** + * Leave this state as init() leaves it at step 0 for a one-row range + * (map_ holding element 0) whose element is unresolved and whose row is + * neither SA-sampled nor the '$' row: init() resolves nothing, points + * the forward map at it and sets tloc at the row. The caller vouches + * that the offset slot was unresolved when the walk was set up; it is + * not read here, so a slot resolved since leaves the state as init() + * left it then. + */ + void initFreshSingle( + const Ebwt& ebwt, + GWHit& hit, + TIndexOffU row) + { + assert_eq(1, map_.size()); + assert_eq(0, mapi_); + assert(!inited_); + assert_neq(row, ebwt._zOff); + assert_neq(row, row & ebwt.eh()._offMask); + ASSERT_ONLY(inited_ = true); + ASSERT_ONLY(lastStep_ = 0); + top = row; + bot = row + 1; + step = 0; + hit.fmap[0].first = 0; + hit.fmap[0].second = 0; + tloc.initFromRow(top, ebwt.eh(), ebwt.ebwt()); + bloc.invalidate(); + } + + /** + * Leave this single-element state as advance() and init() leave it when + * its element is resolved after 'k' (>= 1) steps at row 'rowk', the last + * step having started from row 'rowkm1': top and bot trimmed past the + * resolved row, tloc at the last unresolved row. For committing a walk + * that SpecWalk took instead of advance(). + */ + void finishSingle( + const Ebwt& ebwt, + TIndexOffU k, + TIndexOffU rowk, + TIndexOffU rowkm1) + { + assert_eq(1, map_.size()); + assert_eq(0, mapi_); + assert_eq(0, step); + assert_gt(k, 0); + top = bot = rowk + 1; + step = (TIndexOff)k; + mapi_ = 1; + tloc.initFromRow(rowkm1, ebwt.eh(), ebwt.ebwt(), false); + bloc.invalidate(); + ASSERT_ONLY(lastStep_ = step); + } + + /** + * Do what 'k' (>= 1) calls of advance() do to this state, which tracks + * one row (bloc invalid) whose element is unresolved, when the walk from + * its row takes k LF steps to 'rowk', the first row whose offset + * tryOffset() knows, the last step starting from 'rowkm1': the same + * counts (bwops, nExFmops, resolves), the offset, the forward-map entry + * (each step but the last points it at slot 0), and the state as the + * last step's init() leaves it. The walk is SpecWalk's; nothing else + * writes the element's offset slot between those k steps. + */ + void commitWalk( + const Ebwt& ebwt, + SARangeWithOffs& sa, + GWHit& hit, + TIndexOffU range, + TIndexOffU k, + TIndexOffU rowk, + TIndexOffU rowkm1, + WalkMetrics& met, + PerReadMetrics& prm) + { + assert(!bloc.valid()); + assert_eq(1, size()); + assert_gt(k, 0); + assert_eq(top, tloc.toBWRow()); + assert_eq(rowk, ebwt.walkLeft(top, k)); + const TIndexOffU e = map_[mapi_]; + assert_eq(OFF_MASK, sa.offs[e]); + // As advance() moves the element to the front + map_[0] = e; + map_.resize(1); + met.bwops += k; + prm.nExFmops += k; + if(k > 1) { + // init() of each step that leaves the element unresolved + hit.fmap[e].first = range; + hit.fmap[e].second = 0; + tloc.initFromRow(rowkm1, ebwt.eh(), ebwt.ebwt(), false); + } + step += (TIndexOff)k; + const TIndexOffU toff = ebwt.tryOffset(rowk); + assert_neq(OFF_MASK, toff); + met.resolves++; + sa.offs[e] = toff + (TIndexOffU)step; + assert_eq(sa.offs[e], ebwt.getOffset(sa.topf + e)); + // The last step's init() trims the resolved row + mapi_ = 1; + top = bot = rowk + 1; + ASSERT_ONLY(lastStep_ = step); + } + /** * Clear all state in preparation for the next walk. */ @@ -1082,6 +1589,8 @@ class GWState { TIndexOffU mapi_; // first untrimmed element of map }; +class SpecWalk; + template class GroupWalk2S { public: @@ -1139,6 +1648,33 @@ class GroupWalk2S { assert(hit_.repOk(sa)); } + /** + * Do what init() does for a one-element range whose row is neither + * SA-sampled nor the '$' row and whose offset slot was unresolved at + * the time an unbatched walk would call init(): such an init() reads the + * slot, resolves nothing and counts nothing. The slot is not read, so + * the walk is left as init() left it then even if another walk sharing + * the slot has resolved it since (advanceElement() then only reports + * it, as it would have). + */ + void initFreshSingle( + const Ebwt& ebwtFw, // forward Bowtie index for walking left + SARangeWithOffs& sa) // SA range with offsets, one element + { + assert_eq(1, sa.size()); + reset(); +#ifndef NDEBUG + inited_ = true; +#endif + hit_.init(sa, 0, false, 0); + st_.resize(1); + st_.back().reset(); + st_.back().initMap(1); + st_.ensure(4); + st_.back().initFreshSingle(ebwtFw, hit_, sa.topf); + elt_ += 1; + } + // // ELEMENT-BASED // @@ -1214,6 +1750,87 @@ class GroupWalk2S { return true; } + /** + * Do what advanceElement() does, but when the state holding the element + * tracks one row, take its remaining LF steps from 'spec' (starting the + * walk there if it is not already finished) and commit them at once. + * After each step of a state of several rows, the walks of the + * one-row states the step leaves are started in 'spec' while it has + * free lanes, so that they proceed alongside the walks needed first. + * Defined after SpecWalk. + */ + bool advanceElementSpec( + TIndexOffU elt, // element within the range + const Ebwt& ebwtFw, // forward Bowtie index for walking left + const BitPairReference& ref, // bitpair-encoded reference + SARangeWithOffs& sa, // SA range with offsets + GroupWalkState& gws, // GroupWalk state; scratch space + WalkResult& res, // put the result here + WalkMetrics& met, // metrics + PerReadMetrics& prm, // per-read metrics + SpecWalk& spec); // single-row walks + + /** + * Return true iff this walk has a single element and advanceElement() + * would step to resolve it: nothing reported yet, no step taken, and + * its offset not resolved, neither at init() nor since (by a walk that + * shares the offset's slot). Such an element's walk depends only on + * its BW row, sa.topf. + */ + bool freshSingle(const SARangeWithOffs& sa) const { + return elt_ == 1 && rep_ == 0 && st_.size() == 1 && + st_[0].step == 0 && st_[0].size() == 1 && + sa.offs.get(0) == OFF_MASK; + } + + /** + * Do what advanceElement(0, ...) does for a freshSingle() walk, given the + * walk's outcome as SpecWalk computed it: 'k' LF steps from sa.topf to + * 'rowk', the first row with a known offset, the last step starting from + * 'rowkm1'. Every count advanceElement() keeps (bwops, resolves, + * reports, nExFmops), the offset, the reported flag and the state are + * left as advanceElement() leaves them. + */ + void commitSingle( + const Ebwt& ebwtFw, // forward Bowtie index + SARangeWithOffs& sa, // SA range with offsets + TIndexOffU k, // # LF steps + TIndexOffU rowk, // row reached, offset known + TIndexOffU rowkm1, // row the last step left + WalkResult& res, // put the result here + WalkMetrics& met, // metrics + PerReadMetrics& prm) // per-read metrics + { + assert(inited_); + assert(freshSingle(sa)); + assert_eq(st_[0].top, sa.topf); + assert_gt(k, 0); + assert_eq(rowk, ebwtFw.walkLeft(sa.topf, k)); + TIndexOffU toff = ebwtFw.tryOffset(rowk); + assert_neq(OFF_MASK, toff); + met.bwops += k; + prm.nExFmops += k; + met.resolves++; + toff += k; + assert_eq(toff, ebwtFw.getOffset(sa.topf)); + sa.offs[0] = toff; + st_[0].finishSingle(ebwtFw, k, rowk, rowkm1); + // As advanceElement() does once the element is resolved + if(!hit_.reported(0)) { + hit_.setReported(0); + } + met.reports++; + res.init( + 0, // seed offset + false, // orientation + 0, // range + 0, // element + sa.topf, // bw row + (TIndexOffU)sa.len, // length of hit + sa.offs[0]); // resolved text offset + rep_++; + } + /** * Return true iff all elements have been resolved and reported. */ @@ -1255,6 +1872,11 @@ class GroupWalk2S { */ size_t numElts() const { return elt_; } + /** + * Return the number of BW elements reported so far. + */ + size_t numReported() const { return rep_; } + /** * Return the size occupied by this GroupWalk and all its constituent * objects. @@ -1288,4 +1910,327 @@ class GroupWalk2S { GWHit hit_; }; +/** + * Walks to resolve the offsets of single BW rows, several stepped in turn so + * that their memory waits overlap. A walk starts at a row and takes LF steps + * until it reaches a row whose offset is known (sampled, or the '$' row), + * exactly as GroupWalk2S::advanceElement() walks a single-element range. It + * reads only the index and writes only this object: nothing is counted or + * resolved here. A finished walk is kept, keyed by its starting row, until + * GroupWalk2S::commitSingle() applies it where an unbatched walk would have + * walked; walks that are never needed are dropped. The outcome of a walk + * depends only on its starting row and the index. + */ +class SpecWalk { + +public: + + static const int LANES = 16; // walks stepped in turn + static const int TBITS = 10; // finished walks kept: 2^TBITS, direct-mapped + + SpecWalk() : ebwt_(NULL), nlive_(0) { + for(int i = 0; i < LANES; i++) live_[i] = false; + for(int i = 0; i < (1 << TBITS); i++) key_[i] = OFF_MASK; + } + + /** + * Drop the walks in progress; finished ones are kept. + */ + void dropLive() { + for(int i = 0; i < LANES; i++) live_[i] = false; + nlive_ = 0; + } + + /** + * Find the finished walk from row 'row0'; return false if there is none. + */ + bool lookup( + const Ebwt& ebwt, + TIndexOffU row0, + TIndexOffU& k, + TIndexOffU& rowk, + TIndexOffU& rowkm1) const + { + if(&ebwt != ebwt_) return false; + const size_t h = slot(row0); + if(key_[h] != row0) return false; + k = k_[h]; rowk = rowk_[h]; rowkm1 = rowkm1_[h]; + return true; + } + + /** + * Return the number of walks in progress. + */ + int live() const { return nlive_; } + + /** + * Return true iff a walk from 'row0' is finished or in progress. + */ + bool have(const Ebwt& ebwt, TIndexOffU row0) const { + if(&ebwt != ebwt_) return false; + if(key_[slot(row0)] == row0) return true; + return lane(row0) >= 0; + } + + /** + * Start a walk from 'row0', a row whose offset is not known. If every + * lane is busy, step the walks in progress until one finishes. + */ + void start(const Ebwt& ebwt, TIndexOffU row0) { + if(&ebwt != ebwt_) { + // Results belong to one index + for(int i = 0; i < (1 << TBITS); i++) key_[i] = OFF_MASK; + dropLive(); + ebwt_ = &ebwt; + } + assert(row0 != ebwt._zOff); + assert_neq((row0 & ebwt.eh()._offMask), row0); + while(nlive_ == LANES) { + round(ebwt); + } + for(int i = 0; i < LANES; i++) { + if(!live_[i]) { + Lane& x = lanes_[i]; + x.row0 = x.row = x.prev = row0; + x.k = 0; + x.loc.initFromRow(row0, ebwt.eh(), ebwt.ebwt()); + live_[i] = true; + nlive_++; + return; + } + } + assert(false); + } + + /** + * Step the walks in progress until the walk from 'row0' finishes. + */ + void finish(const Ebwt& ebwt, TIndexOffU row0) { + const int l = lane(row0); + if(l < 0) return; + while(live_[l]) { + round(ebwt); + } + } + + /** + * Step the walks in progress until all have finished. + */ + void finishAll(const Ebwt& ebwt) { + while(nlive_ > 0) { + round(ebwt); + } + } + +protected: + + struct Lane { + TIndexOffU row0; // starting row + TIndexOffU row; // current row + TIndexOffU prev; // row the last step started from + TIndexOffU k; // # steps taken + SideLocus loc; // locus of 'row'; set (and prefetched) as the row is reached + }; + + size_t slot(TIndexOffU row0) const { + return (size_t)(((uint64_t)row0 * 0x9E3779B97F4A7C15ull) >> (64 - TBITS)); + } + + /** + * LF(row) for the row at locus 'l' (not the '$' row): what + * Ebwt::mapLF1(row, l) sets the row to. The occurrences of the row's + * character before it in its side are counted over every 64-bit word of + * the side with masks, rather than by Ebwt::countUpTo()'s loops, whose + * trip counts depend on the row and are mispredicted. + */ + static TIndexOffU lf(const Ebwt& ebwt, const SideLocus& l) { +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + const uint8_t *side = l.side(ebwt.ebwt()); + const int c = unpack_2b_from_8b(side[l._by], l._bp); + const uint64_t pat = c_table[c]; + const int n = (int)l._charOff; // characters before the row in its side + const int NW = (48 * OFF_SIZE) / 32; // 32 characters per word + TIndexOffU cnt = 0; + for(int w = 0; w < NW; w++) { + uint64_t dw; + memcpy(&dw, side + 8 * w, 8); + const uint64_t x0 = dw ^ pat; + const uint64_t x3 = x0 & (x0 >> 1) & 0x5555555555555555ull; + int r = n - 32 * w; + r = r < 0 ? 0 : r; + r = r > 32 ? 32 : r; + cnt += (TIndexOffU)__builtin_popcountll(x3 & kSpecMask[r]); + } + assert_eq(cnt, ebwt.countUpTo(l, c)); + if(c == 0 && l._sideByteOff <= ebwt._zEbwtByteOff && + l._sideByteOff + l._by >= ebwt._zEbwtByteOff) + { + // '$' is represented as an 'A' (as in Ebwt::countBt2Side()) + if((l._sideByteOff + l._by > ebwt._zEbwtByteOff) || + (l._sideByteOff + l._by == ebwt._zEbwtByteOff && l._bp > ebwt._zEbwtBpOff)) + { + cnt--; + } + } + const TIndexOffU *acgt = reinterpret_cast(side + ebwt.eh()._sideBwtSz); + const TIndexOffU ret = endianizeU(acgt[c], ebwt._switchEndian) + cnt + ebwt.fchr()[c]; + assert_eq(ret, ebwt.countBt2Side(l, c)); + return ret; +#else + TIndexOffU row = l.toBWRow(); + ebwt.mapLF1(row, l); + return row; +#endif + } + + int lane(TIndexOffU row0) const { + for(int i = 0; i < LANES; i++) { + if(live_[i] && lanes_[i].row0 == row0) return i; + } + return -1; + } + + /** + * One LF step of every walk in progress. A walk ends at the first row + * whose offset Ebwt::tryOffset() knows, as GWState::init() checks. + */ + void round(const Ebwt& ebwt) { + const EbwtParams& eh = ebwt.eh(); + for(int i = 0; i < LANES; i++) { + if(!live_[i]) continue; + Lane& x = lanes_[i]; + x.prev = x.row; + x.row = lf(ebwt, x.loc); + x.k++; + if(x.row == ebwt._zOff || (x.row & eh._offMask) == x.row) { + if(x.row != ebwt._zOff) { + __builtin_prefetch(ebwt.offs() + (x.row >> eh._offRate)); + } + const size_t h = slot(x.row0); + key_[h] = x.row0; + k_[h] = x.k; + rowk_[h] = x.row; + rowkm1_[h] = x.prev; + live_[i] = false; + nlive_--; + } else { + x.loc.initFromRow(x.row, eh, ebwt.ebwt()); + } + } + } + + const Ebwt* ebwt_; // index the walks belong to + Lane lanes_[LANES]; + bool live_[LANES]; + int nlive_; + TIndexOffU key_[1 << TBITS]; // starting row of a finished walk, OFF_MASK if none + TIndexOffU k_[1 << TBITS]; // # steps + TIndexOffU rowk_[1 << TBITS]; // row reached + TIndexOffU rowkm1_[1 << TBITS]; // row the last step started from +}; + +template +bool GroupWalk2S::advanceElementSpec( + TIndexOffU elt, // element within the range + const Ebwt& ebwtFw, // forward Bowtie index for walking left + const BitPairReference& ref, // bitpair-encoded reference + SARangeWithOffs& sa, // SA range with offsets + GroupWalkState& gws, // GroupWalk state; scratch space + WalkResult& res, // put the result here + WalkMetrics& met, // metrics + PerReadMetrics& prm, // per-read metrics + SpecWalk& spec) // single-row walks +{ + assert(inited_); + assert(!done()); + assert(hit_.repOk(sa)); + assert_lt(elt, sa.size()); // elt must fall within range + // Until we've resolved our element of interest... + while(sa.offs[elt] == OFF_MASK) { + // Get the GWState that contains our element of interest + size_t range = hit_.fmap[elt].first; + st_.ensure(4); + if(!st_[range].bloc.valid()) { + // One row left, the element's: its remaining steps are the walk + // from the state's row, or the rest of the walk from the + // element's own row (resolveElement() walks the rows of small + // ranges) + GWState& st = st_[range]; + assert_eq(elt, st.map(st.mapi())); + TIndexOffU k = 0, rowk = 0, rowkm1 = 0; + bool have = spec.lookup(ebwtFw, st.top, k, rowk, rowkm1); + if(!have && + spec.lookup(ebwtFw, sa.topf + elt, k, rowk, rowkm1) && + k > (TIndexOffU)st.step) + { + k -= (TIndexOffU)st.step; + have = true; + } + if(!have) { + if(!spec.have(ebwtFw, st.top)) { + spec.start(ebwtFw, st.top); + } + spec.finish(ebwtFw, st.top); + have = spec.lookup(ebwtFw, st.top, k, rowk, rowkm1); + } + if(have) { + st.commitWalk(ebwtFw, sa, hit_, (TIndexOffU)range, k, rowk, rowkm1, met, prm); + continue; + } + // The finished walk was displaced by another's: step as before + } + const size_t n0 = st_.size(); + st_[range].advance( + ebwtFw, + ref, + sa, + hit_, + (TIndexOffU)range, + false, + NULL, + st_, + gws, + met, + prm); + // Start the walks of the one-row states this step left (this one + // and those it made) while lanes are free. Most are never needed + // (the extension usually stops after a few elements of a large + // range), so none is started by stepping the others to free a lane: + // they only progress alongside walks that are needed. + for(size_t j = n0; j <= st_.size() && spec.live() < SpecWalk::LANES; j++) { + const GWState& s = st_[j == st_.size() ? range : j]; + if(s.done() || s.bloc.valid()) { + continue; + } + const TIndexOffU e = s.map(s.mapi()); + TIndexOffU k = 0, rowk = 0, rowkm1 = 0; + if(sa.offs[e] != OFF_MASK || + spec.have(ebwtFw, s.top) || + spec.lookup(ebwtFw, sa.topf + e, k, rowk, rowkm1)) + { + continue; + } + spec.start(ebwtFw, s.top); + } + assert(sa.offs[elt] != OFF_MASK || + !st_[hit_.fmap[elt].first].doneResolving(sa)); + } + assert_neq(OFF_MASK, sa.offs[elt]); + // Report it! + if(!hit_.reported(elt)) { + hit_.setReported(elt); + } + met.reports++; + res.init( + 0, // seed offset + false, // orientation + 0, // range + elt, // element + sa.topf + elt, // bw row + (TIndexOffU)sa.len, // length of hit + sa.offs[elt]); // resolved text offset + rep_++; + return true; +} + #endif /*GROUP_WALK_H_*/ diff --git a/index_load.h b/index_load.h new file mode 100644 index 00000000..0d0bf620 --- /dev/null +++ b/index_load.h @@ -0,0 +1,398 @@ +/* + * Copyright 2011, Ben Langmead + * + * This file is part of Bowtie 2. + * + * Bowtie 2 is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Bowtie 2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Bowtie 2. If not, see . + */ + +#ifndef INDEX_LOAD_H_ +#define INDEX_LOAD_H_ + +/** + * index_load.h: + * + * Reads the large arrays of an index (forward and mirror BWT, SA sample, + * bit-packed reference) on helper threads while the main thread goes on + * reading the rest of the index files in the usual order. Reading the + * arrays one after another is mostly the kernel copying from the page + * cache, one file at a time; on several threads the copies overlap. + * + * Each deferred array is read in pieces of CHUNK bytes, queued in file + * order. The helpers take pieces from the queue, and so does the main + * thread once it reaches finish(), so the threads stay busy until the + * last piece whatever the arrays' sizes. A piece is read through its own + * stream, opened when the piece is taken and closed when it is read, so + * only a few streams are open at once; the first piece uses the array's + * own stream, opened by tryDefer(). + * + * The caller allocates each array exactly as a serial load does, then + * calls tryDefer() where the serial load would have read it. tryDefer() + * hands the read over only when a stream on the file can be opened and + * positioned and the file holds the whole array; otherwise it returns + * false and the caller reads the array itself, so a short or unreadable + * file fails with the serial load's message at the serial load's point. + * A piece whose own stream cannot be opened later is read in finish() + * through the array's stream. finish() waits for every read; if one + * failed, it prints the message the serial load would have printed for + * the first failed array (its byte counts rebuilt from the first failed + * piece) and throws 1. Nothing may read a deferred array before finish(). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "assert_helpers.h" + +class IndexLoader { +public: + + /** + * How the serial load reads an array, which decides the message + * printed when the read fails. + */ + enum Kind { + LOOP_EBWT = 0, // Ebwt::readIntoMemory's loop over _ebwt[] + LOOP_OFFS, // Ebwt::readIntoMemory's loop over _offs[] + ONE_REF // BitPairReference's single fread of the .4 file + }; + + /** + * Bytes per piece. 64 MB splits an hg38 index into 56 pieces, so the + * threads finish within about one piece's read of each other. + */ + static const uint64_t CHUNK = (uint64_t)64 << 20; + + /** + * maxThreads: the most helper threads at once (the main thread reads + * too, in finish()); 0 reads nothing in the background (tryDefer() + * always returns false). + */ + explicit IndexLoader(int maxThreads) : + maxThreads_(maxThreads < 0 ? 0 : maxThreads), + live_(0), + next_(0) + { } + + /** + * Waits for every read still in flight, so that no helper writes + * into an array after an exception has freed it. Never throws. + */ + ~IndexLoader() { + joinAll(); + for(size_t i = 0; i < arrays_.size(); i++) { + if(arrays_[i]->file != NULL) fclose(arrays_[i]->file); + delete arrays_[i]; + } + for(size_t i = 0; i < pieces_.size(); i++) { + delete pieces_[i]; + } + } + + bool enabled() const { return maxThreads_ > 0; } + + /** + * Reads len bytes of the file at path, starting at byte off, into + * dest on the helper threads. Returns false, and changes nothing the + * caller can see (errno included), when the read cannot be handed + * off: the loader is disabled, a stream cannot be opened or + * positioned, the file is shorter than off + len, or no helper + * thread can be started. + */ + bool tryDefer(Kind kind, const std::string& path, uint64_t off, void *dest, uint64_t len) { + if(!enabled()) return false; + const int savedErrno = errno; + FILE *f = fopen(path.c_str(), "rb"); + bool ok = f != NULL; + if(ok) ok = fseeko(f, 0, SEEK_END) == 0; + if(ok) { + const off_t end = ftello(f); + ok = end >= 0 && (uint64_t)end >= off && (uint64_t)end - off >= len; + } + if(ok) ok = fseeko(f, (off_t)off, SEEK_SET) == 0; + if(!ok) { + if(f != NULL) fclose(f); + errno = savedErrno; + return false; + } + const uint64_t chunk = CHUNK; + // One piece for an empty array, so that it is still checked + const uint64_t npieces = (len == 0) ? 1 : (len + chunk - 1) / chunk; + std::unique_lock lk(mu_); + const size_t firstPiece = pieces_.size(); + Array *a = NULL; + try { + a = new Array(); + a->kind = kind; + a->path = path; + a->off = off; + a->len = len; + a->first = firstPiece; + a->n = (size_t)npieces; + arrays_.reserve(arrays_.size() + 1); + pieces_.reserve(pieces_.size() + (size_t)npieces); + for(uint64_t k = 0; k < npieces; k++) { + Piece *p = new Piece(); + p->array = arrays_.size(); + p->off = k * chunk; + p->dest = (char *)dest + p->off; + p->len = (len - p->off < chunk) ? (len - p->off) : chunk; + pieces_.push_back(p); // room reserved above + } + a->file = f; // from here on the array closes it + arrays_.push_back(a); // room reserved above + } catch(...) { + dropFrom(firstPiece); + delete a; + fclose(f); + errno = savedErrno; + return false; + } + // Start helpers for the new pieces, up to maxThreads_ in all + while(live_ < maxThreads_ && (size_t)live_ < pieces_.size() - next_) { + bool started = false; + try { + // Room first, so that the started thread is never + // dropped by a failed push_back + threads_.reserve(threads_.size() + 1); + threads_.push_back(std::thread(&IndexLoader::worker, this)); + started = true; + } catch(...) { } + if(!started) break; + live_++; // the new helper waits for mu_ before looking + } + if(live_ == 0) { + // No helper to take the pieces: the caller reads the array + dropFrom(firstPiece); + arrays_.pop_back(); + delete a; // closes nothing: the file is closed here + fclose(f); + errno = savedErrno; + return false; + } + lk.unlock(); + errno = savedErrno; + return true; + } + + /** + * Reads pieces on the main thread until none is left, waits for the + * helpers, then reads any piece whose own stream could not be opened + * through its array's stream. If any read failed, prints the serial + * load's message for the first failed array (in the order they were + * deferred, which is the serial load's order) and throws 1. + */ + void finish() { + const int savedErrno = errno; + take(false); + joinAll(); + for(size_t i = 0; i < pieces_.size(); i++) { + Piece& p = *pieces_[i]; + if(!p.unopened) continue; + Array& a = *arrays_[p.array]; + if(fseeko(a.file, (off_t)(a.off + p.off), SEEK_SET) != 0) { + p.thrown = true; + p.failed = true; + continue; + } + run(p, a.kind, a.file, i + 1 == a.first + a.n); + } + for(size_t i = 0; i < arrays_.size(); i++) { + fclose(arrays_[i]->file); + arrays_[i]->file = NULL; + } + errno = savedErrno; + for(size_t i = 0; i < arrays_.size(); i++) { + const Array& a = *arrays_[i]; + for(size_t k = a.first; k < a.first + a.n; k++) { + if(!pieces_[k]->failed) continue; + std::cerr << message(a, *pieces_[k]) << std::endl; + throw 1; + } + } + } + +private: + + /** One deferred array. */ + struct Array { + Array() : kind(LOOP_EBWT), file(NULL), off(0), len(0), first(0), n(0) { } + Kind kind; + std::string path; + FILE *file; // opened by tryDefer(), positioned at off + uint64_t off; // the array's offset in the file + uint64_t len; + size_t first; // its first piece in pieces_ + size_t n; // number of pieces + }; + + /** One piece of a deferred array. */ + struct Piece { + Piece() : array(0), dest(NULL), off(0), len(0), unopened(false), + failed(false), ioErr(false), thrown(false), before(0), + lastRead(0) { } + size_t array; + char *dest; + uint64_t off; // offset of the piece in its array + uint64_t len; + bool unopened; // its own stream could not be opened + bool failed; + bool ioErr; // failed on ferror() (else at end of file) + bool thrown; // failed on an exception + uint64_t before; // bytes read by calls before the failing one + uint64_t lastRead; // what the failing fread() returned + }; + + /** + * The message the serial load prints for array a when piece p is the + * first piece of a that failed. Every earlier piece read in full, so + * a serial read gets to p's offset and then sees what p saw: its + * first fread() returns everything up to the error or the end of the + * file, with the whole array outstanding. + */ + static std::string message(const Array& a, const Piece& p) { + if(p.thrown) { + return "Error reading index file " + a.path; + } + std::ostringstream os; + if(a.kind == ONE_REF) { + os << "Only read " << (p.off + p.before + p.lastRead) + << " bytes (out of " << a.len + << ") from reference index file " << a.path.c_str(); + return os.str(); + } + const char *what = (a.kind == LOOP_EBWT) ? + "Error reading _ebwt[] array: " : + "Error reading block of _offs[] array: "; + if(!p.ioErr) { + return std::string(what) + "no more data"; + } + // is_fread_err() sets gLastIOErrMsg to this text + if(p.before == 0) { + os << what << (p.off + p.lastRead) << ", " << a.len << "Error Reading File!"; + } else { + os << what << p.lastRead << ", " << (a.len - p.off - p.before) << "Error Reading File!"; + } + return os.str(); + } + + /** + * Reads one piece from file, positioned at the piece, with the serial + * loop's failure tests, keeping what message() needs instead of + * printing. last: the piece ends its array. + */ + static void run(Piece& p, Kind kind, FILE *file, bool last) { + uint64_t bytesLeft = p.len; + char *dest = p.dest; + while(bytesLeft > 0) { + size_t r = fread((void *)dest, 1, bytesLeft, file); + if(ferror(file)) { + p.lastRead = r; + p.ioErr = true; + p.failed = true; + return; + } else if(r == 0) { + p.failed = true; + return; + } + dest += r; + bytesLeft -= r; + p.before += r; + } +#ifndef NDEBUG + if(last && kind == ONE_REF) { + // Make sure there's no more + char c; + size_t ret = fread(&c, 1, 1, file); + assert_eq(0, ret); // should have failed + } +#endif + } + + /** + * Reads piece p of array a: the first piece through the array's + * stream, any other through a stream of its own. A piece whose stream + * cannot be opened or positioned is left for finish(). + */ + static void runPiece(Piece& p, const Array& a, bool first, bool last) { + if(first) { + run(p, a.kind, a.file, last); + return; + } + FILE *f = fopen(a.path.c_str(), "rb"); + if(f == NULL || fseeko(f, (off_t)(a.off + p.off), SEEK_SET) != 0) { + if(f != NULL) fclose(f); + p.unopened = true; + return; + } + run(p, a.kind, f, last); + fclose(f); + } + + /** + * Takes pieces from the queue and reads them until none is left; a + * helper then counts itself out. + */ + void take(bool helper) { + std::unique_lock lk(mu_); + while(next_ < pieces_.size()) { + const size_t i = next_++; + Piece *p = pieces_[i]; + const Array *a = arrays_[p->array]; + const bool first = (i == a->first); + const bool last = (i + 1 == a->first + a->n); + lk.unlock(); + try { + runPiece(*p, *a, first, last); + } catch(...) { + p->thrown = true; + p->failed = true; + } + lk.lock(); + } + if(helper) live_--; + } + + void worker() { take(true); } + + /** Removes the pieces from index first on, none of them taken. */ + void dropFrom(size_t first) { + while(pieces_.size() > first) { + delete pieces_.back(); + pieces_.pop_back(); + } + } + + void joinAll() { + for(size_t i = 0; i < threads_.size(); i++) { + if(threads_[i].joinable()) threads_[i].join(); + } + } + + int maxThreads_; + int live_; // helpers that will still look at the queue + size_t next_; // first piece not yet taken + std::mutex mu_; + std::vector arrays_; + std::vector pieces_; + std::vector threads_; +}; + +#endif /* INDEX_LOAD_H_ */ diff --git a/mm.h b/mm.h index c8afd399..bb2a0b30 100644 --- a/mm.h +++ b/mm.h @@ -29,7 +29,31 @@ * and where there isn't POSIX I/O, */ +#include +#include + #define MM_READ(file, dest, sz) fread(dest, 1, sz, file) #define MM_IS_IO_ERR(file_hd, ret, count) is_fread_err(file_hd, ret, count) +/** + * Alignment, in bytes, of the large index arrays read from disk into the + * heap (BWT, SA sample, ftab, eftab, bit-packed reference). A BWT side + * is 64 bytes (128 for 64-bit offsets) and is read at a random place, so + * a side that starts on a cache line touches one line (two) instead of + * two (three). glibc serves large new[] blocks with mmap and returns an + * address 16 bytes into a page, which is never a line boundary. Only + * where the arrays start in memory changes; their bytes are the file's. + */ +static const size_t MM_INDEX_ALIGN = 64; + +/** + * Round p up to the next multiple of MM_INDEX_ALIGN bytes. The block p + * points into must extend MM_INDEX_ALIGN bytes past what is used. + */ +template +static inline T* mmAlignUp(T* p) { + const uintptr_t a = MM_INDEX_ALIGN; + return (T*)(((uintptr_t)p + a - 1) & ~(a - 1)); +} + #endif /* MM_H_ */ diff --git a/outq.cpp b/outq.cpp index d93dabf2..0ad543ea 100644 --- a/outq.cpp +++ b/outq.cpp @@ -26,8 +26,10 @@ * the read with the given id. */ void OutputQueue::beginReadImpl(TReadId rdid, size_t threadId) { - nstarted_.fetch_add(1); - if(reorder_) { + if(!reorder_) { + perThread_[threadId].nstarted++; + } else { + nstarted_++; // under the lock (reorder_ implies threadSafe_) assert_geq(rdid, cur_); assert_eq(lines_.size(), finished_.size()); assert_eq(lines_.size(), started_.size()); @@ -58,7 +60,7 @@ void OutputQueue::beginRead(TReadId rdid, size_t threadId) { /** * Writer is finished writing to */ -void OutputQueue::finishReadImpl(const BTString& rec, TReadId rdid, size_t threadId) { +void OutputQueue::finishReadImpl(BTString& rec, TReadId rdid, size_t threadId) { if(reorder_) { assert_geq(rdid, cur_); assert_eq(lines_.size(), finished_.size()); @@ -72,20 +74,26 @@ void OutputQueue::finishReadImpl(const BTString& rec, TReadId rdid, size_t threa flush(false, false); // don't force; already have lock } else { // obuf_ is the OutFileBuf for the output file + PerThread& t = perThread_[threadId]; int i = 0; for(i=0; i < perThreadBufSize_; i++) { - obuf_.writeString(perThreadBuf[threadId][i]); + obuf_.writeString(t.buf[i]); //TODO: turn these into atomics nfinished_++; nflushed_++; } - perThreadCounter[threadId] = 0; + t.count = 0; } } -void OutputQueue::finishRead(const BTString& rec, TReadId rdid, size_t threadId) { - if(reorder_ || perThreadCounter[threadId] >= perThreadBufSize_) { +/** + * Without reorder_, the record is swapped into this thread's buffer rather + * than copied: rec then holds an already-written record's buffer, which the + * caller clears before its next use. + */ +void OutputQueue::finishRead(BTString& rec, TReadId rdid, size_t threadId) { + if(reorder_ || perThread_[threadId].count >= perThreadBufSize_) { if(threadSafe_) { ThreadSafe ts(mutex_m); finishReadImpl(rec, rdid, threadId); @@ -94,7 +102,8 @@ void OutputQueue::finishRead(const BTString& rec, TReadId rdid, size_t threadId) } } if(!reorder_) { - perThreadBuf[threadId][perThreadCounter[threadId]++] = rec; + PerThread& t = perThread_[threadId]; + t.buf[t.count++].swap(rec); } } @@ -107,13 +116,14 @@ void OutputQueue::flushImpl(bool force) { int j = 0; for(i=0;i +#include #include /** @@ -49,42 +51,45 @@ class OutputQueue { int perThreadBufSize, TReadId rdid = 0) : obuf_(obuf), + reorder_(reorder), + threadSafe_(threadSafe), + perThreadBufSize_(perThreadBufSize), + nthreads_(nthreads), + perThreadMem_(NULL), + perThread_(NULL), + mutex_m(), cur_(rdid), + nstarted_(0), nfinished_(0), nflushed_(0), lines_(RES_CAT), started_(RES_CAT), - finished_(RES_CAT), - reorder_(reorder), - threadSafe_(threadSafe), - mutex_m(), - nthreads_(nthreads), - perThreadBuf(NULL), - perThreadCounter(NULL), - perThreadBufSize_(perThreadBufSize) + finished_(RES_CAT) { - nstarted_=0; assert(nthreads_ <= 2 || threadSafe); if(!reorder) { - perThreadBuf = new BTString*[nthreads_]; - perThreadCounter = new int[nthreads_]; - size_t i = 0; - for(i=0;i(perThreadMem_); + a = (a + sizeof(PerThread) - 1) & ~(uintptr_t)(sizeof(PerThread) - 1); + perThread_ = reinterpret_cast(a); + for(size_t i = 0; i < nthreads_; i++) { - perThreadBuf[i] = new BTString[perThreadBufSize_]; - perThreadCounter[i] = 0; + new (&perThread_[i]) PerThread(); + perThread_[i].buf = new BTString[perThreadBufSize_]; } } } ~OutputQueue() { - if(perThreadBuf != NULL) { + if(perThread_ != NULL) { for (size_t i = 0; i < nthreads_; i++) { - delete[] perThreadBuf[i]; + delete[] perThread_[i].buf; + perThread_[i].~PerThread(); } - delete[] perThreadBuf; - delete[] perThreadCounter; + delete[] perThreadMem_; } } @@ -97,7 +102,7 @@ class OutputQueue { /** * Writer is finished writing to */ - void finishRead(const BTString& rec, TReadId rdid, size_t threadId); + void finishRead(BTString& rec, TReadId rdid, size_t threadId); /** * Return the number of records currently being buffered. @@ -117,7 +122,13 @@ class OutputQueue { * Return the number of records that have been started so far. */ TReadId numStarted() const { - return nstarted_; + TReadId n = nstarted_; + if(perThread_ != NULL) { + for(size_t i = 0; i < nthreads_; i++) { + n += perThread_[i].nstarted; + } + } + return n; } /** @@ -134,36 +145,52 @@ class OutputQueue { protected: + /** + * A worker thread's buffered records (unordered output) and its count of + * started reads, on a cache line of its own. + */ + struct PerThread { + PerThread() : buf(NULL), count(0), nstarted(0) { } + BTString* buf; // perThreadBufSize_ records + int count; // # records in buf + TReadId nstarted; // # reads this thread started + char pad_[64 - sizeof(BTString*) - 2 * sizeof(TReadId)]; + }; + static_assert(sizeof(PerThread) == 64, "one PerThread per cache line"); + + // Read by every beginRead()/finishRead() and not written after + // construction; padRead_ keeps them off the cache line of the lock and + // of the counters written under it OutFileBuf& obuf_; + bool reorder_; + bool threadSafe_; + int perThreadBufSize_; // size of each thread's output buffer + size_t nthreads_; + char* perThreadMem_; // backing store of perThread_ + PerThread* perThread_; // nthreads_ slots, 64-byte aligned + char padRead_[64]; + + MUTEX_T mutex_m; TReadId cur_; - std::atomic nstarted_; + TReadId nstarted_; // reads started with reorder_ (under the lock) TReadId nfinished_; TReadId nflushed_; EList lines_; EList started_; EList finished_; - bool reorder_; - bool threadSafe_; - MUTEX_T mutex_m; - - // used for output read buffer - size_t nthreads_; - BTString** perThreadBuf; - int* perThreadCounter; - int perThreadBufSize_; private: void flushImpl(bool force); void beginReadImpl(TReadId rdid, size_t threadId); - void finishReadImpl(const BTString& rec, TReadId rdid, size_t threadId); + void finishReadImpl(BTString& rec, TReadId rdid, size_t threadId); }; class OutputQueueMark { public: OutputQueueMark( OutputQueue& q, - const BTString& rec, + BTString& rec, TReadId rdid, size_t threadId) : q_(q), @@ -180,7 +207,7 @@ class OutputQueueMark { protected: OutputQueue& q_; - const BTString& rec_; + BTString& rec_; TReadId rdid_; size_t threadId_; }; diff --git a/pat.cpp b/pat.cpp index 0b137bda..558aa54e 100644 --- a/pat.cpp +++ b/pat.cpp @@ -116,9 +116,9 @@ PatternSource* PatternSource::patsrcFromStrings( * unpaired read, set all the other key fields of the Read * struct. */ -void PatternSourcePerThread::finalize(Read& ra) { +void PatternSourcePerThread::finalize(Read& ra, TReadId rdid) { ra.mate = 1; - ra.rdid = buf_.rdid(); + ra.rdid = rdid; ra.seed = genRandSeed(ra.patFw, ra.qual, ra.name, pp_.seed); ra.finalize(); if(pp_.fixName) { @@ -131,10 +131,10 @@ void PatternSourcePerThread::finalize(Read& ra) { * paired-end read, set all the other key fields of the Read * structs. */ -void PatternSourcePerThread::finalizePair(Read& ra, Read& rb) { +void PatternSourcePerThread::finalizePair(Read& ra, Read& rb, TReadId rdid) { ra.mate = 1; rb.mate = 2; - ra.rdid = rb.rdid = buf_.rdid(); + ra.rdid = rb.rdid = rdid; ra.seed = genRandSeed(ra.patFw, ra.qual, ra.name, pp_.seed); rb.seed = genRandSeed(rb.patFw, rb.qual, rb.name, pp_.seed); ra.finalize(); @@ -165,23 +165,61 @@ pair PatternSourcePerThread::nextReadPair() { buf_.next(); // advance cursor; no parsing or locking needed assert_gt(buf_.cur_buf_, 0); } - // Now fully parse read/pair *outside* the critical section - assert(!buf_.read_a().readOrigBuf.empty()); - assert(buf_.read_a().empty()); - if(!parse(buf_.read_a(), buf_.read_b())) { + // Now fully parse read/pair *outside* the critical section, unless + // parseAhead already did + const uint8_t ahead = ahead_[buf_.cur_buf_]; + if(ahead == AHEAD_NONE) { + if(!parseRead(buf_.cur_buf_)) { + return make_pair(false, false); + } + } else if(ahead == AHEAD_FAILED) { return make_pair(false, false); } + bool this_is_last = buf_.cur_buf_ == static_cast(last_batch_size_-1); + return make_pair(true, this_is_last ? last_batch_ : false); +} + +/** + * Fully parse, trim and finalize the read (pair) at index i of the batch. + * Returns false iff parsing failed. + */ +bool PatternSourcePerThread::parseRead(size_t i) { + Read& ra = buf_.bufa_[i]; + Read& rb = buf_.bufb_[i]; + const TReadId rdid = buf_.rdid_ + i; + assert(!ra.readOrigBuf.empty()); + assert(ra.empty()); + if(!composer_.parse(ra, rb, rdid)) { + return false; + } // Finalize read/pair - if(!buf_.read_b().patFw.empty()) { - trim(buf_.read_a()); - trim(buf_.read_b()); - finalizePair(buf_.read_a(), buf_.read_b()); + if(!rb.patFw.empty()) { + trim(ra); + trim(rb); + finalizePair(ra, rb, rdid); } else { - trim(buf_.read_a()); - finalize(buf_.read_a()); + trim(ra); + finalize(ra, rdid); } - bool this_is_last = buf_.cur_buf_ == static_cast(last_batch_size_-1); - return make_pair(true, this_is_last ? last_batch_ : false); + return true; +} + +/** + * Fully parse up to n of the reads that follow the current one in this + * batch, ahead of nextReadPair. Returns how many reads following the + * current one are parsed. + */ +size_t PatternSourcePerThread::parseAhead(size_t n) { + size_t k = 0; + for(size_t i = buf_.cur_buf_ + 1; + k < n && i < buf_.bufa_.size() && !buf_.bufa_[i].readOrigBuf.empty(); + i++, k++) + { + if(ahead_[i] == AHEAD_NONE) { + ahead_[i] = parseRead(i) ? AHEAD_OK : AHEAD_FAILED; + } + } + return k; } /** diff --git a/pat.h b/pat.h index 53fb83eb..df513a98 100644 --- a/pat.h +++ b/pat.h @@ -31,6 +31,7 @@ #include #include #include +#include #include "alphabet.h" #include "assert_helpers.h" #include "random_source.h" @@ -1164,11 +1165,40 @@ class PatternSourcePerThread { buf_(pp.max_buf), pp_(pp), last_batch_(false), - last_batch_size_(0) { } + last_batch_size_(0), + ahead_(pp.max_buf, AHEAD_NONE) { } // If it returns true, nextReadPair is non-blocking and should return success bool nextReadPairReady() const {return !buf_.exhausted();} + /** + * Fully parse, ahead of time, the reads that follow the current read + * in this batch, at most n of them, so that they can be looked at + * before nextReadPair reaches them; nextReadPair then hands each out + * without parsing it again. Parsing a read reads only its own + * buffers, its read id and the pattern parameters, and writes only its + * own buffers, so when it is parsed does not change the result. + * Returns how many reads following the current one are parsed; the + * k-th of them (k >= 1) is read_a(k)/read_b(k), and aheadOk(k) says + * whether it parsed successfully. + */ + size_t parseAhead(size_t n); + + /// True iff the read k places after the current one parsed successfully + bool aheadOk(size_t k) const { + return ahead_[buf_.cur_buf_ + k] == AHEAD_OK; + } + + /// Index of the current read within this thread's batch + size_t bufIndex() const { return buf_.cur_buf_; } + + /// The read (pair) k places after the current one in this batch + Read& read_a(size_t k) { return buf_.bufa_[buf_.cur_buf_ + k]; } + Read& read_b(size_t k) { return buf_.bufb_[buf_.cur_buf_ + k]; } + + /// Read id of the read (pair) k places after the current one + TReadId rdid(size_t k) const { return buf_.rdid() + k; } + /** * Use objects in the PatternSource and/or PatternComposer * hierarchies to populate the per-thread buffers. @@ -1190,6 +1220,7 @@ class PatternSourcePerThread { */ std::pair nextBatch() { buf_.reset(); + std::fill(ahead_.begin(), ahead_.end(), (uint8_t)AHEAD_NONE); std::pair res = composer_.nextBatch(buf_); buf_.init(); return res; @@ -1200,22 +1231,20 @@ class PatternSourcePerThread { * unpaired read, set all the other key fields of the Read * struct. */ - void finalize(Read& ra); + void finalize(Read& ra, TReadId rdid); /** * Once name/sequence/qualities have been parsed for a * paired-end read, set all the other key fields of the Read * structs. */ - void finalizePair(Read& ra, Read& rb); + void finalizePair(Read& ra, Read& rb, TReadId rdid); /** - * Call into composition layer (which in turn calls into - * format layer) to parse the read. + * Fully parse, trim and finalize the read (pair) at index i of the + * batch. Returns false iff parsing failed. */ - bool parse(Read& ra, Read& rb) { - return composer_.parse(ra, rb, buf_.rdid()); - } + bool parseRead(size_t i); void trim(Read& r) { if (pp_.trimTo.second > 0) { @@ -1243,6 +1272,11 @@ class PatternSourcePerThread { const PatternParams& pp_; // pattern-related parameters bool last_batch_; // true if this is final batch int last_batch_size_; // # reads read in previous batch + + // Per read of the batch: not parsed ahead, parsed ahead successfully, + // or parsed ahead and failed + enum { AHEAD_NONE = 0, AHEAD_OK, AHEAD_FAILED }; + std::vector ahead_; }; /** diff --git a/random_util.h b/random_util.h index 2246694f..e393f96c 100644 --- a/random_util.h +++ b/random_util.h @@ -21,8 +21,63 @@ #define RANDOM_UTIL_H_ #include +#include #include "random_source.h" #include "ds.h" +#if defined(SSE_AVX2) && defined(__AVX2__) +#include +#elif defined(__SSE2__) +#include +#endif + +/** + * Return true iff v equals one of s[0], ..., s[k-1]. The same answer as + * comparing them one at a time; only the number of compares per step differs. + */ +static inline bool random1toNSeen(const size_t* s, size_t k, size_t v) { + size_t i = 0; +#if defined(SSE_AVX2) && defined(__AVX2__) + if(sizeof(size_t) == 8) { + const __m256i nv = _mm256_set1_epi64x((long long)v); + for(; i + 8 <= k; i += 8) { + __m256i a = _mm256_cmpeq_epi64(_mm256_loadu_si256((const __m256i*)(s + i)), nv); + __m256i b = _mm256_cmpeq_epi64(_mm256_loadu_si256((const __m256i*)(s + i + 4)), nv); + a = _mm256_or_si256(a, b); + if(!_mm256_testz_si256(a, a)) { + return true; + } + } + if(i + 4 <= k) { + __m256i a = _mm256_cmpeq_epi64(_mm256_loadu_si256((const __m256i*)(s + i)), nv); + if(!_mm256_testz_si256(a, a)) { + return true; + } + i += 4; + } + } +#elif defined(__SSE2__) + if(sizeof(size_t) == 8) { + // SSE2 has no 64-bit compare: a 64-bit lane is equal iff both of its + // 32-bit halves are + const __m128i nv = _mm_set1_epi64x((long long)v); + for(; i + 4 <= k; i += 4) { + __m128i a = _mm_cmpeq_epi32(_mm_loadu_si128((const __m128i*)(s + i)), nv); + __m128i b = _mm_cmpeq_epi32(_mm_loadu_si128((const __m128i*)(s + i + 2)), nv); + a = _mm_and_si128(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(2, 3, 0, 1))); + b = _mm_and_si128(b, _mm_shuffle_epi32(b, _MM_SHUFFLE(2, 3, 0, 1))); + if(_mm_movemask_epi8(_mm_or_si128(a, b)) != 0) { + return true; + } + } + } +#endif + for(; i < k; i++) { + if(s[i] == v) { + return true; + } + } + return false; +} /** * Return a random integer in [1, N]. Each time it's called it samples again @@ -111,19 +166,12 @@ class Random1toN { } else { assert(!converted_); // Get next pseudo-random but reject it if it's in the seen-list - bool again = true; T rn = 0; size_t seenSz = seen_.size(); - while(again) { + const T* seen = seen_.ptr(); + do { rn = rnd.nextU32() % (T)n_; - again = false; - for(size_t i = 0; i < seenSz; i++) { - if(seen_[i] == rn) { - again = true; - break; - } - } - } + } while(random1toNSeen(seen, seenSz, rn)); // Add it to the seen-list seen_.push_back(rn); cur_++; @@ -134,22 +182,28 @@ class Random1toN { // Add all elements not already in the seen list to the // swap-list assert(!seen_.empty()); - seen_.sort(); - list_.resize(n_ - cur_); - size_t prev = 0; - size_t cur = 0; + // The elements not in the seen list, in increasing order, as + // sorting the seen list and filling the gaps gave: write 0..n-1, + // overwrite the seen ones with a value no element has, and keep + // the others in order + const T gone = std::numeric_limits::max(); + list_.resize(n_); + T* l = list_.ptr(); + for(size_t j = 0; j < n_; j++) { + l[j] = (T)j; + } + const T* sp = seen_.ptr(); // seenSz + 1 entries now for(size_t i = 0; i <= seenSz; i++) { - // Add all the elements between the previous element and - // this one - for(size_t j = prev; j < seen_[i]; j++) { - list_[cur++] = (T)j; - } - prev = seen_[i]+1; + l[sp[i]] = gone; } - for(size_t j = prev; j < n_; j++) { - list_[cur++] = (T)j; + size_t cur = 0; + for(size_t j = 0; j < n_; j++) { + const T v = l[j]; + l[cur] = v; + cur += (v != gone); } assert_eq(cur, n_ - cur_); + list_.resize(cur); seen_.clear(); cur_ = 0; n_ = list_.size(); diff --git a/read.h b/read.h index 93de94ea..a58828bf 100644 --- a/read.h +++ b/read.h @@ -66,13 +66,34 @@ struct Read { * Finish initializing a new read. */ void finalize() { - for(size_t i = 0; i < patFw.length(); i++) { - if((int)patFw[i] > 3) { - ns_++; - } + // The Ns count, constructRevComps() and constructReverses() in one + // pass over patFw, through local pointers: patRc[i] and patFwRev[i] + // come from patFw[n-1-i], patRcRev[i] from patFw[i]. + const size_t n = patFw.length(); + patRc.resize(n); + patFwRev.resize(n); + patRcRev.resize(n); + const char* fw = patFw.buf(); + char* rc = patRc.wbuf(); + char* fwrev = patFwRev.wbuf(); + char* rcrev = patRcRev.wbuf(); + size_t ns = 0; + for(size_t i = 0; i < n; i++) { + const char c = fw[i]; + const char r = fw[n - i - 1]; + ns += ((int)c > 3) ? 1 : 0; + rc[i] = (r == 4 ? 4 : r ^ 3); + fwrev[i] = r; + rcrev[i] = (c == 4 ? 4 : c ^ 3); + } + ns_ += ns; + const size_t nq = qual.length(); + qualRev.resize(nq); + const char* q = qual.buf(); + char* qrev = qualRev.wbuf(); + for(size_t i = 0; i < nq; i++) { + qrev[i] = q[nq - i - 1]; } - constructRevComps(); - constructReverses(); } /** diff --git a/reference.cpp b/reference.cpp index 9f6f4d30..a9835573 100644 --- a/reference.cpp +++ b/reference.cpp @@ -38,8 +38,9 @@ BitPairReference::BitPairReference( bool useShmem, bool mmSweep, bool verbose, - bool startVerbose) : + bool startVerbose, IndexLoader* loader) : buf_(NULL), + bufBase_(NULL), sanityBuf_(NULL), loaded_(true), sanity_(sanity), @@ -200,7 +201,8 @@ BitPairReference::BitPairReference( if(!useShmem_) { // Allocate a buffer to hold the reference string try { - buf_ = new uint8_t[cumsz >> 2]; + bufBase_ = new uint8_t[(cumsz >> 2) + MM_INDEX_ALIGN]; + buf_ = mmAlignUp(bufBase_); if(buf_ == NULL) throw std::bad_alloc(); } catch(std::bad_alloc& e) { cerr << "Error: Ran out of memory allocating space for the bitpacked reference. Please" << endl @@ -212,7 +214,12 @@ BitPairReference::BitPairReference( (s4 + "[ref]"), (cumsz >> 2), &buf_, "ref", (verbose_ || startVerbose)); } - if(shmemLeader) { + if(shmemLeader && !useShmem_ && !sanity_ && loader != NULL && + loader->tryDefer(IndexLoader::ONE_REF, s4, 0, buf_, cumsz >> 2)) + { + // Read on a helper thread; the caller's loader->finish() + // waits for it before buf_ is used + } else if(shmemLeader) { // Open the bitpair-encoded reference file FILE *f4 = fopen(s4.c_str(), "rb"); if(f4 == NULL) { @@ -261,6 +268,11 @@ BitPairReference::BitPairReference( word |= ((i >> 6) & 3) << 24; } byteToU32_[i] = word; + uint32_t mword = 0; + for(int b = 0; b < 4; b++) { + mword |= (uint32_t)(1u << ((word >> (8 * b)) & 0xff)) << (8 * b); + } + byteToMask_[i] = mword; } #ifndef NDEBUG @@ -314,7 +326,7 @@ BitPairReference::BitPairReference( } BitPairReference::~BitPairReference() { - if(buf_ != NULL && !useMm_ && !useShmem_) delete[] buf_; + if(bufBase_ != NULL) delete[] bufBase_; if(sanityBuf_ != NULL) delete[] sanityBuf_; } @@ -578,6 +590,210 @@ int BitPairReference::getStretch( return (int)offset; } +#if defined(SSE_AVX2) && defined(__AVX2__) +/** + * Expand 16 bytes of the 2-bit reference (64 bases, base k of a byte in + * bits 2k..2k+1) into 64 mask bytes (1 << base) at 'dst'. The same bytes + * byteToMask_ gives for the 16 source bytes in turn. + */ +static inline void expandMasks64(const uint8_t *src, uint8_t *dst) { + // Nibble n holds two bases: n & 3 first, n >> 2 second + const __m256i lutFirst = _mm256_setr_epi8( + 1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, + 1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8); + const __m256i lutSecond = _mm256_setr_epi8( + 1, 1, 1, 1, 2, 2, 2, 2, 4, 4, 4, 4, 8, 8, 8, 8, + 1, 1, 1, 1, 2, 2, 2, 2, 4, 4, 4, 4, 8, 8, 8, 8); + const __m256i lowNibble = _mm256_set1_epi8(0x0f); + __m128i s = _mm_loadu_si128(reinterpret_cast(src)); + // Source bytes 0-7 in the low half of lane 0, 8-15 in the low half of lane 1 + __m256i y = _mm256_permute4x64_epi64(_mm256_castsi128_si256(s), 0x50); + __m256i lo = _mm256_and_si256(y, lowNibble); + __m256i hi = _mm256_and_si256(_mm256_srli_epi16(y, 4), lowNibble); + // Per lane, the 16 nibbles of its 8 source bytes in base order + __m256i nib = _mm256_unpacklo_epi8(lo, hi); + __m256i m1 = _mm256_shuffle_epi8(lutFirst, nib); + __m256i m2 = _mm256_shuffle_epi8(lutSecond, nib); + // Lane 0: bases 0-15 and 16-31; lane 1: bases 32-47 and 48-63 + __m256i a = _mm256_unpacklo_epi8(m1, m2); + __m256i b = _mm256_unpackhi_epi8(m1, m2); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(dst), + _mm256_permute2x128_si256(a, b, 0x20)); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(dst + 32), + _mm256_permute2x128_si256(a, b, 0x31)); +} +#endif + +/** + * Load a stretch of the reference string into memory at 'dest' as masks + * (see reference.h). The same walk over the records as getStretch(), with + * every byte written as its mask instead of its code. + */ +int BitPairReference::getStretchMasks( + uint32_t *destU32, + size_t tidx, + size_t toff, + size_t count + ASSERT_ONLY(, SStringExpandable& destU32_2)) const +{ + ASSERT_ONLY(size_t origCount = count); + ASSERT_ONLY(size_t origToff = toff); + if(count == 0) return 0; + uint8_t *dest = (uint8_t*)destU32; +#ifndef NDEBUG + // As in getStretch(): check one call in ten against getStretchNaive() + destU32_2.clear(); + uint8_t *dest_2 = NULL; + int off2; + if((rand() % 10) == 0) { + destU32_2.resize((origCount >> 2) + 2); + off2 = getStretchNaive(destU32_2.wbuf(), tidx, origToff, origCount); + dest_2 = ((uint8_t*)destU32_2.wbuf()) + off2; + } +#endif + destU32[0] = 0x10101010; // Add Ns, which we might end up using later + uint64_t reci = refRecOffs_[tidx]; // first record for target reference sequence + uint64_t recf = refRecOffs_[tidx+1]; // last record (exclusive) for target seq + assert_gt(recf, reci); + uint64_t cur = 4; // keep a cushion of 4 bases at the beginning + uint64_t bufOff = refOffs_[tidx]; + uint64_t off = 0; + int64_t offset = 4; + bool firstStretch = true; + uint64_t left = reci; + uint64_t right = recf; + uint64_t mid = 0; + // For all records pertaining to the target reference sequence... + for(uint64_t i = reci; i < recf; i++) { + ASSERT_ONLY(uint64_t origBufOff = bufOff); + assert_geq(toff, off); + if (firstStretch && recf > reci + 16){ + // binary search finds smallest i s.t. toff >= cumRefOff_[i] + while (left < right-1) { + mid = left + ((right - left) >> 1); + if (cumRefOff_[mid] <= toff) + left = mid; + else + right = mid; + } + off = cumRefOff_[left]; + bufOff = cumUnambig_[left]; + ASSERT_ONLY(origBufOff = bufOff); + i = left; + assert(cumRefOff_[i+1] == 0 || cumRefOff_[i+1] > toff); + } + off += recs_[i].off; // skip Ns at beginning of stretch + assert_gt(count, 0); + if(toff < off) { + size_t cpycnt = min((size_t)(off - toff), count); + memset(&dest[cur], 16, cpycnt); + count -= cpycnt; + toff += cpycnt; + cur += cpycnt; + if(count == 0) break; + } + assert_geq(toff, off); + if(toff < off + recs_[i].len) { + bufOff += toff - off; // move bufOff pointer forward + } else { + bufOff += recs_[i].len; + } + off += recs_[i].len; + assert(off == cumRefOff_[i+1] || cumRefOff_[i+1] == 0); + if(toff < off) { + if(firstStretch) { + if(toff + 8 < off && count > 8) { + // We already added some Ns, so we have to do + // a fixup at the beginning of the buffer so + // that we can start clobbering at cur >> 2 + if(cur & 3) { + offset -= (cur & 3); + } + uint64_t curU32 = cur >> 2; + // Do the initial few bases + if(bufOff & 3) { + const uint64_t bufElt = (bufOff) >> 2; + const int64_t low2 = bufOff & 3; + destU32[curU32] = byteToMask_[buf_[bufElt]]; + for(int j = 0; j < low2; j++) { + ((char *)(&destU32[curU32]))[j] = 16; + } + curU32++; + offset += low2; + const int64_t chars = 4 - low2; + count -= chars; + bufOff += chars; + toff += chars; + } + assert_eq(0, bufOff & 3); + uint64_t bufOffU32 = bufOff >> 2; + uint64_t countLim = count >> 2; + uint64_t offLim = ((off - (toff + 4)) >> 2); + uint64_t lim = min(countLim, offLim); + uint64_t j = 0; +#if defined(SSE_AVX2) && defined(__AVX2__) + // 64 bases at a time + for(; j + 16 <= lim; j += 16) { + expandMasks64(&buf_[bufOffU32], (uint8_t*)&destU32[curU32]); + bufOffU32 += 16; + curU32 += 16; + } +#endif + // Do the fast thing for as far as possible + for(; j < lim; j++) { + destU32[curU32] = byteToMask_[buf_[bufOffU32++]]; + curU32++; + } + toff += (lim << 2); + assert_leq(toff, off); + assert_leq((lim << 2), count); + count -= (lim << 2); + bufOff = bufOffU32 << 2; + cur = curU32 << 2; + } + // Do the slow thing for the rest + for(; toff < off && count > 0; toff++) { + assert_lt(bufOff, bufSz_); + const uint64_t bufElt = (bufOff) >> 2; + const uint64_t shift = (bufOff & 3) << 1; + dest[cur++] = (uint8_t)(1 << ((buf_[bufElt] >> shift) & 3)); + bufOff++; + count--; + } + firstStretch = false; + } else { + // Do the slow thing + for(; toff < off && count > 0; toff++) { + assert_lt(bufOff, bufSz_); + const uint64_t bufElt = (bufOff) >> 2; + const uint64_t shift = (bufOff & 3) << 1; + dest[cur++] = (uint8_t)(1 << ((buf_[bufElt] >> shift) & 3)); + bufOff++; + count--; + } + } + } + if(count == 0) break; + assert_eq(recs_[i].len, bufOff - origBufOff); + assert_geq(toff, off); + } // end for loop over records + // In any chars are left after scanning all the records, + // they must be ambiguous + while(count > 0) { + count--; + dest[cur++] = 16; + } + assert_eq(0, count); +#ifndef NDEBUG + if(dest_2 != NULL) { + for(size_t k = 0; k < origCount; k++) { + assert_eq((int)dest[offset + k], 1 << dest_2[k]); + } + } +#endif + return (int)offset; +} + /** * Parse the input fasta files, populating the szs list and writing the diff --git a/reference.h b/reference.h index 9b8e2e7a..8fb2d0b6 100644 --- a/reference.h +++ b/reference.h @@ -32,10 +32,14 @@ #include "ref_read.h" #include "sequence_io.h" #include "mm.h" +#include "index_load.h" #include "shmem.h" #include "timer.h" #include "sstring.h" #include "btypes.h" +#if defined(SSE_AVX2) && defined(__AVX2__) +#include +#endif /** @@ -73,7 +77,8 @@ class BitPairReference { bool useShmem = false, bool mmSweep = false, bool verbose = false, - bool startVerbose = false); + bool startVerbose = false, + IndexLoader* loader = NULL); ~BitPairReference(); @@ -115,6 +120,20 @@ class BitPairReference { size_t count ASSERT_ONLY(, SStringExpandable& destU32_2)) const; + /** + * Like getStretch(), but every byte it writes is a mask: 1 << c for a + * base c (A=1, C=2, G=4, T=8) and 16 for an ambiguous position, i.e. + * exactly the byte getStretch() writes after the conversion + * b -> 1 << b, including the 4-byte cushion in front. Returns the same + * offset as getStretch() for the same arguments. + */ + int getStretchMasks( + uint32_t *destU32, + size_t tidx, + size_t toff, + size_t count + ASSERT_ONLY(, SStringExpandable& destU32_2)) const; + /** * Return the number of reference sequences. */ @@ -165,6 +184,7 @@ class BitPairReference { protected: uint32_t byteToU32_[256]; + uint32_t byteToMask_[256]; // byteToU32_ with every byte b replaced by 1 << b EList recs_; /// records describing unambiguous stretches // following two lists are purely for the binary search in getStretch @@ -174,6 +194,7 @@ class BitPairReference { EList refOffs_; /// buf_ begin offsets per ref seq EList refRecOffs_; /// record begin/end offsets per ref seq uint8_t *buf_; /// the whole reference as a big bitpacked byte array + uint8_t *bufBase_; /// what new[] returned for buf_ (NULL when buf_ is mapped or shared) uint8_t *sanityBuf_;/// for sanity-checking buf_ TIndexOffU bufSz_; /// size of buf_ TIndexOffU bufAllocSz_; diff --git a/sam.cpp b/sam.cpp index b0f108f9..fc82cb99 100644 --- a/sam.cpp +++ b/sam.cpp @@ -135,14 +135,15 @@ void SamConfig::printAlignedOptFlags( const { char buf[1024]; + size_t nbuf = 0; // length of the number itoa10 last wrote into buf bool uniqueAlignment; assert(summ.bestScore(rd.mate < 2).valid()); if(print_as_) { // AS:i: Alignment score generated by aligner - itoa10(res.score().score(), buf); + nbuf = (size_t)(itoa10(res.score().score(), buf) - buf); WRITE_SEP(); o.append("AS:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_xs_) { // XS:i: Suboptimal alignment score @@ -153,10 +154,10 @@ void SamConfig::printAlignedOptFlags( sco = summ.bestUnchosenUScore(); } if(sco.valid()) { - itoa10(sco.score(), buf); + nbuf = (size_t)(itoa10(sco.score(), buf) - buf); WRITE_SEP(); o.append("XS:i:"); - o.append(buf); + o.append(buf, nbuf); } else { // If we did not find a secondary alignment for the read and // the average seed hits is less than the max DP failures allowed, @@ -171,10 +172,10 @@ void SamConfig::printAlignedOptFlags( } if(print_xn_) { // XN:i: Number of ambiguous bases in the referenece - itoa10(res.refNs(), buf); + nbuf = (size_t)(itoa10(res.refNs(), buf) - buf); WRITE_SEP(); o.append("XN:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_x0_) { @@ -213,31 +214,31 @@ void SamConfig::printAlignedOptFlags( } if(print_xm_) { // XM:i: Number of mismatches in the alignment - itoa10(num_mm, buf); + nbuf = (size_t)(itoa10(num_mm, buf) - buf); WRITE_SEP(); o.append("XM:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_xo_) { // XO:i: Number of gap opens - itoa10(num_go, buf); + nbuf = (size_t)(itoa10(num_go, buf) - buf); WRITE_SEP(); o.append("XO:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_xg_) { // XG:i: Number of gap extensions (incl. opens) - itoa10(num_gx, buf); + nbuf = (size_t)(itoa10(num_gx, buf) - buf); WRITE_SEP(); o.append("XG:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_nm_) { // NM:i: Edit dist. to the ref, Ns count, clipping doesn't - itoa10(res.ned().size(), buf); + nbuf = (size_t)(itoa10(res.ned().size(), buf) - buf); WRITE_SEP(); o.append("NM:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_md_) { // MD:Z: String for mms. [0-9]+(([A-Z]|\^[A-Z]+)[0-9]+)*2 @@ -260,38 +261,38 @@ void SamConfig::printAlignedOptFlags( if(print_ys_ && summ.paired()) { // YS:i: Alignment score of opposite mate assert(res.oscore().valid()); - itoa10(res.oscore().score(), buf); + nbuf = (size_t)(itoa10(res.oscore().score(), buf) - buf); WRITE_SEP(); o.append("YS:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_yn_) { // YN:i: Minimum valid score for this mate TAlScore mn = sc.scoreMin.f(rd.length()); - itoa10(mn, buf); + nbuf = (size_t)(itoa10(mn, buf) - buf); WRITE_SEP(); o.append("YN:i:"); - o.append(buf); + o.append(buf, nbuf); // Yn:i: Perfect score for this mate TAlScore pe = sc.perfectScore(rd.length()); - itoa10(pe, buf); + nbuf = (size_t)(itoa10(pe, buf) - buf); WRITE_SEP(); o.append("Yn:i:"); - o.append(buf); + o.append(buf, nbuf); if(summ.paired()) { assert(rdo != NULL); // ZN:i: Minimum valid score for opposite mate TAlScore mn = sc.scoreMin.f(rdo->length()); - itoa10(mn, buf); + nbuf = (size_t)(itoa10(mn, buf) - buf); WRITE_SEP(); o.append("ZN:i:"); - o.append(buf); + o.append(buf, nbuf); // Zn:i: Perfect score for opposite mate TAlScore pe = sc.perfectScore(rdo->length()); - itoa10(pe, buf); + nbuf = (size_t)(itoa10(pe, buf) - buf); WRITE_SEP(); o.append("Zn:i:"); - o.append(buf); + o.append(buf, nbuf); } } if(print_xss_) { @@ -302,28 +303,28 @@ void SamConfig::printAlignedOptFlags( } TAlScore bst = one ? prm.bestLtMinscMate1 : prm.bestLtMinscMate2; if(bst > std::numeric_limits::min()) { - itoa10(bst, buf); + nbuf = (size_t)(itoa10(bst, buf) - buf); WRITE_SEP(); o.append("Xs:i:"); - o.append(buf); + o.append(buf, nbuf); } if(flags.partOfPair()) { // Ys:i: Best invalid alignment score of opposite mate bst = one ? prm.bestLtMinscMate2 : prm.bestLtMinscMate1; if(bst > std::numeric_limits::min()) { - itoa10(bst, buf); + nbuf = (size_t)(itoa10(bst, buf) - buf); WRITE_SEP(); o.append("Ys:i:"); - o.append(buf); + o.append(buf, nbuf); } } } if(print_zs_) { // ZS:i: Pseudo-random seed for read - itoa10(rd.seed, buf); + nbuf = (size_t)(itoa10(rd.seed, buf) - buf); WRITE_SEP(); o.append("ZS:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_yt_) { // YT:Z: String representing alignment type @@ -358,15 +359,15 @@ void SamConfig::printAlignedOptFlags( if(summ.bestCScore().valid()) { WRITE_SEP(); o.append("ZP:i:"); - itoa10(summ.bestCScore().score(), buf); - o.append(buf); + nbuf = (size_t)(itoa10(summ.bestCScore().score(), buf) - buf); + o.append(buf, nbuf); } // Zp:i: Score of second-best concordant paired-end alignment if(summ.bestUnchosenCScore().valid()) { WRITE_SEP(); o.append("Zp:i:"); - itoa10(summ.bestUnchosenCScore().score(), buf); - o.append(buf); + nbuf = (size_t)(itoa10(summ.bestUnchosenCScore().score(), buf) - buf); + o.append(buf, nbuf); } } if(print_zu_) { @@ -376,8 +377,8 @@ void SamConfig::printAlignedOptFlags( WRITE_SEP(); o.append("ZU:i:"); if(best.valid()) { - itoa10(best.score(), buf); - o.append(buf); + nbuf = (size_t)(itoa10(best.score(), buf) - buf); + o.append(buf, nbuf); } else { o.append("NA"); } @@ -385,8 +386,8 @@ void SamConfig::printAlignedOptFlags( WRITE_SEP(); o.append("Zu:i:"); if(secbest.valid()) { - itoa10(secbest.score(), buf); - o.append(buf); + nbuf = (size_t)(itoa10(secbest.score(), buf) - buf); + o.append(buf, nbuf); } else { o.append("NA"); } @@ -404,124 +405,124 @@ void SamConfig::printAlignedOptFlags( size_t total_usecs = (tv_end.tv_sec - prm.tv_beg.tv_sec) * 1000000 + (tv_end.tv_usec - prm.tv_beg.tv_usec); - itoa10(total_usecs, buf); + nbuf = (size_t)(itoa10(total_usecs, buf) - buf); o.append("XT:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_xd_) { // XD:i: Extend DPs WRITE_SEP(); - itoa10(prm.nExDps, buf); + nbuf = (size_t)(itoa10(prm.nExDps, buf) - buf); o.append("XD:i:"); - o.append(buf); + o.append(buf, nbuf); // Xd:i: Mate DPs WRITE_SEP(); - itoa10(prm.nMateDps, buf); + nbuf = (size_t)(itoa10(prm.nMateDps, buf) - buf); o.append("Xd:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_xu_) { // XU:i: Extend ungapped tries WRITE_SEP(); - itoa10(prm.nExUgs, buf); + nbuf = (size_t)(itoa10(prm.nExUgs, buf) - buf); o.append("XU:i:"); - o.append(buf); + o.append(buf, nbuf); // Xu:i: Mate ungapped tries WRITE_SEP(); - itoa10(prm.nMateUgs, buf); + nbuf = (size_t)(itoa10(prm.nMateUgs, buf) - buf); o.append("Xu:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_ye_) { // YE:i: Streak of failed DPs at end WRITE_SEP(); - itoa10(prm.nDpFail, buf); + nbuf = (size_t)(itoa10(prm.nDpFail, buf) - buf); o.append("YE:i:"); - o.append(buf); + o.append(buf, nbuf); // Ye:i: Streak of failed ungaps at end WRITE_SEP(); - itoa10(prm.nUgFail, buf); + nbuf = (size_t)(itoa10(prm.nUgFail, buf) - buf); o.append("Ye:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_yl_) { // YL:i: Longest streak of failed DPs WRITE_SEP(); - itoa10(prm.nDpFailStreak, buf); + nbuf = (size_t)(itoa10(prm.nDpFailStreak, buf) - buf); o.append("YL:i:"); - o.append(buf); + o.append(buf, nbuf); // Yl:i: Longest streak of failed ungaps WRITE_SEP(); - itoa10(prm.nUgFailStreak, buf); + nbuf = (size_t)(itoa10(prm.nUgFailStreak, buf) - buf); o.append("Yl:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_yu_) { // YU:i: Index of last succesful DP WRITE_SEP(); - itoa10(prm.nDpLastSucc, buf); + nbuf = (size_t)(itoa10(prm.nDpLastSucc, buf) - buf); o.append("YU:i:"); - o.append(buf); + o.append(buf, nbuf); // Yu:i: Index of last succesful DP WRITE_SEP(); - itoa10(prm.nUgLastSucc, buf); + nbuf = (size_t)(itoa10(prm.nUgLastSucc, buf) - buf); o.append("Yu:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_xp_) { // XP:Z: String describing seed hits WRITE_SEP(); o.append("XP:B:I,"); - itoa10(prm.nSeedElts, buf); - o.append(buf); + nbuf = (size_t)(itoa10(prm.nSeedElts, buf) - buf); + o.append(buf, nbuf); o.append(','); - itoa10(prm.nSeedEltsFw, buf); - o.append(buf); + nbuf = (size_t)(itoa10(prm.nSeedEltsFw, buf) - buf); + o.append(buf, nbuf); o.append(','); - itoa10(prm.nSeedEltsRc, buf); - o.append(buf); + nbuf = (size_t)(itoa10(prm.nSeedEltsRc, buf) - buf); + o.append(buf, nbuf); o.append(','); - itoa10(prm.seedMean, buf); - o.append(buf); + nbuf = (size_t)(itoa10(prm.seedMean, buf) - buf); + o.append(buf, nbuf); o.append(','); - itoa10(prm.seedMedian, buf); - o.append(buf); + nbuf = (size_t)(itoa10(prm.seedMedian, buf) - buf); + o.append(buf, nbuf); } if(print_yr_) { // YR:i: Redundant seed hits WRITE_SEP(); - itoa10(prm.nRedundants, buf); + nbuf = (size_t)(itoa10(prm.nRedundants, buf) - buf); o.append("YR:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_zb_) { // ZB:i: Ftab ops for seed alignment WRITE_SEP(); - itoa10(prm.nFtabs, buf); + nbuf = (size_t)(itoa10(prm.nFtabs, buf) - buf); o.append("ZB:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_zr_) { // ZR:Z: Redundant path skips in seed alignment WRITE_SEP(); o.append("ZR:Z:"); - itoa10(prm.nRedSkip, buf); o.append(buf); + nbuf = (size_t)(itoa10(prm.nRedSkip, buf) - buf); o.append(buf, nbuf); o.append(','); - itoa10(prm.nRedFail, buf); o.append(buf); + nbuf = (size_t)(itoa10(prm.nRedFail, buf) - buf); o.append(buf, nbuf); o.append(','); - itoa10(prm.nRedIns, buf); o.append(buf); + nbuf = (size_t)(itoa10(prm.nRedIns, buf) - buf); o.append(buf, nbuf); } if(print_zf_) { // ZF:i: FM Index ops for seed alignment WRITE_SEP(); - itoa10(prm.nSdFmops, buf); + nbuf = (size_t)(itoa10(prm.nSdFmops, buf) - buf); o.append("ZF:i:"); - o.append(buf); + o.append(buf, nbuf); // Zf:i: FM Index ops for offset resolution WRITE_SEP(); - itoa10(prm.nExFmops, buf); + nbuf = (size_t)(itoa10(prm.nExFmops, buf) - buf); o.append("Zf:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_zm_) { // ZM:Z: Print FM index op string for best-first search @@ -532,9 +533,9 @@ void SamConfig::printAlignedOptFlags( if(print_zi_) { // ZI:i: Seed extend loop iterations WRITE_SEP(); - itoa10(prm.nExIters, buf); + nbuf = (size_t)(itoa10(prm.nExIters, buf) - buf); o.append("ZI:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_zt_) { // ZT:Z: Extra features for MAPQ estimation @@ -593,61 +594,61 @@ void SamConfig::printAlignedOptFlags( } o.append("ZT:Z:"); // AS:i for current mate - itoa10((int)best[0].score(), buf); - o.append(buf); + nbuf = (size_t)(itoa10((int)best[0].score(), buf) - buf); + o.append(buf, nbuf); o.append(","); // diff for current mate if(diff[0] > MN) { - itoa10((int)diff[0], buf); - o.append(buf); + nbuf = (size_t)(itoa10((int)diff[0], buf) - buf); + o.append(buf, nbuf); } else { o.append("NA"); } o.append(","); // edit distance diff for current mate if(diffEd[0] != ED_MAX) { - itoa10((int)diffEd[0], buf); - o.append(buf); + nbuf = (size_t)(itoa10((int)diffEd[0], buf) - buf); + o.append(buf, nbuf); } else { o.append("NA"); } o.append(","); // AS:i for other mate if(best[1].score() > MN) { - itoa10((int)best[1].score(), buf); - o.append(buf); + nbuf = (size_t)(itoa10((int)best[1].score(), buf) - buf); + o.append(buf, nbuf); } else { o.append("NA"); } o.append(","); // diff for other mate if(diff[1] > MN) { - itoa10((int)diff[1], buf); - o.append(buf); + nbuf = (size_t)(itoa10((int)diff[1], buf) - buf); + o.append(buf, nbuf); } else { o.append("NA"); } o.append(","); // Sum of AS:i for aligned pairs if(best_conc > MN) { - itoa10((int)best_conc, buf); - o.append(buf); + nbuf = (size_t)(itoa10((int)best_conc, buf) - buf); + o.append(buf, nbuf); } else { o.append("NA"); } o.append(","); // Diff for aligned pairs if(diff_conc > MN) { - itoa10((int)diff_conc, buf); - o.append(buf); + nbuf = (size_t)(itoa10((int)diff_conc, buf) - buf); + o.append(buf, nbuf); } else { o.append("NA"); } o.append(","); // Edit distance diff for aligned pairs if(diffEd_conc != ED_MAX) { - itoa10((int)diffEd_conc, buf); - o.append(buf); + nbuf = (size_t)(itoa10((int)diffEd_conc, buf) - buf); + o.append(buf, nbuf); } else { o.append("NA"); } @@ -655,32 +656,32 @@ void SamConfig::printAlignedOptFlags( // strand aligned to int mate = (rd.mate < 2 ? 0 : 1); o.append(","); - itoa10((int)((prm.seedsPerNucMS[2 * mate] + prm.seedsPerNucMS[2 * mate + 1]) * 1000), buf); - o.append(buf); + nbuf = (size_t)(itoa10((int)((prm.seedsPerNucMS[2 * mate] + prm.seedsPerNucMS[2 * mate + 1]) * 1000), buf) - buf); + o.append(buf, nbuf); o.append(","); - itoa10((int)((prm.seedPctUniqueMS[2 * mate] + prm.seedPctUniqueMS[2 * mate + 1]) * 1000), buf); - o.append(buf); + nbuf = (size_t)(itoa10((int)((prm.seedPctUniqueMS[2 * mate] + prm.seedPctUniqueMS[2 * mate + 1]) * 1000), buf) - buf); + o.append(buf, nbuf); o.append(","); - itoa10((int)((prm.seedPctRepMS[2 * mate] + prm.seedPctRepMS[2 * mate + 1]) * 1000), buf); - o.append(buf); + nbuf = (size_t)(itoa10((int)((prm.seedPctRepMS[2 * mate] + prm.seedPctRepMS[2 * mate + 1]) * 1000), buf) - buf); + o.append(buf, nbuf); o.append(","); - itoa10((int)((prm.seedHitAvgMS[2 * mate] + prm.seedHitAvgMS[2 * mate + 1]) + 0.5f), buf); - o.append(buf); + nbuf = (size_t)(itoa10((int)((prm.seedHitAvgMS[2 * mate] + prm.seedHitAvgMS[2 * mate + 1]) + 0.5f), buf) - buf); + o.append(buf, nbuf); // Flags related to seed hits again, but specific both to this mate and // to the strand aligned to int fw = res.fw() ? 0 : 1; o.append(","); - itoa10((int)(prm.seedsPerNucMS[2 * mate + fw] * 1000), buf); - o.append(buf); + nbuf = (size_t)(itoa10((int)(prm.seedsPerNucMS[2 * mate + fw] * 1000), buf) - buf); + o.append(buf, nbuf); o.append(","); - itoa10((int)(prm.seedPctUniqueMS[2 * mate + fw] * 1000), buf); - o.append(buf); + nbuf = (size_t)(itoa10((int)(prm.seedPctUniqueMS[2 * mate + fw] * 1000), buf) - buf); + o.append(buf, nbuf); o.append(","); - itoa10((int)(prm.seedPctRepMS[2 * mate + fw] * 1000), buf); - o.append(buf); + nbuf = (size_t)(itoa10((int)(prm.seedPctRepMS[2 * mate + fw] * 1000), buf) - buf); + o.append(buf, nbuf); o.append(","); - itoa10((int)(prm.seedHitAvgMS[2 * mate + fw] + 0.5f), buf); - o.append(buf); + nbuf = (size_t)(itoa10((int)(prm.seedHitAvgMS[2 * mate + fw] + 0.5f), buf) - buf); + o.append(buf, nbuf); if (uniqueAlignment) o.append(",unique alignment"); } @@ -701,26 +702,27 @@ void SamConfig::printEmptyOptFlags( const { char buf[1024]; + size_t nbuf = 0; // length of the number itoa10 last wrote into buf if(print_yn_) { // YN:i: Minimum valid score for this mate TAlScore mn = sc.scoreMin.f(rd.length()); - itoa10(mn, buf); + nbuf = (size_t)(itoa10(mn, buf) - buf); WRITE_SEP(); o.append("YN:i:"); - o.append(buf); + o.append(buf, nbuf); // Yn:i: Perfect score for this mate TAlScore pe = sc.perfectScore(rd.length()); - itoa10(pe, buf); + nbuf = (size_t)(itoa10(pe, buf) - buf); WRITE_SEP(); o.append("Yn:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_zs_) { // ZS:i: Pseudo-random seed for read - itoa10(rd.seed, buf); + nbuf = (size_t)(itoa10(rd.seed, buf) - buf); WRITE_SEP(); o.append("ZS:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_yt_) { // YT:Z: String representing alignment type @@ -754,124 +756,124 @@ void SamConfig::printEmptyOptFlags( size_t total_usecs = (tv_end.tv_sec - prm.tv_beg.tv_sec) * 1000000 + (tv_end.tv_usec - prm.tv_beg.tv_usec); - itoa10(total_usecs, buf); + nbuf = (size_t)(itoa10(total_usecs, buf) - buf); o.append("XT:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_xd_) { // XD:i: Extend DPs WRITE_SEP(); - itoa10(prm.nExDps, buf); + nbuf = (size_t)(itoa10(prm.nExDps, buf) - buf); o.append("XD:i:"); - o.append(buf); + o.append(buf, nbuf); // Xd:i: Mate DPs WRITE_SEP(); - itoa10(prm.nMateDps, buf); + nbuf = (size_t)(itoa10(prm.nMateDps, buf) - buf); o.append("Xd:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_xu_) { // XU:i: Extend ungapped tries WRITE_SEP(); - itoa10(prm.nExUgs, buf); + nbuf = (size_t)(itoa10(prm.nExUgs, buf) - buf); o.append("XU:i:"); - o.append(buf); + o.append(buf, nbuf); // Xu:i: Mate ungapped tries WRITE_SEP(); - itoa10(prm.nMateUgs, buf); + nbuf = (size_t)(itoa10(prm.nMateUgs, buf) - buf); o.append("Xu:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_ye_) { // YE:i: Streak of failed DPs at end WRITE_SEP(); - itoa10(prm.nDpFail, buf); + nbuf = (size_t)(itoa10(prm.nDpFail, buf) - buf); o.append("YE:i:"); - o.append(buf); + o.append(buf, nbuf); // Ye:i: Streak of failed ungaps at end WRITE_SEP(); - itoa10(prm.nUgFail, buf); + nbuf = (size_t)(itoa10(prm.nUgFail, buf) - buf); o.append("Ye:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_yl_) { // YL:i: Longest streak of failed DPs WRITE_SEP(); - itoa10(prm.nDpFailStreak, buf); + nbuf = (size_t)(itoa10(prm.nDpFailStreak, buf) - buf); o.append("YL:i:"); - o.append(buf); + o.append(buf, nbuf); // Yl:i: Longest streak of failed ungaps WRITE_SEP(); - itoa10(prm.nUgFailStreak, buf); + nbuf = (size_t)(itoa10(prm.nUgFailStreak, buf) - buf); o.append("Yl:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_yu_) { // YU:i: Index of last succesful DP WRITE_SEP(); - itoa10(prm.nDpLastSucc, buf); + nbuf = (size_t)(itoa10(prm.nDpLastSucc, buf) - buf); o.append("YU:i:"); - o.append(buf); + o.append(buf, nbuf); // Yu:i: Index of last succesful DP WRITE_SEP(); - itoa10(prm.nUgLastSucc, buf); + nbuf = (size_t)(itoa10(prm.nUgLastSucc, buf) - buf); o.append("Yu:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_xp_) { // XP:Z: String describing seed hits WRITE_SEP(); o.append("XP:B:I,"); - itoa10(prm.nSeedElts, buf); - o.append(buf); + nbuf = (size_t)(itoa10(prm.nSeedElts, buf) - buf); + o.append(buf, nbuf); o.append(','); - itoa10(prm.nSeedEltsFw, buf); - o.append(buf); + nbuf = (size_t)(itoa10(prm.nSeedEltsFw, buf) - buf); + o.append(buf, nbuf); o.append(','); - itoa10(prm.nSeedEltsRc, buf); - o.append(buf); + nbuf = (size_t)(itoa10(prm.nSeedEltsRc, buf) - buf); + o.append(buf, nbuf); o.append(','); - itoa10(prm.seedMean, buf); - o.append(buf); + nbuf = (size_t)(itoa10(prm.seedMean, buf) - buf); + o.append(buf, nbuf); o.append(','); - itoa10(prm.seedMedian, buf); - o.append(buf); + nbuf = (size_t)(itoa10(prm.seedMedian, buf) - buf); + o.append(buf, nbuf); } if(print_yr_) { // YR:i: Redundant seed hits WRITE_SEP(); - itoa10(prm.nRedundants, buf); + nbuf = (size_t)(itoa10(prm.nRedundants, buf) - buf); o.append("YR:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_zb_) { // ZB:i: Ftab ops for seed alignment WRITE_SEP(); - itoa10(prm.nFtabs, buf); + nbuf = (size_t)(itoa10(prm.nFtabs, buf) - buf); o.append("ZB:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_zr_) { // ZR:Z: Redundant path skips in seed alignment WRITE_SEP(); o.append("ZR:Z:"); - itoa10(prm.nRedSkip, buf); o.append(buf); + nbuf = (size_t)(itoa10(prm.nRedSkip, buf) - buf); o.append(buf, nbuf); o.append(','); - itoa10(prm.nRedFail, buf); o.append(buf); + nbuf = (size_t)(itoa10(prm.nRedFail, buf) - buf); o.append(buf, nbuf); o.append(','); - itoa10(prm.nRedIns, buf); o.append(buf); + nbuf = (size_t)(itoa10(prm.nRedIns, buf) - buf); o.append(buf, nbuf); } if(print_zf_) { // ZF:i: FM Index ops for seed alignment WRITE_SEP(); - itoa10(prm.nSdFmops, buf); + nbuf = (size_t)(itoa10(prm.nSdFmops, buf) - buf); o.append("ZF:i:"); - o.append(buf); + o.append(buf, nbuf); // Zf:i: FM Index ops for offset resolution WRITE_SEP(); - itoa10(prm.nExFmops, buf); + nbuf = (size_t)(itoa10(prm.nExFmops, buf) - buf); o.append("Zf:i:"); - o.append(buf); + o.append(buf, nbuf); } if(print_zm_) { // ZM:Z: Print FM index op string for best-first search @@ -882,9 +884,9 @@ void SamConfig::printEmptyOptFlags( if(print_zi_) { // ZI:i: Seed extend loop iterations WRITE_SEP(); - itoa10(prm.nExIters, buf); + nbuf = (size_t)(itoa10(prm.nExIters, buf) - buf); o.append("ZI:i:"); - o.append(buf); + o.append(buf, nbuf); } } diff --git a/scoring.h b/scoring.h index 54038a62..7e152485 100644 --- a/scoring.h +++ b/scoring.h @@ -21,6 +21,8 @@ #define SCORING_H_ #include +#include +#include #include "qual.h" #include "simple_func.h" @@ -169,6 +171,7 @@ class Scoring { initPens(mmpens, mmcostType, mmpMin_, mmpMax_); initPens(npens, npenType, npen, npen); initPens(matchBonuses, matchType, matchConst, matchConst); + initQualProf(); assert(repOk()); } @@ -179,6 +182,7 @@ class Scoring { matchType = COST_MODEL_CONSTANT; matchConst = bonus; initPens(matchBonuses, matchType, matchConst, matchConst); + initQualProf(); assert(repOk()); } @@ -190,6 +194,7 @@ class Scoring { mmpMax = mmpMax_; mmpMin = mmpMin_; initPens(mmpens, mmcostType, mmpMin, mmpMax); + initQualProf(); } /** @@ -199,6 +204,7 @@ class Scoring { npenType = nType; npen = n; initPens(npens, npenType, npen, npen); + initQualProf(); } #ifndef NDEBUG @@ -459,6 +465,56 @@ class Scoring { int mmpens[256]; // map from qualities to mm penalty int npens[256]; // map from N qualities to penalty + /** + * score(rdc, 1 << refc, q) for the five reference characters refc + * (A, C, G, T, N) at once, in the forms SwAligner's 8-bit query-profile + * builders store, with the extremes they track. One entry per read + * character rdc (0-4) and quality q (0-255). Derived from matchBonuses, + * mmpens and npens by initQualProf(), which every function that changes + * those arrays calls; code that writes the arrays directly must call it + * too. + */ + struct QualProf { + uint64_t ee; // byte refc: (uint8_t)-score, end-to-end penalty + uint64_t loc; // byte refc: (uint8_t)score, local score before bias + int eeMax; // the -score whose (size_t) is largest, or 0 + int locMin; // the smallest score, or 0 if none is negative + int locPen; // the largest -score of a negative score, or 0 + int locBonus; // the largest non-negative score, or 0 + }; + QualProf qualProf[5][256]; + + /** + * Fill qualProf from score(). + */ + void initQualProf() { + for(int rdc = 0; rdc < 5; rdc++) { + for(int q = 0; q < 256; q++) { + QualProf& e = qualProf[rdc][q]; + e.ee = e.loc = 0; + e.eeMax = e.locMin = e.locPen = e.locBonus = 0; + for(int refc = 0; refc < 5; refc++) { + int sc = score(rdc, 1 << refc, q); + e.ee |= (uint64_t)(uint8_t)(-sc) << (8 * refc); + e.loc |= (uint64_t)(uint8_t)sc << (8 * refc); + if((size_t)(-sc) > (size_t)e.eeMax) { + e.eeMax = -sc; + } + if(sc < e.locMin) { + e.locMin = sc; + } + if(sc < 0) { + if(-sc > e.locPen) { + e.locPen = -sc; + } + } else if(sc > e.locBonus) { + e.locBonus = sc; + } + } + } + } + } + static Scoring base1() { const double DMAX = std::numeric_limits::max(); SimpleFunc scoreMin(SIMPLE_FUNC_LINEAR, 0.0f, DMAX, 37.0f, 0.3f); diff --git a/sse_wrap.h b/sse_wrap.h index cf6e7370..865ec58e 100644 --- a/sse_wrap.h +++ b/sse_wrap.h @@ -36,6 +36,7 @@ typedef __m256i SSERegI; #define sse_adds_epi16(x, y) _mm256_adds_epi16(x, y) #define sse_adds_epu8(x, y) _mm256_adds_epu8(x, y) +#define sse_and_siall(x, y) _mm256_and_si256(x, y) #define sse_cmpeq_epi16(x, y) _mm256_cmpeq_epi16(x, y) #define sse_cmpeq_epi8(x, y) _mm256_cmpeq_epi8(x, y) #define sse_cmpgt_epi16(x, y) _mm256_cmpgt_epi16(x, y) @@ -126,6 +127,7 @@ typedef __m256i SSERegI; typedef simde__m128i SSERegI; #define sse_adds_epi16(x, y) simde_mm_adds_epi16(x, y) #define sse_adds_epu8(x, y) simde_mm_adds_epu8(x, y) +#define sse_and_siall(x, y) simde_mm_and_si128(x, y) #define sse_cmpeq_epi16(x, y) simde_mm_cmpeq_epi16(x, y) #define sse_cmpeq_epi8(x, y) simde_mm_cmpeq_epi8(x, y) #define sse_cmpgt_epi16(x, y) simde_mm_cmpgt_epi16(x, y) @@ -155,6 +157,7 @@ typedef simde__m128i SSERegI; typedef __m128i SSERegI; #define sse_adds_epi16(x, y) _mm_adds_epi16(x, y) #define sse_adds_epu8(x, y) _mm_adds_epu8(x, y) +#define sse_and_siall(x, y) _mm_and_si128(x, y) #define sse_cmpeq_epi16(x, y) _mm_cmpeq_epi16(x, y) #define sse_cmpeq_epi8(x, y) _mm_cmpeq_epi8(x, y) #define sse_cmpgt_epi16(x, y) _mm_cmpgt_epi16(x, y) diff --git a/sstring.h b/sstring.h index 606a9455..19720b7d 100644 --- a/sstring.h +++ b/sstring.h @@ -2006,6 +2006,35 @@ class SStringExpandable { append(b, strlen(b)); } + /** + * Append xform[b[0]], xform[b[1]], ... up to the first character that + * maps to '\0'. The same bytes as append(s.toZBufXForm(xform)) for an + * SStringExpandable s holding b[0..sz), without the intermediate print + * buffer and the strlen over it. + */ + void appendXForm(const char* b, size_t sz, const char* xform) { + if(sz_ < len_ + sz) expandCopy((len_ + sz + S) * M); + T* dst = cs_ + len_; + size_t i = 0; + for(; i < sz; i++) { + const char c = xform[(int)b[i]]; + if(c == 0) break; + dst[i] = c; + } + len_ += i; + } + + /** + * Exchange buffers, lengths and capacities with another string of the + * same type; no characters are copied. + */ + void swap(SStringExpandable& o) { + T* cs = cs_; cs_ = o.cs_; o.cs_ = cs; + char* pcs = printcs_; printcs_ = o.printcs_; o.printcs_ = pcs; + size_t l = len_; len_ = o.len_; o.len_ = l; + size_t z = sz_; sz_ = o.sz_; o.sz_ = z; + } + /** * Return the length of the string. */ From 9bdab09bc935280d63c5a6a48f569a5c5c33bee1 Mon Sep 17 00:00:00 2001 From: Jacob Schreiber Date: Sun, 18 Oct 2026 10:54:05 -0400 Subject: [PATCH 3/3] Add the 8-bit backtraces' wrap offset to the pointer as a signed value In the 8-bit end-to-end and local backtraces, the step from vector row 0 to the previous element of the last vector row is btUpWrap = (colstride - ROWSTRIDE) * NBYTES_PER_REG - 1, a size_t. When the matrix has one vector row (reads of up to 16 bases with 16-byte vectors), it is (size_t)-1, and btcur + btUpWrap wraps around the address space to btcur - 1. The address is the intended one, but the wrap is undefined behaviour, which clang's -fsanitize=pointer-overflow reports; the sanitized build in the macOS CI job aborted on it. Casting to ptrdiff_t at the addition makes the step a plain -1. With gcc 13 on x86-64, the release aligners' machine code is identical with and without the cast. Co-Authored-By: Claude Opus 5.5 --- aligner_swsse_ee_u8.cpp | 2 +- aligner_swsse_loc_u8.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aligner_swsse_ee_u8.cpp b/aligner_swsse_ee_u8.cpp index 88f636ad..93627055 100644 --- a/aligner_swsse_ee_u8.cpp +++ b/aligner_swsse_ee_u8.cpp @@ -2301,7 +2301,7 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( assert_leq(col, origCol); // The cell above; used only when row > 0 const bool btwrap = (btrowvec == 0); - uint8_t * const btup = btwrap ? btcur + btUpWrap : btcur - btUp; + uint8_t * const btup = btwrap ? btcur + (ptrdiff_t)btUpWrap : btcur - btUp; // (size_t)-1 when nvecrow_ is 1 assert_eq(btcur[0], ((TCScore*)(d.mat_.matbuf_.ptr() + col * btcolstride + btrowvec * ROWSTRIDE))[row / btnvecrow]); assert(row == 0 || btup[btH] == d.mat_.helt(row - 1, col)); // Get score in this cell diff --git a/aligner_swsse_loc_u8.cpp b/aligner_swsse_loc_u8.cpp index 886eae68..d47f465b 100644 --- a/aligner_swsse_loc_u8.cpp +++ b/aligner_swsse_loc_u8.cpp @@ -2092,7 +2092,7 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( assert_leq(col, origCol); // The cell above; used only when row > 0 const bool btwrap = (btrowvec == 0); - uint8_t * const btup = btwrap ? btcur + btUpWrap : btcur - btUp; + uint8_t * const btup = btwrap ? btcur + (ptrdiff_t)btUpWrap : btcur - btUp; // (size_t)-1 when nvecrow_ is 1 assert_eq(btcur[0], ((TCScore*)(d.mat_.matbuf_.ptr() + col * btcolstride + btrowvec * ROWSTRIDE))[row / btnvecrow]); assert(row == 0 || btup[btH] == d.mat_.helt(row - 1, col)); // Get score in this cell