fix: handle lane-split flowcells and run 10x Multiome with cellranger-arc

BRB/email.py

@@ -45,10 +45,11 @@ def finishedEmail(config, msg):
         # Only inform deepseq if no workflow failed, and no group was
         # skipped (a flowcell with a skipped group -- e.g. owned by a live
         # PID from another process -- was not actually fully processed this
-        # pass, so deepSeq should not be told it's ready).
+        # pass, so deepSeq should not be told it's ready), and no Multiome
+        # group needs attention (pairs waiting for another lane or unpaired).
         statuses = [i[4] for i in msg]
         if statuses.count("FAILED") == 0 and not any(
-            s.startswith("SKIPPED") for s in statuses
+            str(s).startswith(("SKIPPED", "ATTENTION")) for s in statuses
         ):
             recipient = config.get("Email", "deepSeq")
             _html.add(

BRB/findFinishedFlowCells.py

@@ -14,6 +14,11 @@ def flowCellProcessed(config):
 
 
 def markFinished(config):
+    """
+    Write analysis.done marker for a processed flow cell.
+    Creates/updates analysis.done under config[Paths][baseData]/config[Options][runID]
+    to signal the flowcell has been fully processed.
+    """
     _p = Path(config["Paths"]["baseData"], config["Options"]["runID"], "analysis.done")
     _p.touch()
     log.info(f"{_p} created, flow cell processed.")

BRB/run.py

@@ -80,6 +80,14 @@ def run_brb(configfile, sequencer):
             print(errMsg, file=sys.stderr)
             raise
 
+        # Multiome groups with pairs that could not run this pass (waiting
+        # for another lane folder, unpaired, ...): make them hard to miss.
+        for row in msg:
+            if str(row[4]).startswith("ATTENTION"):
+                warn = f"Multiome {row[0]} ({config.get('Options', 'runID')}): {row[4]}"
+                log.warning(warn)
+                print(f"[bold red]!!! {warn}[/bold red]")
+
         # Email finished message
         log.info("Create e-mail")
         log.info(msg)

BigRedButton.ini

@@ -76,6 +76,7 @@ Users=path_to_user_file
 # separate pipelines specific for 10x Single Cell processing
 RNA=/path/to/repo/10X_snakepipe/10x_scRNA/10x_scRNA
 ATAC=/path/to/repo/10X_snakepipe/10x_scATAC/10x_scATAC
+ARC=/path/to/repo/10X_snakepipe/10x_ARC/10x_ARC
 
 
 [external]

configs/BigRedButton_dev.ini

@@ -0,0 +1,105 @@
+[Paths]
+#Where full/partial flowcell demultiplexing results are stored. Copies of parts of this are delivered to groups
+baseData=/path/to/dissectBCL_results
+# groupdata = /path if PI A and pi B have folders as /path/A/sequencing_data and /path/B/sequencing_data
+groupData=/path/to/basedir/of/PI/folders
+# where sequencing facility inspects QC reports
+seqFacDir=/path/to/seqfacdir
+# where the log-files show appear
+logPath=/path/to/logdir
+
+#Groups are stored under $groupData/groupName
+groupData=/data
+bioinfoCoreDir=/path/to/bioinfocoreDir
+
+[Options]
+#How long to sleep between runs, in hours, may be fractional
+sleepTime=1
+#valid library types, separate by a comma
+validAnalysisTypes=ChIP-Seq,stranded mRNA-Seq,stranded RNA-Seq,mRNA-Seq,RNA-Seq,WGS,ATAC-Seq
+#Pipeline names, one for each of the aforementioned validAnalysisTypes. Each pipeline is actually a function in the PushButton submodule
+pipelines=DNA,RNA,RNA,RNA,RNA,DNA,ATAC
+#valid organisms, separate by a comma
+validOrganisms=human,mouse,drosophila
+#An ordered list of the organism names used in the pipeline (the order matches validOrganisms)
+organismNames=hg38,mm10,dm6
+#Snakemake workflow base directory
+snakemakeWorkflowBaseDir=/path/anaconda3/envs/snakePipes-V.x.y/
+# How many library-groups to dispatch concurrently per flowcell. Each one is a
+# local snakePipes driver; RELACS groups additionally fan out to 10 local
+# demultiplex_relacs processes. When BigRedButton is started with -s/--sequencer,
+# two processes share this host and each takes half of this value (minimum 1).
+poolSize=2
+# Path to the scancel binary used to cancel a crashed flowcell's Slurm jobs.
+# Defaults to "scancel" (resolved off PATH). On hosts where Slurm's client
+# binaries only appear on PATH after `module load slurm` (rapidus, at
+# least), run_brb won't see them: set this to the absolute path scancel
+# resolves to after that module load (e.g. `which scancel` in that shell).
+scancelBin=scancel
+#Don't set these, they're just for passing info around
+runID=
+
+[Queue]
+queueName=queue_name
+parallelProcesses=10
+
+[Parkour]
+QueryURL=http://someserver.com/api/analysis_list/analysis_list/
+ResultsURL=
+# Parkour's internal_pis endpoint: PIs whose /data directory diverges from their
+# Parkour name carry a deliver_to there (see BRB.misc.resolveDeliverTo,
+# PushButton.GetResults). Required; BRB stops with an error if unreachable.
+InternalPIsURL=http://someserver.com/api/internal_pis/
+user=foo@bar.com
+password=supersecret
+cert=path/to/cert.pem
+
+[Internals]
+# Passed as the `organizations` param to InternalPIsURL above.
+Organizations=MPI-IE
+
+[Email]
+#Lists of recipients can be comma separated
+errorTo=mustermann@ie-freiburg.mpg.de
+finishedTo=mustermann@ie-freiburg.mpg.de
+deepSeq=mustermann@ie-freiburg.mpg.de, another.example@ie-freiburg.mpg.de
+fromAddress=some.user@ie-freiburg.mpg.de
+host=localhost
+
+[Galaxy]
+URL=URL_to_galaxy_server
+API key=the_api_key
+verify=False
+Users=path_to_user_file
+
+[10x]
+# separate pipelines specific for 10x Single Cell processing
+RNA=/path/to/repo/10X_snakepipe/10x_scRNA/10x_scRNA
+ATAC=/path/to/repo/10X_snakepipe/10x_scATAC/10x_scATAC
+ARC=/path/to/repo/10X_snakepipe/10x_ARC/10x_ARC
+ARC=/path/to/repo/10X_snakepipe/10x_ARC/10x_ARC
+
+
+[external]
+# Projects that don't belong to any PI/group under $groupData (e.g. external
+# collaborators) are skipped by default, regardless of library type. The two
+# settings below opt specific libraries back in for processing anyway; their
+# output goes under $baseData/<runID>/Analysis_<project>/ instead of a PI's
+# groupData tree, since there's no group directory to put it in.
+#
+# Library types (comma separated) that should still be processed for
+# external projects. Matched against the Parkour "library type" field (see
+# validAnalysisTypes above).
+AnalysisTypes =
+# Library protocols (comma separated) that should still be processed for
+# external projects, matched via startswith() against the Parkour "library
+# protocol" field. Use this for protocol-specific opt-ins (e.g. "ChIP RELACS
+# high-throughput") without opening up every library of that type to
+# external processing.
+LibraryProtocols =
+# Where BRB fexsends the final results tarball for external projects
+# processed via AnalysisTypes/LibraryProtocols above (currently only used by
+# the RELACS pipeline).
+fexRecipient = mustermann@ie-freiburg.mpg.de
+# Path to the fexsend binary used for the above.
+fexsendBin = ~/.local/bin/fexsend

pyproject.toml

@@ -35,6 +35,9 @@ dependencies = [
 BigRedButton = "BRB.run:run_brb"
 demultiplex_relacs = "BRB.demultiplex_relacs:main"
 
+
+[tool.setuptools]
+packages = ["BRB"]
 [tool.setuptools_scm]
 
 [tool.ruff]

tests/test_PushButton.py

@@ -637,9 +637,9 @@ def test_no_check_call_remains(self):
         source = Path(PushButton.__file__).read_text()
         assert "subprocess.check_call" not in source
 
-    def test_all_twelve_sites_converted(self):
+    def test_all_thirteen_sites_converted(self):
         source = Path(PushButton.__file__).read_text()
-        assert source.count("runManagedSubprocess(") == 12
+        assert source.count("runManagedSubprocess(") == 13
 
     def test_the_two_cwd_sites_keep_their_cwd(self):
         source = Path(PushButton.__file__).read_text()