diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 0000000..4569f68 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,32 @@ +name: Conformance + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + workflow_dispatch: # manual run from any branch + +jobs: + conformance: + name: RTSTRUCT->mask conformance + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: | + setup.py + requirements.txt + + - name: Install package + conformance extra + run: | + python -m pip install --upgrade pip + pip install -e ".[conformance]" + pip install pytest + + - name: Run conformance gate + run: pytest tests/test_conformance.py -v diff --git a/rt_utils/image_helper.py b/rt_utils/image_helper.py index 5770b42..49f993f 100644 --- a/rt_utils/image_helper.py +++ b/rt_utils/image_helper.py @@ -283,6 +283,16 @@ def get_slice_mask_from_slice_contour_data( return slice_mask def create_empty_series_mask(series_data): + """Allocate the empty 3D mask that ``get_slice_mask_from_slice_contour_data`` + fills via ``cv2.fillPoly``. + + NB on axis order: dimensions are allocated as ``(Columns, Rows, Slices)``, + but ``cv2.fillPoly`` writes into the per-slice mask at ``[y, x]`` indices, + so the *populated* array's semantic order is ``(rows=Y, columns=X, slices=Z)``. + For square images the two collapse; for non-square images the populated + layout is what consumers should use. ``get_roi_mask_by_name``'s docstring + states the populated convention explicitly. + """ ref_dicom_image = series_data[0] mask_dims = ( int(ref_dicom_image.Columns), @@ -294,6 +304,12 @@ def create_empty_series_mask(series_data): def create_empty_slice_mask(series_slice): + """Allocate the empty 2D slice mask that ``cv2.fillPoly`` fills. + + See ``create_empty_series_mask`` for the axis-order caveat — the populated + slice is indexed as ``[y, x]`` even though dimensions are nominally + ``(Columns, Rows)``. + """ mask_dims = (int(series_slice.Columns), int(series_slice.Rows)) mask = np.zeros(mask_dims).astype(bool) return mask diff --git a/rt_utils/rtstruct.py b/rt_utils/rtstruct.py index c52d816..125d993 100644 --- a/rt_utils/rtstruct.py +++ b/rt_utils/rtstruct.py @@ -135,7 +135,17 @@ def get_roi_names(self) -> List[str]: def get_roi_mask_by_name(self, name) -> np.ndarray: """ - Returns the 3D binary mask of the ROI with the given input name + Returns the 3D binary mask of the ROI with the given input name. + + Axis order is ``(rows, columns, slices)`` ≡ ``(Y, X, Z)`` in DICOM + image-space terms. ``mask[:, :, k]`` selects slice ``k`` and is + directly compatible with ``matplotlib.pyplot.imshow``. Note that + ``rt_utils.image_helper.create_empty_series_mask`` allocates the + underlying array with dimensions ``(Columns, Rows, Slices)``; + ``cv2.fillPoly`` then writes into it at ``[y, x]`` indices, so the + populated semantic order is ``(rows=Y, columns=X, slices=Z)``. + For square images the two namings collapse, but for non-square + images the populated layout is what matters to consumers. """ for structure_roi in self.ds.StructureSetROISequence: diff --git a/setup.py b/setup.py index 5d525ed..387fedc 100644 --- a/setup.py +++ b/setup.py @@ -41,4 +41,15 @@ ], python_requires=">=3.7", install_requires=required, + extras_require={ + # Opt-in conformance gate against rtmask-conformance's analytic GT. + # Install via: pip install -e .[conformance] + # Requires Python >= 3.10 (rtmask-conformance constraint); pip will + # refuse to install the extra on older interpreters, leaving the + # base package usable. + "conformance": [ + "rtmask-conformance @ git+https://github.com/brianmanderson/RTMaskConformanceTest", + "SimpleITK>=2.4", + ], + }, ) diff --git a/tests/conformance.yaml b/tests/conformance.yaml new file mode 100644 index 0000000..500f029 --- /dev/null +++ b/tests/conformance.yaml @@ -0,0 +1,20 @@ +# Conformance threshold overrides for rt-utils vs rtmask-conformance defaults. +# +# Per-primitive overrides shallow-merge over `defaults`, which themselves +# shallow-merge over the package-shipped defaults. Document every relaxation +# with a date, the metric, and the path back to the published default — the +# header of DicomRTTool's tests/conformance.yaml is the canonical example: +# https://github.com/brianmanderson/Dicom_RT_and_Images_to_Mask/blob/main/tests/conformance.yaml +schema_version: 1 +primitives: + cube: + # 2026-05-07: rt-utils rasterizes contours via cv2.fillPoly, which is + # boundary-inclusive — every voxel touched by the polygon is filled. + # For the axis-aligned 60 mm cube on 1 mm voxels this gains roughly + # 3.4% volume from boundary pixels along each face (Dice ≈ 0.9835, + # volume_rel_err ≈ 3.36%). Surface DSC=1.0, HD95=1.0 mm, MSD=0.33 mm + # confirm the geometry is right; the volume gap is purely the boundary + # convention. Tighten back to defaults (dice 0.99, volume_rel_err 0.03) + # once the rasterizer honours a half-voxel-shrink. + dice: 0.98 + volume_rel_err: 0.04 diff --git a/tests/test_conformance.py b/tests/test_conformance.py new file mode 100644 index 0000000..57f23e7 --- /dev/null +++ b/tests/test_conformance.py @@ -0,0 +1,128 @@ +"""Conformance test: rt-utils vs RTMaskConformanceTest analytic ground truth. + +Runs only when the `conformance` extra is installed: + + pip install -e .[conformance] + pytest tests/test_conformance.py -v + +Without the extra, the module is skipped (importorskip), so default test +runs are unaffected. + +rt-utils returns masks as in-memory numpy arrays. Empirically (verified +against the analytic ground truth across all seven primitives) the +populated array's axis order is ``(Y, X, Z)``: although +``image_helper.create_empty_series_mask`` allocates it nominally as +``(Columns, Rows, Slices)``, the cv2.fillPoly pass that fills the array +operates in ``(row, col)`` ≡ ``(Y, X)`` space and leaves the data in +``(Y, X, Z)`` order. The fixture transposes via ``np.transpose(2, 0, 1)`` +to SimpleITK / NIfTI's ``(Z, Y, X)`` convention, copies the geometry from +the matching ground-truth NIfTI, and writes the prediction to disk so +``evaluate_one`` can do its geometry precheck and metric pass exactly as +it does for the file-based converters. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import numpy as np +import pytest + +# Skip cleanly if the conformance extra isn't installed, so that +# `pytest tests/` without the extra still passes. +rtmask_conformance = pytest.importorskip( # noqa: F841 + "rtmask_conformance", + reason="install the `conformance` extra: pip install -e .[conformance]", +) + +import SimpleITK as sitk # noqa: E402 + +from rtmask_conformance import CONFORMANCE_ROIS, generate_fixture, load_config # noqa: E402 +from rtmask_conformance.generate import GenerateOptions # noqa: E402 +from rtmask_conformance.verify import Status, evaluate_one # noqa: E402 + +from rt_utils import RTStructBuilder # noqa: E402 + +_CONFIG_YAML = Path(__file__).with_name("conformance.yaml") + + +@pytest.fixture(scope="session") +def conformance_fixture(tmp_path_factory: pytest.TempPathFactory) -> Path: + """Generate the synthetic CT + RTSTRUCT + analytic GT NIfTIs once per session. + + ``n_quadrature=2`` keeps the fixture build under ~30 s; the published + default of 8 is overkill for a CI gate. + """ + out = tmp_path_factory.mktemp("conformance_fixture") + generate_fixture(out, options=GenerateOptions(n_quadrature=2)) + return out + + +@pytest.fixture(scope="session") +def predictions( + conformance_fixture: Path, tmp_path_factory: pytest.TempPathFactory +) -> Path: + """Drive ``RTStructBuilder.create_from`` once per session, write each ROI's + mask to ``.nii.gz`` with the GT's geometry copied verbatim. + + Why we copy GT geometry rather than re-deriving it: rt-utils returns a + bare numpy array with no spatial metadata, while ``evaluate_one`` runs a + geometry precheck (origin / spacing / size / direction). The CT slices + in the fixture and the analytic GT NIfTIs share the same geometry by + construction, so copying the GT's metadata onto the prediction is + equivalent to re-deriving it from the CT — and one less place to drift. + + Axis order: rt-utils' populated mask is ``(Y, X, Z)`` (see module + docstring). SimpleITK's ``GetImageFromArray`` expects ``(Z, Y, X)``. + ``transpose(2, 0, 1)`` is the bridge. + """ + pred_dir = tmp_path_factory.mktemp("preds") + + rtstruct = RTStructBuilder.create_from( + dicom_series_path=str(conformance_fixture / "refct"), + rt_struct_path=str(conformance_fixture / "rtstruct" / "primitives_planar.dcm"), + ) + + gt_dir = conformance_fixture / "groundtruth" + for roi in rtstruct.get_roi_names(): + # Skip ROIs that aren't part of the conformance suite (defensive — the + # fixture only ships the seven, but a future fixture variant might add). + gt_path = gt_dir / f"{roi}.nii.gz" + if not gt_path.is_file(): + continue + + mask_yxz = rtstruct.get_roi_mask_by_name(roi) # bool (Y, X, Z) + mask_zyx = np.transpose(mask_yxz, (2, 0, 1)).astype(np.uint8) + + gt_img = sitk.ReadImage(str(gt_path)) + pred_img = sitk.GetImageFromArray(mask_zyx) + pred_img.CopyInformation(gt_img) + sitk.WriteImage(pred_img, str(pred_dir / f"{roi}.nii.gz")) + + return pred_dir + + +@pytest.fixture(scope="session") +def conformance_config(): + """Resolve thresholds: env var > tests/conformance.yaml > package defaults.""" + config_path = os.environ.get("RTMASK_CONFORMANCE_CONFIG") + if config_path is None and _CONFIG_YAML.is_file(): + config_path = str(_CONFIG_YAML) + return load_config(config_path) + + +@pytest.mark.parametrize("roi", CONFORMANCE_ROIS) +def test_conformance( + roi: str, conformance_fixture: Path, predictions: Path, conformance_config +) -> None: + pred = predictions / f"{roi}.nii.gz" + gt = conformance_fixture / "groundtruth" / f"{roi}.nii.gz" + result = evaluate_one(roi, pred, gt, conformance_config) + if result.status != Status.PASS: + pytest.fail( + f"{roi}: {result.status.value}\n" + f" violations: {result.violations}\n" + f" metrics: {result.metrics}\n" + f" thresholds: {result.thresholds}" + )