diff --git a/docker/validator/Dockerfile b/docker/validator/Dockerfile index 13454dd..3ac874d 100644 --- a/docker/validator/Dockerfile +++ b/docker/validator/Dockerfile @@ -20,10 +20,10 @@ COPY docker/validator/pyproject.toml docker/validator/uv.lock /app/ RUN uv sync --frozen --no-dev --no-install-project # Pre-cache SentenceTransformer models (avoids runtime download). The -# canonical Qwen3 model is always loaded; the bge-small bundle is for the -# optional ORO-1474 shadow scorer enabled via SHADOW_SCORE_MODEL at runtime. +# canonical Qwen3 model is always loaded; the bge-small + e5-base bundles +# are candidate shadow scorers enabled via SHADOW_SCORE_MODEL at runtime. ARG HF_TOKEN="" -RUN HF_HUB_ENABLE_HF_TRANSFER=0 HF_TOKEN=${HF_TOKEN} .venv/bin/python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('Qwen/Qwen3-Embedding-0.6B'); SentenceTransformer('BAAI/bge-small-en-v1.5')" +RUN HF_HUB_ENABLE_HF_TRANSFER=0 HF_TOKEN=${HF_TOKEN} .venv/bin/python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('Qwen/Qwen3-Embedding-0.6B'); SentenceTransformer('BAAI/bge-small-en-v1.5'); SentenceTransformer('intfloat/e5-base-v2')" # Copy validator code COPY subnet/ /app/subnet/ diff --git a/src/agent/rewards/orm.py b/src/agent/rewards/orm.py index f67c10f..d2e14b7 100644 --- a/src/agent/rewards/orm.py +++ b/src/agent/rewards/orm.py @@ -1,6 +1,7 @@ import json import logging import os +import time from collections import Counter SENTENCE_MODEL_NAME = "Qwen/Qwen3-Embedding-0.6B" @@ -75,12 +76,32 @@ def _log_shadow_sim(product_title: str, gt_title: str, canonical_sim: float) -> if shadow is None: return try: + t0 = time.perf_counter() product_emb = shadow.encode([product_title]) gt_emb = shadow.encode([gt_title]) + shadow_encode_ms = (time.perf_counter() - t0) * 1000 + t1 = time.perf_counter() shadow_sim = float(shadow.similarity(product_emb, gt_emb)[0][0]) + shadow_sim_ms = (time.perf_counter() - t1) * 1000 except Exception as exc: # noqa: BLE001 _logger.warning("Shadow sim failed for product=%r gt=%r: %s", product_title[:60], gt_title[:60], exc) return + + canonical = _get_sentence_model() + canonical_encode_ms = None + canonical_sim_ms = None + if canonical is not None: + try: + t0 = time.perf_counter() + c_product_emb = canonical.encode([product_title]) + c_gt_emb = canonical.encode([gt_title]) + canonical_encode_ms = (time.perf_counter() - t0) * 1000 + t1 = time.perf_counter() + canonical.similarity(c_product_emb, c_gt_emb) + canonical_sim_ms = (time.perf_counter() - t1) * 1000 + except Exception as exc: # noqa: BLE001 + _logger.warning("Canonical timing probe failed: %s", exc) + _logger.info( "shadow_sim %s", json.dumps( @@ -91,6 +112,14 @@ def _log_shadow_sim(product_title: str, gt_title: str, canonical_sim: float) -> "shadow_model": os.environ.get(SHADOW_MODEL_ENV, ""), "product_title": product_title, "gt_title": gt_title, + "shadow_encode_ms": round(shadow_encode_ms, 3), + "shadow_sim_ms": round(shadow_sim_ms, 3), + "canonical_encode_ms": ( + round(canonical_encode_ms, 3) if canonical_encode_ms is not None else None + ), + "canonical_sim_ms": ( + round(canonical_sim_ms, 3) if canonical_sim_ms is not None else None + ), } ), ) diff --git a/tests/test_scoring_perf.py b/tests/test_scoring_perf.py index a134060..871d3b6 100644 --- a/tests/test_scoring_perf.py +++ b/tests/test_scoring_perf.py @@ -93,6 +93,11 @@ def test_shadow_sim_logged_when_enabled(mock_get, mock_shadow, monkeypatch, capl assert payload["shadow_sim"] == pytest.approx(0.42) assert payload["shadow_model"] == "BAAI/bge-small-en-v1.5" assert payload["gt_title"] == "the gt title" + # Timing fields populated (real numbers, not None) when both models load. + assert isinstance(payload["shadow_encode_ms"], (int, float)) + assert isinstance(payload["shadow_sim_ms"], (int, float)) + assert isinstance(payload["canonical_encode_ms"], (int, float)) + assert isinstance(payload["canonical_sim_ms"], (int, float)) def test_shadow_sim_silent_when_disabled(monkeypatch, caplog):