🐛 Fix term position weighting

sdk/nexent/core/nlp/tokenizer.py

@@ -36,24 +36,26 @@ def calculate_term_weights(text, use_idf=False, doc_freqs=None, total_docs=1):
     text = text.lower()
 
     # Tokenization with POS tagging
-    words = pseg.cut(text)
+    words = [
+        (word, flag)
+        for word, flag in pseg.cut(text)
+        if word not in analyse.default_tfidf.stop_words and word.strip()
+    ]
     term_stats = defaultdict(float)
     total_weight = 0.0
 
     # First pass: calculate term frequency + POS weight + position weight
     for idx, (word, flag) in enumerate(words):
-        # Filter out stop words and whitespace
-        if word not in analyse.default_tfidf.stop_words and word.strip():
-            # Get the first letter of POS tag (Chinese POS tagging convention)
-            pos = flag[0].lower()
-            # Get the base weight for the POS
-            pos_weight = POS_WEIGHTS.get(pos, 1.0)
-            # Position weight enhancement (words at the beginning and end of the sentence are more important)
-            position_factor = 1.2 if idx < 3 or idx > len(text) / 3 else 1.0
-            # Combined weight = POS weight * position factor
-            combined_weight = pos_weight * position_factor
-            term_stats[word] += combined_weight
-            total_weight += combined_weight
+        # Get the first letter of POS tag (Chinese POS tagging convention)
+        pos = flag[0].lower()
+        # Get the base weight for the POS
+        pos_weight = POS_WEIGHTS.get(pos, 1.0)
+        # Position weight enhancement (words at the beginning and end of the sentence are more important)
+        position_factor = 1.2 if idx < 3 or idx >= len(words) - 3 else 1.0
+        # Combined weight = POS weight * position factor
+        combined_weight = pos_weight * position_factor
+        term_stats[word] += combined_weight
+        total_weight += combined_weight
 
     # Calculate TF weight (term frequency weight)
     tf_weights = {term: weight / total_weight for term, weight in term_stats.items()}

test/sdk/core/nlp/test_tokenizer.py

@@ -0,0 +1,41 @@
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+
+REPO_ROOT = Path(__file__).resolve().parents[4]
+MODULE_PATH = REPO_ROOT / "sdk" / "nexent" / "core" / "nlp" / "tokenizer.py"
+
+
+def _load_tokenizer_module():
+    spec = importlib.util.spec_from_file_location("nexent_tokenizer", MODULE_PATH)
+    if spec is None or spec.loader is None:
+        raise RuntimeError("Could not load the tokenizer module")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+tokenizer = _load_tokenizer_module()
+
+
+def test_position_factor_uses_retained_token_positions(monkeypatch):
+    retained_tokens = [(f"term{i}", "n") for i in range(10)]
+    token_stream = [
+        (" ", "x"),
+        *retained_tokens[:5],
+        ("stop", "x"),
+        *retained_tokens[5:],
+    ]
+    monkeypatch.setattr(tokenizer.pseg, "cut", lambda _: iter(token_stream))
+    monkeypatch.setattr(tokenizer.analyse.default_tfidf, "stop_words", {"stop"})
+
+    weights = tokenizer.calculate_term_weights("a deliberately long raw input")
+
+    assert weights["term0"] == pytest.approx(1.0)
+    assert weights["term2"] == pytest.approx(1.0)
+    assert weights["term3"] == pytest.approx(1 / 1.2)
+    assert weights["term6"] == pytest.approx(1 / 1.2)
+    assert weights["term7"] == pytest.approx(1.0)
+    assert weights["term9"] == pytest.approx(1.0)