diff --git a/.gitignore b/.gitignore index 0f32231..883875b 100644 --- a/.gitignore +++ b/.gitignore @@ -216,4 +216,7 @@ __marimo__/ .streamlit/secrets.toml # Secrest files -secret* \ No newline at end of file +secret* + +data/ +output/ \ No newline at end of file diff --git a/api/accent/align.py b/api/accent/align.py index c83e156..cb8ccaa 100644 --- a/api/accent/align.py +++ b/api/accent/align.py @@ -1,11 +1,12 @@ """Align Yahoo Furigana tokens with OJAD per-mora accent entries. -`align_accent()` walks the Yahoo token list and consumes OJAD entries -in order, emitting one `WordAccentResult` per Yahoo token. Numeric -tokens have no Yahoo furigana to compare against so they use a -length-anchor heuristic (stop when OJAD reaches the next Yahoo -token's reading); kana / kanji tokens use literal length + content -matching under kata2hira folding. +`align_accent()` builds a Needleman-Wunsch-style DP over +(yahoo_token, ojad_entry) pairs: for every Yahoo token we consider letting +it consume k contiguous OJAD entries (k โˆˆ [0, K_MAX]), with the per-token +cost depending on token shape (punct / numeric / kana) and edit distance +over rendaku-folded strings for kana tokens. This replaces the older +greedy aligner whose +1 fallback path cascaded a single mismatch into +type-0 fallback for every downstream token. Alignment itself uses `numeric_pattern` and `is_kana_or_kanji`. The adjacent `punctuation_marks`, `skip_marks`, and `clean_query` are @@ -128,207 +129,279 @@ def is_kana_or_kanji(char: Any) -> bool: return False -async def align_accent( - furigana_results: list[Any], ojad_results: list[dict[str, Any]] -) -> list[WordAccentResult]: - """Align yahoo furigana with OJAD results, return final accent marked result.""" - final_response_results = [] - ojad_idx_cnt = 0 - - logger.debug(f"๐Ÿ” [Data Check] First item:{furigana_results[0]}") - - for i, furigana_result in enumerate(furigana_results): - yahoo_furigana = furigana_result.furigana - yahoo_surface = furigana_result.surface - - yahoo_furigana_hira = jaconv.kata2hira(yahoo_furigana) - accents: list[AccentInfo] = [] - - logger.debug(f"Processing Yahoo Word [{i}]: {yahoo_surface} ({yahoo_furigana})") +# Yahoo returns "dictionary form" furigana (no rendaku), while OJAD returns the +# actually pronounced kana (with rendaku/sequential-voicing applied). When +# Yahoo says "ใตใ‚“ใ‹ใ‚“" and OJAD says "ใทใ‚“ใ‹ใ‚“", literal startswith / equality +# checks would never match and the alignment would cascade-fail. We compare +# under a normalisation that folds each voiced/half-voiced kana to its +# voiceless base, so ใทโ†”ใต, ใฐโ†”ใฏ, ใ”โ†”ใ“ etc. all alias together. +_VOICING_FOLD: dict[str, str] = { + "ใŒ": "ใ‹", "ใŽ": "ใ", "ใ": "ใ", "ใ’": "ใ‘", "ใ”": "ใ“", + "ใ–": "ใ•", "ใ˜": "ใ—", "ใš": "ใ™", "ใœ": "ใ›", "ใž": "ใ", + "ใ ": "ใŸ", "ใข": "ใก", "ใฅ": "ใค", "ใง": "ใฆ", "ใฉ": "ใจ", + "ใฐ": "ใฏ", "ใณ": "ใฒ", "ใถ": "ใต", "ใน": "ใธ", "ใผ": "ใป", + "ใฑ": "ใฏ", "ใด": "ใฒ", "ใท": "ใต", "ใบ": "ใธ", "ใฝ": "ใป", +} # fmt: skip + + +def _norm(s: str) -> str: + """Kataโ†’hira plus voicing fold for rendaku-tolerant alignment.""" + hira = jaconv.kata2hira(s) + return "".join(_VOICING_FOLD.get(c, c) for c in hira) + + +# --- DP aligner ---------------------------------------------------------------- +# +# The greedy aligner this replaced had two fatal failure modes: a numeric +# anchor that over-consumed when Yahoo and OJAD disagreed on a phrase +# boundary, and a fallback path that advanced OJAD by exactly +1 โ€” so a +# single mismatch cascaded into type-0 fallback for every downstream token. +# +# Instead we now build a Needleman-Wunsch-style DP over (yahoo_token, +# ojad_entry) pairs. Each cell dp[i][j] holds the minimum total cost to +# explain Yahoo tokens [0..i) using OJAD entries [0..j). For every (i, j) +# we try consuming k OJAD entries for token i with k โˆˆ [0, K_MAX]; the +# per-token cost depends on token shape (punctuation / numeric / kana) and +# uses edit distance over rendaku-folded strings for kana tokens. A bad +# token costs O(1); downstream tokens stay aligned. + +_K_MAX = 16 # max OJAD entries one Yahoo token can consume +_INF = float("inf") +_FALLBACK_COST = 3.0 # cost of giving up on a single token (k=0 for kana/numeric) +_OJAD_PUNCT_TEXTS = {"ใ€", "ใ€‚", ",", ".", "?", "!", "๏ผ", "๏ผŸ"} + + +# Substitutions are cheaper than insertions/deletions: a substitution keeps +# the yahooโ†”ojad mora-count alignment intact (the kind of mismatch we +# *expect* โ€” rendaku, reading variants like ็ญ‰โ†’ใจใ†/ใชใฉ), while ins/del +# means the two sources disagree on mora count, which is much less common +# and almost always a worse alignment. With sub<0.5, the DP correctly +# prefers a same-length span with two substitutions (cost 0.8) over a +# shorter span with one deletion (cost 1.0). This breaks the tie that was +# letting OJAD's `ใ†` from `็ญ‰โ†’ใจใ†` leak forward onto the next token. +_SUB_COST = 0.4 + + +def _edit_distance(a: str, b: str) -> float: + """Weighted Levenshtein with sub_cost=0.4, ins/del=1.0. + Used over rendaku-folded strings only.""" + if a == b: + return 0.0 + if not a: + return float(len(b)) + if not b: + return float(len(a)) + prev: list[float] = [float(j) for j in range(len(b) + 1)] + for i, ca in enumerate(a, 1): + curr: list[float] = [float(i)] + [0.0] * len(b) + for j, cb in enumerate(b, 1): + if ca == cb: + curr[j] = prev[j - 1] + else: + curr[j] = min( + prev[j - 1] + _SUB_COST, # substitute + prev[j] + 1.0, # delete from a + curr[j - 1] + 1.0, # insert into a + ) + prev = curr + return prev[-1] - # Identify if the word is numeric - is_numeric = bool(numeric_pattern.match(yahoo_surface)) - # ignore non-kana/kanji and non-numeric words - if ( - not furigana_result.subword - and any(not is_kana_or_kanji(chr) for chr in yahoo_furigana) - and not is_numeric - ): - logger.debug(" -> Skipped (Not Kana/Kanji)") - accents.append( +def _is_punct_token(furigana: str, is_numeric: bool) -> bool: + if is_numeric or not furigana: + return False + return all(not is_kana_or_kanji(c) for c in furigana) + + +def _match_cost( + token: Any, span_texts: list[str], is_numeric: bool, is_punct: bool +) -> float: + """Cost of letting `token` consume the given OJAD-text span.""" + k = len(span_texts) + concat = "".join(span_texts) + + if is_punct: + if k == 0: + return 0.0 + if k == 1: + stripped = span_texts[0].strip() + yahoo_stripped = token.furigana.strip() + # Free-consume only if the OJAD entry actually matches this + # token's punct (or is empty). Without this, two adjacent + # punct tokens (e.g. "ใ€‚" then "\n") could both consume the + # single OJAD "ใ€‚" at zero cost and DP would arbitrarily give + # it to the wrong one. + if not stripped: + return 0.0 + if stripped == yahoo_stripped: + return 0.0 + return _INF + + if is_numeric: + if k == 0: + return _FALLBACK_COST + # Numerics have no Yahoo furigana to compare against. Accept any + # reasonable count of morae; only penalise blatantly over-long spans. + upper = max(4, len(token.surface) * 4) + return 0.0 if k <= upper else float(k - upper) + + # Kana / kanji token: compare under rendaku fold. + if k == 0: + return _FALLBACK_COST + y_norm = _norm(token.furigana) + o_norm = _norm(concat) + # Cheap length pre-filter โ€” keeps the DP fast and prevents pathological + # "consume 12 OJAD entries to match a 2-mora Yahoo token" alignments. + if abs(len(y_norm) - len(o_norm)) > 3: + return _INF + return _edit_distance(y_norm, o_norm) + + +def _build_word_result(token: Any, ojad_span: list[dict[str, Any]]) -> WordAccentResult: + """Wrap an aligned (token, OJAD-span) pair into a WordAccentResult.""" + yahoo_surface = token.surface + yahoo_furigana = token.furigana + is_numeric = bool(numeric_pattern.match(yahoo_surface)) + subword = ( + [WordResult(furigana=s.furigana, surface=s.surface) for s in token.subword] + if token.subword + else [] + ) + + if not ojad_span: + # k=0 path: emit type-0 fallback so the downstream override pass and + # callers still see one AccentInfo per token. + return WordAccentResult( + surface=yahoo_surface, + furigana=yahoo_furigana, + accent=[ AccentInfo( - furigana=yahoo_surface, + furigana=yahoo_furigana, accent_marking_type=0, - length=len(yahoo_surface), - ) - ) - final_response_results.append( - WordAccentResult( - furigana=yahoo_furigana, surface=yahoo_surface, accent=accents + length=len(yahoo_furigana), ) - ) - - # Move OJAD index if skipped punctuation - if ojad_idx_cnt < len(ojad_results) and jaconv.kata2hira( - ojad_results[ojad_idx_cnt]["text"].strip() - ) in ["ใ€", "ใ€‚", ",", "."]: - ojad_idx_cnt += 1 - continue - - # Synchronize OJAD index - ojad_idx = ojad_idx_cnt - - # Check OJAD boundary - if ojad_idx >= len(ojad_results): - logger.warning(f" -> OJAD Index Out of Bounds ({ojad_idx})") - else: - logger.debug( - f"-> Comparing Yahoo '{yahoo_furigana_hira}'" - f" vs OJAD '{ojad_results[ojad_idx]['text']}'" - ) - - # Move non-numeric OJAD index to the matching position - if not is_numeric: - while ojad_idx < len(ojad_results) and not yahoo_furigana_hira.startswith( - jaconv.kata2hira(ojad_results[ojad_idx]["text"]) - ): - ojad_idx += 1 - - # catch the furigana from Yahoo with OJAD results - ojad_furigana = "" - temp_accents = [] # Use temp list to avoid partial data - - # Define anchor(next Yahoo furigana) for numeric mode - next_yahoo_furigana = None - if i + 1 < len(furigana_results): - next_yahoo_furigana = jaconv.kata2hira(furigana_results[i + 1].furigana) - - # Backup index - temp_ojad_idx = ojad_idx - - # Number mode: grab OJAD until the anchor - if is_numeric: - while temp_ojad_idx < len(ojad_results): - raw_text = ojad_results[temp_ojad_idx]["text"].strip() - ojad_text = jaconv.kata2hira(raw_text) - - # Stop if reached the anchor - if next_yahoo_furigana and next_yahoo_furigana.startswith(ojad_text): - break - - # Stop if consumed too much data - if len(ojad_furigana) > max(len(yahoo_surface) * 4, 12): - logger.warning( - f" -> Numeric consumption exceeded limit '{yahoo_surface}'." - ) - break - - ojad_furigana += ojad_text - temp_accents.append( - AccentInfo( - furigana=ojad_text, - accent_marking_type=ojad_results[temp_ojad_idx]["accent"], - length=len(ojad_text), - ) - ) - temp_ojad_idx += 1 - # Normal mode: grab OJAD until length match - else: - while len(ojad_furigana) < len(yahoo_furigana) and temp_ojad_idx < len( - ojad_results - ): - ojad_text = ojad_results[temp_ojad_idx]["text"] - ojad_furigana += ojad_text - temp_accents.append( - AccentInfo( - furigana=ojad_text, - accent_marking_type=ojad_results[temp_ojad_idx]["accent"], - length=len(ojad_text), - ) - ) - temp_ojad_idx += 1 - - # Final matching check - is_match = False - if is_numeric: - # Numeric mode: only check if OJAD has furigana grabbed - is_match = len(ojad_furigana) > 0 - else: - # Normal mode: check length and content - is_match = len(ojad_furigana) == len(yahoo_furigana) and jaconv.kata2hira( - ojad_furigana - ) == jaconv.kata2hira(yahoo_furigana) - - if is_match: - logger.debug(f" -> MATCHED! OJAD: {ojad_furigana}") - accents.extend(temp_accents) - - # Build final accent info list - accent_info_list = [] - for idx, accent in enumerate(accents): - accent_info_list.append( - AccentInfo( - furigana=accent.furigana, - accent_marking_type=accent.accent_marking_type, - length=accent.length, - ) + ], + subword=subword, + ) + + # Drop OJAD entries with empty text (phrase-boundary sentinels). They + # carry no audible mora and would surface as a stray "(โ€ฆ||0)" row. + voiced_span = [e for e in ojad_span if e["text"]] + if not voiced_span: + return WordAccentResult( + surface=yahoo_surface, + furigana=yahoo_furigana, + accent=[ + AccentInfo( + furigana=yahoo_furigana, + accent_marking_type=0, + length=len(yahoo_furigana), ) + ], + subword=subword, + ) + accents = [ + AccentInfo( + furigana=e["text"], + accent_marking_type=e["accent"], + length=len(e["text"]), + ) + for e in voiced_span + ] + # Numerics had no Yahoo furigana to begin with โ€” surface the OJAD reading. + display = "".join(e["text"] for e in voiced_span) if is_numeric else yahoo_furigana + return WordAccentResult( + surface=yahoo_surface, + furigana=display, + accent=accents, + subword=subword, + ) - ojad_idx_cnt = temp_ojad_idx # Update global index - display_furigana = ojad_furigana if is_numeric else yahoo_furigana +def _fallback_word(token: Any) -> WordAccentResult: + return _build_word_result(token, []) - # Build final response - if furigana_result.subword: - yahoo_subword = furigana_result.subword - logger.debug( - f"[Type Check] yahoo_subword element type: {type(yahoo_subword[0])}" - ) - logger.debug(f"[Data Check] yahoo_subword content: {yahoo_subword}") - final_response_results.append( - WordAccentResult( - furigana=display_furigana, - surface=yahoo_surface, - accent=accent_info_list, - subword=[ - WordResult(furigana=s.furigana, surface=s.surface) - for s in yahoo_subword - ], - ) - ) - else: - final_response_results.append( - WordAccentResult( - furigana=display_furigana, - surface=yahoo_surface, - accent=accent_info_list, - ) - ) - else: - # [ERROR BLOCK] - logger.error( - "-> MATCH FAILED." - f"Yahoo: {yahoo_furigana} vs OJAD Assembly: {ojad_furigana}" - ) - - # Fallback to Yahoo furigana with no accent info - accent_info = AccentInfo( - furigana=yahoo_furigana, - accent_marking_type=0, - length=len(yahoo_furigana), - ) - - final_response_results.append( - WordAccentResult( - furigana=yahoo_furigana, - surface=yahoo_surface, - accent=[accent_info], - ) - ) - # Move OJAD index to next item to avoid infinite loop - if ojad_idx_cnt < len(ojad_results): - ojad_idx_cnt += 1 +async def align_accent( + furigana_results: list[Any], ojad_results: list[dict[str, Any]] +) -> list[WordAccentResult]: + """Align yahoo furigana with OJAD per-moji entries via global DP. - return final_response_results + Returns one WordAccentResult per Yahoo token. Each token consumes a + (possibly empty) contiguous span of OJAD entries; the assignment that + minimises total mismatch cost wins. + """ + n = len(furigana_results) + m = len(ojad_results) + + if n == 0: + return [] + if m == 0: + return [_fallback_word(t) for t in furigana_results] + + # Pre-compute per-token classification and OJAD texts. + token_kinds: list[tuple[bool, bool]] = [] + for t in furigana_results: + is_num = bool(numeric_pattern.match(t.surface)) + is_pct = _is_punct_token(t.furigana, is_num) + token_kinds.append((is_num, is_pct)) + ojad_texts = [e["text"] for e in ojad_results] + + # dp[i][j] = best cost aligning tokens [0..i) to ojad entries [0..j). + dp: list[list[float]] = [[_INF] * (m + 1) for _ in range(n + 1)] + back: list[list[int]] = [[-1] * (m + 1) for _ in range(n + 1)] + dp[0][0] = 0.0 + + for i in range(n): + token = furigana_results[i] + is_num, is_pct = token_kinds[i] + for j in range(m + 1): + base = dp[i][j] + if base == _INF: + continue + k_limit = min(_K_MAX, m - j) + for k in range(0, k_limit + 1): + cost = _match_cost(token, ojad_texts[j : j + k], is_num, is_pct) + if cost == _INF: + continue + new_cost = base + cost + if new_cost < dp[i + 1][j + k]: + dp[i + 1][j + k] = new_cost + back[i + 1][j + k] = j + + # Pick the best terminal state. Prefer fully consuming OJAD; otherwise + # take the cheapest end (trailing empty entries get inherited for free + # by the previous token's span since their text contributes nothing to + # edit distance). + best_j = m + best_cost = dp[n][m] + if best_cost == _INF: + for j in range(m + 1): + if dp[n][j] < best_cost: + best_cost = dp[n][j] + best_j = j + + if best_cost == _INF: + logger.error( + "DP alignment found no valid path (n=%d, m=%d); falling back per token.", + n, + m, + ) + return [_fallback_word(t) for t in furigana_results] + + # Backtrack to recover the OJAD span each token consumed. + spans: list[tuple[int, int]] = [(0, 0)] * n + cur_j = best_j + for i in range(n, 0, -1): + prev_j = back[i][cur_j] + if prev_j < 0: + logger.error("DP backtrace broken at i=%d, j=%d", i, cur_j) + return [_fallback_word(t) for t in furigana_results] + spans[i - 1] = (prev_j, cur_j) + cur_j = prev_j + + logger.debug("DP alignment cost=%.2f spans=%s", best_cost, spans) + return [ + _build_word_result(furigana_results[i], ojad_results[s:e]) + for i, (s, e) in enumerate(spans) + ] diff --git a/api/accent/pipeline.py b/api/accent/pipeline.py index 58ff7e7..49c73ec 100644 --- a/api/accent/pipeline.py +++ b/api/accent/pipeline.py @@ -1,34 +1,179 @@ """MarkAccent orchestrator. -Threads the three data-layer modules together: +Threads the data-layer modules together and applies the surface-level +regex override layer on either side of the OJAD alignment: + 1. `furigana.fetch_furigana` โ€” tokenise + read with Yahoo Furigana - 2. `ojad.get_ojad_result` โ€” pull per-mora pitch contour from OJAD - 3. `align.align_accent` โ€” match tokens โ†” OJAD spans โ†’ WordAccentResult + 2. `reading_overrides.apply_furigana_overrides` โ€” fix date / weekday- + bracket readings BEFORE alignment so OJAD's numeric-anchor logic + doesn't cascade-fail on overridden spans + 3. `ojad.get_ojad_result` โ€” pull per-mora pitch contour from OJAD + 4. `align.align_accent` โ€” DP-match tokens โ†” OJAD spans + 5. `reading_overrides.apply_accent_overrides` โ€” re-apply overrides + over (furigana, accent) so the final response stays consistent + +Plus pre-/post-processing helpers for URL stripping, non-Japanese +short-circuit, sentence splitting (used by the streaming endpoint), +and the streaming chunk-fanout itself. -The route handler in `routes.py` wraps this with FastAPI request handling. +The route handlers in `routes.py` wrap this with FastAPI request handling. """ from __future__ import annotations +import asyncio +import json import logging +import re +from typing import Any, AsyncIterator import httpx import neologdn from api.accent.align import align_accent from api.accent.furigana import fetch_furigana -from api.accent.models import AccentResponse, ErrorInfo +from api.accent.models import ( + AccentResponse, + ErrorInfo, + Request, + WordAccentResult, +) from api.accent.ojad import get_ojad_result +from api.accent.reading_overrides import ( + apply_accent_overrides, + apply_furigana_overrides, +) logger = logging.getLogger("api") +# Hiragana / katakana / CJK Unified Ideographs (incl. Extension A). A +# chunk with no chars in this set is treated as pure English / code / +# markdown / URL โ€” pipeline is skipped entirely and the line is echoed +# back verbatim so document reconstruction still works. +_CJK_RE = re.compile( + "[" + "ใ€-ใ‚Ÿ" # Hiragana + "ใ‚ -ใƒฟ" # Katakana + "ใ€-ไถฟ" # CJK Unified Ideographs Extension A + "ไธ€-้ฟฟ" # CJK Unified Ideographs + "]" +) + +# Sentence terminators that close a Japanese clause: kuten (ใ€‚), full-width +# question (๏ผŸ), full-width exclamation (๏ผ), and full-width period (๏ผŽ). +# ASCII `.!?` are intentionally excluded โ€” they appear in abbreviations, +# decimals, and code/identifier fragments that we don't want to split on. +# A zero-width split (lookbehind) keeps the terminator attached to the +# preceding sentence so accent prediction still sees the clause boundary. +_SENTENCE_SPLIT_RE = re.compile("(?<=[ใ€‚๏ผ๏ผŸ๏ผŽ])") + +# URLs are stripped before the pipeline runs. OJAD's phrasing scraper +# produces only noise for Latin punctuation runs, and Yahoo's tokenizer +# can fragment a URL across several alphabet/symbol tokens โ€” both drag +# the alignment DP off-rail for the surrounding Japanese. We swap each +# URL for one fixed-string placeholder (which Yahoo keeps as a single +# "alphabet" word), run the pipeline, then walk the result and restore +# the originals in order. +# URL body stops at whitespace, any Japanese char (so `โ€ฆใฏhttps://x.jp/aใงใ™` +# strips just the URL, leaving `ใงใ™` to be processed), or common quoting +# punctuation `,()<>[]"'` (so `(https://x.jp)` strips just the URL). +_URL_RE = re.compile(r"https?://[^\sใ€€-้ฟฟ,()<>\[\]\"']+") +_URL_PLACEHOLDER = "URLPLACEHOLDER" + + +def _has_japanese(text: str) -> bool: + """True if `text` contains any hiragana, katakana, or CJK ideograph.""" + return bool(_CJK_RE.search(text)) + + +def _split_sentences(line: str) -> list[str]: + """Split a line into sentence-sized chunks for parallel processing. + + OJAD's phrasing module degrades badly on long inputs (a single + misaligned mora can cascade across the whole paragraph), and the + streaming endpoint can't parallelise within a `\\n`-delimited chunk. + Splitting on full-width sentence terminators fixes both: each sentence + is short enough for OJAD to handle reliably, and they fan out across + the in-flight Semaphore. + """ + return [s for s in _SENTENCE_SPLIT_RE.split(line) if s.strip()] + + +def _strip_urls(text: str) -> tuple[str, list[str]]: + """Replace each URL with `_URL_PLACEHOLDER`, returning URLs in order.""" + urls: list[str] = [] + + def repl(m: re.Match[str]) -> str: + urls.append(m.group(0)) + return _URL_PLACEHOLDER + + return _URL_RE.sub(repl, text), urls + + +def _restore_urls( + result: list[WordAccentResult], urls: list[str] +) -> list[WordAccentResult]: + """Swap placeholder tokens in `result` back to their original URLs.""" + if not urls: + return result + it = iter(urls) + out: list[WordAccentResult] = [] + for w in result: + if w.surface == _URL_PLACEHOLDER: + url = next(it, None) + if url is None: + # Placeholder count exceeded URL count: leave the token + # untouched. Indicates a Yahoo tokenization surprise; the + # output is still readable. + out.append(w) + continue + out.append( + WordAccentResult(surface=url, furigana=url, accent=[], subword=[]) + ) + else: + out.append(w) + return out + + async def process_accent_chunk(text: str, client: httpx.AsyncClient) -> AccentResponse: - """Run the full MarkAccent pipeline on a single chunk of text.""" + """Run the full MarkAccent pipeline on a single chunk of text. + + Shared by `/api/MarkAccent/` (whole input as one chunk) and + `/api/MarkAccent/stream/` (one call per `\\n`/sentence-split piece). + """ try: query_text = neologdn.normalize(text, tilde="normalize") - furigana_response = await fetch_furigana(query_text, client) + # Strip URLs first so a pure-URL line is detected as non-Japanese + # by the language check below and short-circuits the pipeline. + stripped_text, urls = _strip_urls(query_text) + + # No hiragana/katakana/kanji outside URLs โ€” passthrough the line + # as a single token. Callers reconstructing the document still + # see the chunk in the stream; we just skip the Yahoo + OJAD + # round-trips entirely. + if not _has_japanese(stripped_text): + return AccentResponse( + status=200, + result=[ + WordAccentResult( + surface=query_text, + furigana=query_text, + accent=[], + subword=[], + ) + ], + error=None, + ) + + # Apply furigana overrides BEFORE alignment: many of the overrides + # (e.g. "4ๆ—ฅ"โ†’"ใ‚ˆใฃใ‹", "27ๆ—ฅ"โ†’"ใซใ˜ใ‚…ใ†ใ—ใกใซใก") merge a numeric + # surface with the counter into one token whose furigana matches what + # OJAD reads as a single phrase. align_accent's numeric-anchor logic + # otherwise cascades-fails on these inputs because numeric tokens lack + # any Yahoo furigana for OJAD to align against. + furigana_response = await fetch_furigana(stripped_text, client) # Check yahoo furigana response if furigana_response.status != 200 or not furigana_response.result: @@ -39,19 +184,85 @@ async def process_accent_chunk(text: str, client: httpx.AsyncClient) -> AccentRe error=furigana_response.error, ) - furigana_results = furigana_response.result + furigana_results = apply_furigana_overrides(furigana_response.result) logger.debug(f"Yahoo Results Count: {len(furigana_results)}") - _ojad_surface, ojad_results = await get_ojad_result(query_text, client) + _ojad_surface, ojad_results = await get_ojad_result(stripped_text, client) final_results = await align_accent(furigana_results, ojad_results) + final_results = apply_accent_overrides(final_results) + final_results = _restore_urls(final_results, urls) return AccentResponse(status=200, result=final_results) except Exception as e: logger.exception(f"Unexpected error occurred: {text}") + # Some httpx exceptions (PoolTimeout, ReadTimeout) have empty + # str(); fall back to the type name so the client sees something. + detail = str(e) or repr(e) or type(e).__name__ return AccentResponse( status=500, result=None, - error=ErrorInfo(code=500, message=f"Error: {e}"), + error=ErrorInfo(code=500, message=f"Error: {detail}"), ) + + +# Streaming endpoint: OJAD's u-tokyo backend and (to a lesser extent) Yahoo's +# furigana API both fall over when hit with 30+ parallel scrapes โ€” the symptom +# was most chunks of a long document returning empty-string httpx errors. Cap +# in-flight work so well-behaved inputs still parallelise (a 4-chunk +# paragraph fans out fully) without hammering the upstream services. +_STREAM_CONCURRENCY = 4 + + +async def stream_accent_chunks( + request: Request, client: httpx.AsyncClient +) -> AsyncIterator[bytes]: + """Yield one NDJSON line per (line_idx, sub_idx) chunk in input order. + + Each emitted object carries `{"chunk": line_idx, "subchunk": sub_idx}`: + `line_idx` is the original `\\n`-split index (blank lines are dropped from + the stream); `sub_idx` distinguishes sentences inside one line. A line + with no terminator yields one subchunk with `sub_idx=0`. + """ + # (line_idx, sub_idx, text). Long paragraphs are split into sentence- + # sized chunks because OJAD's phrasing predictor degrades on long + # inputs and a single misalignment used to cascade across the whole + # paragraph. Splitting also fans the work out under the semaphore. + chunks: list[tuple[int, int, str]] = [] + for line_idx, line in enumerate(request.text.split("\n")): + if not line.strip(): + continue + for sub_idx, sentence in enumerate(_split_sentences(line)): + chunks.append((line_idx, sub_idx, sentence)) + + if not chunks: + return + + semaphore = asyncio.Semaphore(_STREAM_CONCURRENCY) + + async def run_chunk(line: str) -> AccentResponse: + async with semaphore: + return await process_accent_chunk(line, client) + + tasks = [asyncio.create_task(run_chunk(text)) for _, _, text in chunks] + # Yield in input order so the client renders chunks monotonically. + for (chunk_idx, sub_idx, _text), task in zip(chunks, tasks): + try: + resp = await task + payload: dict[str, Any] = { + "chunk": chunk_idx, + "subchunk": sub_idx, + **resp.model_dump(), + } + except Exception as exc: + logger.exception(f"Streaming chunk {chunk_idx}.{sub_idx} failed") + detail = str(exc) or repr(exc) or type(exc).__name__ + payload = { + "chunk": chunk_idx, + "subchunk": sub_idx, + "status": 500, + "result": None, + "error": {"code": 500, "message": f"Error: {detail}"}, + } + yield (json.dumps(payload, ensure_ascii=False) + "\n").encode("utf-8") diff --git a/api/accent/reading_overrides.py b/api/accent/reading_overrides.py new file mode 100644 index 0000000..d65bae8 --- /dev/null +++ b/api/accent/reading_overrides.py @@ -0,0 +1,457 @@ +""" +Predefined regex override layer for Yahoo Furigana / Suzuki-kun (OJAD) accent +results. + +Yahoo's furigana service is context-blind โ€” it gets common date and weekday- +bracket readings wrong (e.g. "5ๆ—ฅ" โ†’ ใซใก instead of ใ„ใคใ‹, "(ๅœŸ)" โ†’ ใคใก +instead of ใฉ). We post-process Yahoo's tokenised response with a list of +`FuriganaOverride` entries: each entry is a regex against the concatenated +surface text, plus the replacement tokens that should appear instead. + +The same overrides are applied a second time after OJAD alignment, replacing +both furigana and accent in one go. + +Patterns are written with character classes that accept half-width, full-width, +and kanji-numeral variants of the same surface, so "3ๆœˆ5ๆ—ฅ(ๅœŸ)" / "๏ผ“ๆœˆ๏ผ•ๆ—ฅ๏ผˆๅœŸ๏ผ‰" +/ "ไธ‰ๆœˆไบ”ๆ—ฅ๏ผˆๅœŸ๏ผ‰" all trigger the same overrides. + +If a match doesn't fall on Yahoo's token boundaries we log a warning and leave +the match alone โ€” Yahoo's result passes through unchanged. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass +from typing import Callable, TypeVar + +from api.accent.models import AccentInfo, WordAccentResult, WordResult + +logger = logging.getLogger("api") + + +@dataclass(frozen=True) +class ReplacementToken: + """One token in an override's replacement list. + + `surface=None` means: inherit the surface from the matched substring (full + text for a single-token replacement, or per-position when the replacement + count equals the match length โ€” see _resolve_surface). + + `furigana=None` means: echo the resolved surface (useful for non-kana + positions like brackets, mirroring Yahoo's own fallback in + `api/accent/furigana.py`). + + `accent` is a per-moji sequence of (kana, accent_marking_type) โ€” 0=none, + 1=heiban, 2=fall โ€” matching the existing AccentInfo schema. Empty tuple โ†’ + fall back to a single accent_marking_type=0 entry covering the whole + furigana. + """ + + furigana: str | None = None + surface: str | None = None + accent: tuple[tuple[str, int], ...] = () + + +@dataclass(frozen=True) +class FuriganaOverride: + pattern: re.Pattern[str] + replacements: tuple[ReplacementToken, ...] + description: str = "" + + +# accent_marking_type values (mirror AccentInfo) +_NONE, _HEIBAN, _FALL = 0, 1, 2 + +# Numeric / kanji-numeral class used in lookbehind & lookahead so partial +# matches don't fire (e.g. avoid "11ๆ—ฅ" โ†’ "1ๆ—ฅ" or "ไบŒๅไบ”ๆ—ฅ" โ†’ "ไบ”ๆ—ฅ"). +_DIGIT_CLASS = r"\dไธ€ไบŒไธ‰ๅ››ไบ”ๅ…ญไธƒๅ…ซไนๅ็™พๅƒ" +_NOT_NUM_BEHIND = rf"(? tuple[tuple[str, int], ...]: + return tuple((c, t) for c in text) + + +def _atamadaka_seq(text: str) -> tuple[tuple[str, int], ...]: + if not text: + return () + return ((text[0], _FALL),) + tuple((c, _HEIBAN) for c in text[1:]) + + +# Numeric variant helpers: keep N-prefixed patterns (Nๆ—ฅ, Nๆ—ฅ้–“, Nๆญณ) from +# repeating the (arabic, fullwidth, kanji) triple per row. + +_FULLWIDTH_TRANS = str.maketrans("0123456789", "๏ผ๏ผ‘๏ผ’๏ผ“๏ผ”๏ผ•๏ผ–๏ผ—๏ผ˜๏ผ™") + + +def _int_to_kanji(n: int) -> str: + """Traditional kanji numeral for n (1-99). + + Examples: 1โ†’'ไธ€', 10โ†’'ๅ', 14โ†’'ๅๅ››', 20โ†’'ไบŒๅ', 24โ†’'ไบŒๅๅ››', 31โ†’'ไธ‰ๅไธ€'. + """ + if not 1 <= n <= 99: + raise ValueError(f"_int_to_kanji supports 1-99, got {n}") + digits = "ใ€‡ไธ€ไบŒไธ‰ๅ››ไบ”ๅ…ญไธƒๅ…ซไน" + tens, ones = divmod(n, 10) + if tens == 0: + return digits[ones] + tens_part = "ๅ" if tens == 1 else digits[tens] + "ๅ" + return tens_part + (digits[ones] if ones else "") + + +def _numeric_pattern(n: int) -> str: + """Regex alternation matching n in arabic / full-width / traditional-kanji. + + Alternatives are emitted longest-first so multi-char kanji forms like + 'ไบŒๅๅ››' aren't shadowed by their numeric-form prefixes. + """ + arabic = str(n) + fullwidth = arabic.translate(_FULLWIDTH_TRANS) + kanji = _int_to_kanji(n) + variants = sorted({arabic, fullwidth, kanji}, key=len, reverse=True) + return "(?:" + "|".join(variants) + ")" + + +def _day_of_week_overrides() -> list[FuriganaOverride]: + readings: list[tuple[str, str]] = [ + ("ๆœˆ", "ใ’ใค"), + ("็ซ", "ใ‹"), + ("ๆฐด", "ใ™ใ„"), + ("ๆœจ", "ใ‚‚ใ"), + ("้‡‘", "ใใ‚“"), + ("ๅœŸ", "ใฉ"), + ("ๆ—ฅ", "ใซใก"), + ] + out: list[FuriganaOverride] = [] + for kanji, reading in readings: + # surface=None on all three lets _resolve_surface inherit per-position + # from the matched substring, preserving half-width vs full-width + # brackets in the input. furigana=None on the brackets echoes their + # surface (Yahoo also returns the bracket char as its own furigana). + out.append( + FuriganaOverride( + pattern=re.compile(rf"[(๏ผˆ]{kanji}[)๏ผ‰]"), + replacements=( + ReplacementToken(), # left bracket: both inherit + ReplacementToken(furigana=reading, accent=_moji_seq(reading)), + ReplacementToken(), # right bracket: both inherit + ), + description=f"ๆ›œๆ—ฅ ({kanji})", + ) + ) + return out + + +def _date_overrides() -> list[FuriganaOverride]: + # 1-10ๆ—ฅ, 14ๆ—ฅ, 20ๆ—ฅ, 24ๆ—ฅ are irregular (ใคใ„ใŸใก, ใตใคใ‹, ..., ใฏใคใ‹). + # 11-31ๆ—ฅ are regular (ใ˜ใ‚…ใ†ใ„ใกใซใก etc.) but Yahoo often returns the + # literal digits as their own "furigana" for numeric tokens; the accent + # endpoint's align_accent also frequently misaligns numeric spans. + # Listing every day-of-month here gives both endpoints a deterministic + # reading + accent for any date. + # + # Only 1ๆ—ฅ (ใคใ„ใŸใก) is atamadaka โ€” the rest sit in heiban-style. + readings: list[tuple[int, str, tuple[tuple[str, int], ...]]] = [ + (1, "ใคใ„ใŸใก", _atamadaka_seq("ใคใ„ใŸใก")), + (2, "ใตใคใ‹", _moji_seq("ใตใคใ‹")), + (3, "ใฟใฃใ‹", _moji_seq("ใฟใฃใ‹")), + (4, "ใ‚ˆใฃใ‹", _moji_seq("ใ‚ˆใฃใ‹")), + (5, "ใ„ใคใ‹", _moji_seq("ใ„ใคใ‹")), + (6, "ใ‚€ใ„ใ‹", _moji_seq("ใ‚€ใ„ใ‹")), + (7, "ใชใฎใ‹", _moji_seq("ใชใฎใ‹")), + (8, "ใ‚ˆใ†ใ‹", _moji_seq("ใ‚ˆใ†ใ‹")), + (9, "ใ“ใ“ใฎใ‹", _moji_seq("ใ“ใ“ใฎใ‹")), + (10, "ใจใŠใ‹", _moji_seq("ใจใŠใ‹")), + (11, "ใ˜ใ‚…ใ†ใ„ใกใซใก", _moji_seq("ใ˜ใ‚…ใ†ใ„ใกใซใก")), + (12, "ใ˜ใ‚…ใ†ใซใซใก", _moji_seq("ใ˜ใ‚…ใ†ใซใซใก")), + (13, "ใ˜ใ‚…ใ†ใ•ใ‚“ใซใก", _moji_seq("ใ˜ใ‚…ใ†ใ•ใ‚“ใซใก")), + (14, "ใ˜ใ‚…ใ†ใ‚ˆใฃใ‹", _moji_seq("ใ˜ใ‚…ใ†ใ‚ˆใฃใ‹")), + (15, "ใ˜ใ‚…ใ†ใ”ใซใก", _moji_seq("ใ˜ใ‚…ใ†ใ”ใซใก")), + (16, "ใ˜ใ‚…ใ†ใ‚ใใซใก", _moji_seq("ใ˜ใ‚…ใ†ใ‚ใใซใก")), + (17, "ใ˜ใ‚…ใ†ใ—ใกใซใก", _moji_seq("ใ˜ใ‚…ใ†ใ—ใกใซใก")), + (18, "ใ˜ใ‚…ใ†ใฏใกใซใก", _moji_seq("ใ˜ใ‚…ใ†ใฏใกใซใก")), + (19, "ใ˜ใ‚…ใ†ใใซใก", _moji_seq("ใ˜ใ‚…ใ†ใใซใก")), + (20, "ใฏใคใ‹", _moji_seq("ใฏใคใ‹")), + (21, "ใซใ˜ใ‚…ใ†ใ„ใกใซใก", _moji_seq("ใซใ˜ใ‚…ใ†ใ„ใกใซใก")), + (22, "ใซใ˜ใ‚…ใ†ใซใซใก", _moji_seq("ใซใ˜ใ‚…ใ†ใซใซใก")), + (23, "ใซใ˜ใ‚…ใ†ใ•ใ‚“ใซใก", _moji_seq("ใซใ˜ใ‚…ใ†ใ•ใ‚“ใซใก")), + (24, "ใซใ˜ใ‚…ใ†ใ‚ˆใฃใ‹", _moji_seq("ใซใ˜ใ‚…ใ†ใ‚ˆใฃใ‹")), + (25, "ใซใ˜ใ‚…ใ†ใ”ใซใก", _moji_seq("ใซใ˜ใ‚…ใ†ใ”ใซใก")), + (26, "ใซใ˜ใ‚…ใ†ใ‚ใใซใก", _moji_seq("ใซใ˜ใ‚…ใ†ใ‚ใใซใก")), + (27, "ใซใ˜ใ‚…ใ†ใ—ใกใซใก", _moji_seq("ใซใ˜ใ‚…ใ†ใ—ใกใซใก")), + (28, "ใซใ˜ใ‚…ใ†ใฏใกใซใก", _moji_seq("ใซใ˜ใ‚…ใ†ใฏใกใซใก")), + (29, "ใซใ˜ใ‚…ใ†ใใซใก", _moji_seq("ใซใ˜ใ‚…ใ†ใใซใก")), + (30, "ใ•ใ‚“ใ˜ใ‚…ใ†ใซใก", _moji_seq("ใ•ใ‚“ใ˜ใ‚…ใ†ใซใก")), + (31, "ใ•ใ‚“ใ˜ใ‚…ใ†ใ„ใกใซใก", _moji_seq("ใ•ใ‚“ใ˜ใ‚…ใ†ใ„ใกใซใก")), + ] + return [ + FuriganaOverride( + pattern=re.compile( + rf"{_NOT_NUM_BEHIND}{_numeric_pattern(n)}ๆ—ฅ{_NOT_NUM_AHEAD}" + ), + replacements=(ReplacementToken(furigana=furigana, accent=accent),), + description=f"็‰นๆฎŠๆ—ฅๆœŸ {n}ๆ—ฅ", + ) + for n, furigana, accent in readings + ] + + +def _duration_overrides() -> list[FuriganaOverride]: + """Nๆ—ฅ้–“ (counter for days as a duration) expansions. + + Yahoo tokenises e.g. `1ๆ—ฅ้–“` as [`1`, `ๆ—ฅ้–“`] and gives the numeric + token no furigana (its result is just the literal digit), so the + furigana endpoint surfaces unreadable output like `1ใซใกใ‹ใ‚“`. We + override the full Nๆ—ฅ้–“ span so the user sees a complete reading. + + Most readings are the existing date reading + `ใ‹ใ‚“`, with two + intentional deviations: + - `1ๆ—ฅ้–“` โ†’ `ใ„ใกใซใกใ‹ใ‚“` (NOT `ใคใ„ใŸใกใ‹ใ‚“` โ€” the 1st-of-month + reading is impossible when 1ๆ—ฅ is a duration). + - `7ๆ—ฅ้–“` โ†’ `ใ—ใกใซใกใ‹ใ‚“` (preferred in modern technical writing + over the older `ใชใฎใ‹ใ‹ใ‚“`). + """ + readings: list[tuple[int, str]] = [ + (1, "ใ„ใกใซใกใ‹ใ‚“"), + (2, "ใตใคใ‹ใ‹ใ‚“"), + (3, "ใฟใฃใ‹ใ‹ใ‚“"), + (4, "ใ‚ˆใฃใ‹ใ‹ใ‚“"), + (5, "ใ„ใคใ‹ใ‹ใ‚“"), + (6, "ใ‚€ใ„ใ‹ใ‹ใ‚“"), + (7, "ใ—ใกใซใกใ‹ใ‚“"), + (8, "ใ‚ˆใ†ใ‹ใ‹ใ‚“"), + (9, "ใ“ใ“ใฎใ‹ใ‹ใ‚“"), + (10, "ใจใŠใ‹ใ‹ใ‚“"), + (11, "ใ˜ใ‚…ใ†ใ„ใกใซใกใ‹ใ‚“"), + (12, "ใ˜ใ‚…ใ†ใซใซใกใ‹ใ‚“"), + (13, "ใ˜ใ‚…ใ†ใ•ใ‚“ใซใกใ‹ใ‚“"), + (14, "ใ˜ใ‚…ใ†ใ‚ˆใฃใ‹ใ‹ใ‚“"), + (15, "ใ˜ใ‚…ใ†ใ”ใซใกใ‹ใ‚“"), + (16, "ใ˜ใ‚…ใ†ใ‚ใใซใกใ‹ใ‚“"), + (17, "ใ˜ใ‚…ใ†ใ—ใกใซใกใ‹ใ‚“"), + (18, "ใ˜ใ‚…ใ†ใฏใกใซใกใ‹ใ‚“"), + (19, "ใ˜ใ‚…ใ†ใใซใกใ‹ใ‚“"), + (20, "ใฏใคใ‹ใ‹ใ‚“"), + (21, "ใซใ˜ใ‚…ใ†ใ„ใกใซใกใ‹ใ‚“"), + (22, "ใซใ˜ใ‚…ใ†ใซใซใกใ‹ใ‚“"), + (23, "ใซใ˜ใ‚…ใ†ใ•ใ‚“ใซใกใ‹ใ‚“"), + (24, "ใซใ˜ใ‚…ใ†ใ‚ˆใฃใ‹ใ‹ใ‚“"), + (25, "ใซใ˜ใ‚…ใ†ใ”ใซใกใ‹ใ‚“"), + (26, "ใซใ˜ใ‚…ใ†ใ‚ใใซใกใ‹ใ‚“"), + (27, "ใซใ˜ใ‚…ใ†ใ—ใกใซใกใ‹ใ‚“"), + (28, "ใซใ˜ใ‚…ใ†ใฏใกใซใกใ‹ใ‚“"), + (29, "ใซใ˜ใ‚…ใ†ใใซใกใ‹ใ‚“"), + (30, "ใ•ใ‚“ใ˜ใ‚…ใ†ใซใกใ‹ใ‚“"), + (31, "ใ•ใ‚“ใ˜ใ‚…ใ†ใ„ใกใซใกใ‹ใ‚“"), + ] + return [ + FuriganaOverride( + pattern=re.compile(rf"{_NOT_NUM_BEHIND}{_numeric_pattern(n)}ๆ—ฅ้–“"), + replacements=( + ReplacementToken(furigana=furigana, accent=_moji_seq(furigana)), + ), + description=f"ๆœŸ้–“ {n}ๆ—ฅ้–“", + ) + for n, furigana in readings + ] + + +def _age_overrides() -> list[FuriganaOverride]: + """Irregular age readings. + + 20ๆญณ / ไบŒๅๆญณ (and the casual ๆ‰ variant) โ†’ ใฏใŸใก, not the regular + ใซใ˜ใ‚…ใฃใ•ใ„. Only 20 is irregular for ages; the rest are systematic + so we don't need a 1-99 table here. + """ + return [ + FuriganaOverride( + pattern=re.compile( + rf"{_NOT_NUM_BEHIND}{_numeric_pattern(20)}[ๆญณๆ‰]{_NOT_NUM_AHEAD}" + ), + replacements=( + ReplacementToken(furigana="ใฏใŸใก", accent=_atamadaka_seq("ใฏใŸใก")), + ), + description="20ๆญณ โ†’ ใฏใŸใก", + ), + ] + + +# Order matters: _collect_matches breaks ties on (start, -length) and +# discards anything overlapping an earlier pick. `Nๆ—ฅ้–“` (3-4 chars) is +# strictly longer than `Nๆ—ฅ` at the same start, so duration entries +# automatically win over date entries for the same N when ้–“ follows. +OVERRIDES: list[FuriganaOverride] = ( + _day_of_week_overrides() + + _duration_overrides() + + _date_overrides() + + _age_overrides() +) + + +# ---------- Apply algorithm (shared between furigana & accent variants) ---------- + + +@dataclass +class _Match: + start: int + end: int + override: FuriganaOverride + + +def _collect_matches(text: str) -> list[_Match]: + """Run every override's regex and return non-overlapping matches. + + Earlier start wins; ties broken by longer match. + """ + raw: list[_Match] = [] + for ov in OVERRIDES: + for rm in ov.pattern.finditer(text): + raw.append(_Match(start=rm.start(), end=rm.end(), override=ov)) + raw.sort(key=lambda x: (x.start, -(x.end - x.start))) + chosen: list[_Match] = [] + last_end = 0 + for cm in raw: + if cm.start < last_end: + continue + chosen.append(cm) + last_end = cm.end + return chosen + + +def _resolve_surface( + replacements: tuple[ReplacementToken, ...], + position: int, + matched_text: str, +) -> str: + """Pick the surface for a replacement token. + + - explicit `surface` always wins; + - single-token replacement with `surface=None` โ†’ use full matched substring; + - multi-token replacement with `surface=None`, where the number of tokens + equals the matched span length, โ†’ inherit per-position from the matched + substring (this preserves half/full-width brackets, kanji vs arabic + digits, etc. that the regex character classes accepted); + - otherwise โ†’ warn and fall back to the furigana string. + """ + rt = replacements[position] + if rt.surface is not None: + return rt.surface + n = len(replacements) + if n == 1: + return matched_text + if n == len(matched_text): + return matched_text[position] + fallback = rt.furigana if rt.furigana is not None else matched_text + logger.warning( + "Override replacement at position %d missing explicit `surface` and " + "cannot inherit (n_replacements=%d, match_len=%d); falling back to %r", + position, + n, + len(matched_text), + fallback, + ) + return fallback + + +T = TypeVar("T") + + +def _apply( + words: list[T], + surface_of: Callable[[T], str], + build: Callable[[tuple[ReplacementToken, ...], int, str], T], +) -> list[T]: + if not words: + return list(words) + + surfaces = [surface_of(w) for w in words] + full_text = "".join(surfaces) + + # offsets[i] = char position of token i's start; offsets[-1] = total length + offsets: list[int] = [0] + for s in surfaces: + offsets.append(offsets[-1] + len(s)) + boundary_to_index = {off: i for i, off in enumerate(offsets)} + + matches = _collect_matches(full_text) + if not matches: + return list(words) + + out: list[T] = [] + cursor = 0 + for m in matches: + start_idx = boundary_to_index.get(m.start) + end_idx = boundary_to_index.get(m.end) + if start_idx is None or end_idx is None: + logger.warning( + "Override %r match at [%d, %d) does not align with Yahoo token " + "boundaries โ€” skipping", + m.override.description or m.override.pattern.pattern, + m.start, + m.end, + ) + continue + if start_idx < cursor: + # earlier non-overlapping match already consumed this region + continue + while cursor < start_idx: + out.append(words[cursor]) + cursor += 1 + matched_text = full_text[m.start : m.end] + replacements = m.override.replacements + for idx in range(len(replacements)): + out.append(build(replacements, idx, matched_text)) + cursor = end_idx + while cursor < len(words): + out.append(words[cursor]) + cursor += 1 + return out + + +def _resolve_furigana(rt: ReplacementToken, surface: str) -> str: + return rt.furigana if rt.furigana is not None else surface + + +def apply_furigana_overrides(words: list[WordResult]) -> list[WordResult]: + """Post-process Yahoo Furigana results, replacing matched spans.""" + + def build( + repls: tuple[ReplacementToken, ...], pos: int, matched: str + ) -> WordResult: + rt = repls[pos] + surface = _resolve_surface(repls, pos, matched) + return WordResult(surface=surface, furigana=_resolve_furigana(rt, surface)) + + return _apply(words, lambda w: w.surface, build) + + +def apply_accent_overrides( + words: list[WordAccentResult], +) -> list[WordAccentResult]: + """Post-process accent-aligned results, replacing both furigana and accent.""" + + def build( + repls: tuple[ReplacementToken, ...], pos: int, matched: str + ) -> WordAccentResult: + rt = repls[pos] + surface = _resolve_surface(repls, pos, matched) + furigana = _resolve_furigana(rt, surface) + if rt.accent: + accent = [ + AccentInfo(furigana=moji, accent_marking_type=t, length=len(moji)) + for moji, t in rt.accent + ] + else: + accent = [ + AccentInfo( + furigana=furigana, + accent_marking_type=_NONE, + length=len(furigana), + ) + ] + return WordAccentResult(surface=surface, furigana=furigana, accent=accent) + + return _apply(words, lambda w: w.surface, build) diff --git a/api/accent/routes.py b/api/accent/routes.py index 13b3e18..dbe81f3 100644 --- a/api/accent/routes.py +++ b/api/accent/routes.py @@ -1,8 +1,9 @@ """FastAPI routers for MarkAccent and MarkFurigana. Each endpoint is a thin wrapper around its data layer: - - /MarkFurigana/ โ†’ `furigana.fetch_furigana` - - /MarkAccent/ โ†’ `pipeline.process_accent_chunk` + - /MarkFurigana/ โ†’ `furigana.fetch_furigana` + - /MarkAccent/ โ†’ `pipeline.process_accent_chunk` + - /MarkAccent/stream/ โ†’ `pipeline.stream_accent_chunks` (NDJSON) Two separate routers (rather than one shared one) keep the OpenAPI tagging clean and let main.py register each with its own @@ -15,10 +16,11 @@ import httpx from fastapi import APIRouter, Depends +from fastapi.responses import StreamingResponse from api.accent.furigana import fetch_furigana from api.accent.models import AccentResponse, FuriganaResponse, Request -from api.accent.pipeline import process_accent_chunk +from api.accent.pipeline import process_accent_chunk, stream_accent_chunks from api.dependencies import get_http_client logger = logging.getLogger("api") @@ -57,3 +59,19 @@ async def mark_accent( """Receive POST request, return an AccentResponse object.""" logger.info(f"[API] Received Request Text: {request.text}") return await process_accent_chunk(request.text, client) + + +@accent_router.post("/MarkAccent/stream/", tags=["MarkAccent"]) +async def mark_accent_stream( + request: Request, + client: httpx.AsyncClient = Depends(get_http_client), +) -> StreamingResponse: + """Split the input on `\\n` (line) and then on full-width sentence + terminators (sub-chunk), process each piece in parallel under a bounded + semaphore, and stream one NDJSON line per piece in input order. + """ + logger.info(f"[API] Received streaming request: {request.text!r}") + return StreamingResponse( + stream_accent_chunks(request, client), + media_type="application/x-ndjson", + ) diff --git a/main.py b/main.py index e6a83b7..25cccc9 100644 --- a/main.py +++ b/main.py @@ -1,8 +1,8 @@ """ -An API interface that provide two functionalities -(1) Accent Marker (/api/MarkAccent/) +An API interface that provide the following functionalities +(1) Accent Marker (/api/MarkAccent/ + /api/MarkAccent/stream/) (2) Furigana Marker (/api/MarkFurigana/) -(3) Usage Query (/api/UsageQuery/) +(3) Usage Query (/api/UsageQuery/) (4) Dictionary Query (/api/DictQuery/) (5) Sentence Query (/api/SentenceQuery/) """ diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..1d211e7 --- /dev/null +++ b/test.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# Send a test text to the local API and print one +# (surface|furigana|accent_marking_type) line per moji. +# +# Usage: +# ./test.sh # default text on MarkAccent +# ./test.sh "ไธ‰ๆœˆไบ”ๆ—ฅ๏ผˆๅœŸ๏ผ‰" # custom text +# PORT=8000 ./test.sh # different port +# ENDPOINT=MarkFurigana ./test.sh # furigana endpoint (accent="-") +# STREAM=1 ./test.sh $'first\nsecond' # streaming endpoint, NDJSON + +set -euo pipefail + +TEXT="${1:-3ๆœˆ5ๆ—ฅ(ๅœŸ)}" +PORT="${PORT:-8000}" +ENDPOINT="${ENDPOINT:-MarkAccent}" +STREAM="${STREAM:-0}" + +if [[ "$STREAM" == "1" ]]; then + URL="http://127.0.0.1:${PORT}/api/${ENDPOINT}/stream/" +else + URL="http://127.0.0.1:${PORT}/api/${ENDPOINT}/" +fi + +PAYLOAD=$(uv run python -c \ + 'import json, sys; print(json.dumps({"text": sys.argv[1]}))' "$TEXT") + +if [[ "$STREAM" == "1" ]]; then + # Streaming mode: pipe NDJSON straight to a per-line viewer. + read -r -d '' STREAM_VIEWER <<'PY' || true +import sys, json + +seen = 0 +for raw in sys.stdin: + raw = raw.strip() + if not raw: + continue + seen += 1 + d = json.loads(raw) + chunk = d["chunk"] + sub = d.get("subchunk", 0) + status = d["status"] + err = d.get("error") + result = d.get("result") or [] + print(f"--- chunk {chunk}.{sub} status={status} words={len(result)} ---") + if err: + print(f" ERROR: {err}") + continue + for w in result: + surface = w["surface"] + accents = w.get("accent") or [] + if not accents: + print(f" ({surface}|{w['furigana']}|-)") + continue + for a in accents: + moji = a["furigana"] + t = a["accent_marking_type"] + print(f" ({surface}|{moji}|{t})") +if seen == 0: + print("(empty stream โ€” no non-blank input lines)") +PY + + # -N disables curl's output buffering so each NDJSON line lands in the + # viewer as soon as the server flushes it. + curl -sN -X POST "$URL" \ + -H 'Content-Type: application/json' \ + --data-raw "$PAYLOAD" \ + | uv run python -c "$STREAM_VIEWER" + exit 0 +fi + +# Non-streaming mode (original behaviour). +HTTP_STATUS=$(curl -s -o /tmp/test_sh_body.$$ -w '%{http_code}' \ + -X POST "$URL" -H 'Content-Type: application/json' --data-raw "$PAYLOAD" \ + || true) +if [[ "$HTTP_STATUS" != "200" || ! -s /tmp/test_sh_body.$$ ]]; then + echo "Request to $URL failed (HTTP ${HTTP_STATUS:-no-response})." >&2 + echo "Is the server running? Try: uv run uvicorn main:app --host 127.0.0.1 --port ${PORT}" >&2 + rm -f /tmp/test_sh_body.$$ + exit 1 +fi + +read -r -d '' FORMAT_SCRIPT <<'PY' || true +import json, sys + +data = json.load(sys.stdin) +if data.get("status") != 200 or not data.get("result"): + print("ERROR:", data.get("error") or data) + sys.exit(1) + +for w in data["result"]: + surface = w["surface"] + accents = w.get("accent") or [] + if not accents: + print(f"({surface}|{w['furigana']}|-)") + continue + for a in accents: + print(f"({surface}|{a['furigana']}|{a['accent_marking_type']})") +PY + +uv run python -c "$FORMAT_SCRIPT" < /tmp/test_sh_body.$$ +rm -f /tmp/test_sh_body.$$