fix: solve vLLM->BLM misrecognition via glossary postprocessing

The bench and CLI kept emitting BLM for vLLM. Three layers:

1. rules.py: add default rule BLM -> vLLM (word-boundary, case-insensitive) so the default post_mode=rules path restores it everywhere.

2. benchmark/runner.py: the bench never ran postprocessing - it measured raw engine text, so entity retention (66.7%) never reflected rules/glossary. _transcribe_clip now builds Segments and runs run_postprocess(settings, glossary) before metrics; manifest top-level glossary {pattern: replacement} is supported and recorded in run_config (post_mode/glossary).

3. glossary plumbing: CLI transcribe gains --glossary KEY=VALUE (repeatable, parsed + validated); BatchPipeline.run accepts glossary= and passes it to run_postprocess; the in-proc worker forwards job.options.glossary or post_correction.

Notebook: transcribe cells pass --glossary BLM=vLLM, bench manifest carries glossary, postprocess section documents rules/glossary/hotword.

+ 9 unit tests (BLM rule, boundary/case behavior, bench postprocess + glossary entity retention). 147 tests pass, ruff clean.
This commit is contained in:
2026-08-12 21:06:42 +09:00
parent f171ce4992
commit 6bb89eb51f
9 changed files with 237 additions and 22 deletions
+143
View File
@@ -0,0 +1,143 @@
"""벤치마크 단위 테스트 — 후처리 적용 + glossary 지원.
벤치 지표는 원시 전사가 아니라 후처리(rules/glossary)를 거친 결과를 측정해야
한다 (vLLM→BLM 같은 오인식 복원 포함).
"""
from __future__ import annotations
from luke_scribe.benchmark.runner import _run_model, _transcribe_clip
from luke_scribe.config import Settings
from luke_scribe.engine.base import TranscriptionOptions
REF_TEXT = "오늘은 vLLM 서버를 Kubernetes 클러스터에 배포합니다"
ENTITIES = [
{"canonical": "vLLM", "surface": "vLLM", "start_char": 4, "end_char": 8},
{"canonical": "Kubernetes", "surface": "Kubernetes", "start_char": 13, "end_char": 23},
]
class _FakeOwner:
"""세그먼트 dict를 반환하는 가짜 owner (dict 계약 사용)."""
def __init__(self, texts: list[str]) -> None:
self._texts = texts
def transcribe(self, req): # noqa: ANN001
segs = [
{
"index": i,
"start": i * 2.0,
"end": i * 2.0 + 2.0,
"text": t,
"avg_logprob": -0.2,
"no_speech_prob": 0.01,
}
for i, t in enumerate(self._texts)
]
return {
"segments": iter(segs),
"device": "cpu",
"compute_type": "int8",
"attempted_profiles": [{}],
"info": {"language": "ko"},
}
class TestTranscribeClipPostprocess:
def test_rules_fix_blm_to_vllm(self):
owner = _FakeOwner(["오늘은 BLM 서버를 Kubernetes 클러스터에 배포합니다"])
settings = Settings(_env_file=None, post_mode="rules", post_enabled=True)
clip = {"audio_path": "x.mp3", "duration_sec": 10.0}
out = _transcribe_clip(owner, TranscriptionOptions(), clip, settings, None)
assert "vLLM" in out["text"]
assert "BLM" not in out["text"]
def test_glossary_applied(self):
owner = _FakeOwner(["브이엘엘엠 서버"])
settings = Settings(_env_file=None, post_mode="glossary", post_enabled=True)
clip = {"audio_path": "x.mp3", "duration_sec": 10.0}
out = _transcribe_clip(
owner, TranscriptionOptions(), clip, settings, {"브이엘엘엠": "vLLM"}
)
assert "vLLM" in out["text"]
def test_postprocess_disabled_keeps_raw(self):
owner = _FakeOwner(["오늘은 BLM 서버"])
settings = Settings(_env_file=None, post_mode="rules", post_enabled=False)
clip = {"audio_path": "x.mp3", "duration_sec": 10.0}
out = _transcribe_clip(owner, TranscriptionOptions(), clip, settings, None)
assert "BLM" in out["text"]
class TestRunModelPostprocess:
def _report(self) -> dict:
return {"run_config": {"hotword_set": []}}
def _clip(self, ref_path: str) -> dict:
return {
"id": "c1",
"audio_path": "x.mp3",
"reference_path": ref_path,
"duration_sec": 10.0,
"entities": ENTITIES,
}
def test_entity_retention_full_with_rules(self, tmp_path):
ref = tmp_path / "ref.txt"
ref.write_text(REF_TEXT, encoding="utf-8")
owner = _FakeOwner(["오늘은 BLM 서버를 Kubernetes 클러스터에 배포합니다"])
settings = Settings(_env_file=None, post_mode="rules", post_enabled=True)
agg = _run_model(
owner,
[self._clip(str(ref))],
"large-v3-turbo",
"cpu",
"int8",
1,
[],
self._report(),
settings,
None,
)
assert agg["summary"]["entity_retention"] == 1.0
assert agg["summary"]["failure_rate"] == 0.0
def test_glossary_raises_entity_retention(self, tmp_path):
ref = tmp_path / "ref.txt"
ref.write_text(REF_TEXT, encoding="utf-8")
owner = _FakeOwner(["오늘은 BLM 서버를 Kubernetes 클러스터에 배포합니다"])
settings = Settings(_env_file=None, post_mode="glossary", post_enabled=True)
agg = _run_model(
owner,
[self._clip(str(ref))],
"large-v3-turbo",
"cpu",
"int8",
1,
[],
self._report(),
settings,
{"BLM": "vLLM"},
)
assert agg["summary"]["entity_retention"] == 1.0
def test_raw_text_fails_entity_retention_without_postprocess(self, tmp_path):
# 후처리 없이 raw 전사("BLM")를 측정하면 vLLM 엔티티가 보존되지 않는다
ref = tmp_path / "ref.txt"
ref.write_text(REF_TEXT, encoding="utf-8")
owner = _FakeOwner(["오늘은 BLM 서버를 Kubernetes 클러스터에 배포합니다"])
settings = Settings(_env_file=None, post_mode="none", post_enabled=False)
agg = _run_model(
owner,
[self._clip(str(ref))],
"large-v3-turbo",
"cpu",
"int8",
1,
[],
self._report(),
settings,
None,
)
assert agg["summary"]["entity_retention"] < 1.0