fix: solve vLLM->BLM misrecognition via glossary postprocessing
The bench and CLI kept emitting BLM for vLLM. Three layers:
1. rules.py: add default rule BLM -> vLLM (word-boundary, case-insensitive) so the default post_mode=rules path restores it everywhere.
2. benchmark/runner.py: the bench never ran postprocessing - it measured raw engine text, so entity retention (66.7%) never reflected rules/glossary. _transcribe_clip now builds Segments and runs run_postprocess(settings, glossary) before metrics; manifest top-level glossary {pattern: replacement} is supported and recorded in run_config (post_mode/glossary).
3. glossary plumbing: CLI transcribe gains --glossary KEY=VALUE (repeatable, parsed + validated); BatchPipeline.run accepts glossary= and passes it to run_postprocess; the in-proc worker forwards job.options.glossary or post_correction.
Notebook: transcribe cells pass --glossary BLM=vLLM, bench manifest carries glossary, postprocess section documents rules/glossary/hotword.
+ 9 unit tests (BLM rule, boundary/case behavior, bench postprocess + glossary entity retention). 147 tests pass, ruff clean.
This commit is contained in:
@@ -0,0 +1,143 @@
|
||||
"""벤치마크 단위 테스트 — 후처리 적용 + glossary 지원.
|
||||
|
||||
벤치 지표는 원시 전사가 아니라 후처리(rules/glossary)를 거친 결과를 측정해야
|
||||
한다 (vLLM→BLM 같은 오인식 복원 포함).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from luke_scribe.benchmark.runner import _run_model, _transcribe_clip
|
||||
from luke_scribe.config import Settings
|
||||
from luke_scribe.engine.base import TranscriptionOptions
|
||||
|
||||
REF_TEXT = "오늘은 vLLM 서버를 Kubernetes 클러스터에 배포합니다"
|
||||
ENTITIES = [
|
||||
{"canonical": "vLLM", "surface": "vLLM", "start_char": 4, "end_char": 8},
|
||||
{"canonical": "Kubernetes", "surface": "Kubernetes", "start_char": 13, "end_char": 23},
|
||||
]
|
||||
|
||||
|
||||
class _FakeOwner:
|
||||
"""세그먼트 dict를 반환하는 가짜 owner (dict 계약 사용)."""
|
||||
|
||||
def __init__(self, texts: list[str]) -> None:
|
||||
self._texts = texts
|
||||
|
||||
def transcribe(self, req): # noqa: ANN001
|
||||
segs = [
|
||||
{
|
||||
"index": i,
|
||||
"start": i * 2.0,
|
||||
"end": i * 2.0 + 2.0,
|
||||
"text": t,
|
||||
"avg_logprob": -0.2,
|
||||
"no_speech_prob": 0.01,
|
||||
}
|
||||
for i, t in enumerate(self._texts)
|
||||
]
|
||||
return {
|
||||
"segments": iter(segs),
|
||||
"device": "cpu",
|
||||
"compute_type": "int8",
|
||||
"attempted_profiles": [{}],
|
||||
"info": {"language": "ko"},
|
||||
}
|
||||
|
||||
|
||||
class TestTranscribeClipPostprocess:
|
||||
def test_rules_fix_blm_to_vllm(self):
|
||||
owner = _FakeOwner(["오늘은 BLM 서버를 Kubernetes 클러스터에 배포합니다"])
|
||||
settings = Settings(_env_file=None, post_mode="rules", post_enabled=True)
|
||||
clip = {"audio_path": "x.mp3", "duration_sec": 10.0}
|
||||
out = _transcribe_clip(owner, TranscriptionOptions(), clip, settings, None)
|
||||
assert "vLLM" in out["text"]
|
||||
assert "BLM" not in out["text"]
|
||||
|
||||
def test_glossary_applied(self):
|
||||
owner = _FakeOwner(["브이엘엘엠 서버"])
|
||||
settings = Settings(_env_file=None, post_mode="glossary", post_enabled=True)
|
||||
clip = {"audio_path": "x.mp3", "duration_sec": 10.0}
|
||||
out = _transcribe_clip(
|
||||
owner, TranscriptionOptions(), clip, settings, {"브이엘엘엠": "vLLM"}
|
||||
)
|
||||
assert "vLLM" in out["text"]
|
||||
|
||||
def test_postprocess_disabled_keeps_raw(self):
|
||||
owner = _FakeOwner(["오늘은 BLM 서버"])
|
||||
settings = Settings(_env_file=None, post_mode="rules", post_enabled=False)
|
||||
clip = {"audio_path": "x.mp3", "duration_sec": 10.0}
|
||||
out = _transcribe_clip(owner, TranscriptionOptions(), clip, settings, None)
|
||||
assert "BLM" in out["text"]
|
||||
|
||||
|
||||
class TestRunModelPostprocess:
|
||||
def _report(self) -> dict:
|
||||
return {"run_config": {"hotword_set": []}}
|
||||
|
||||
def _clip(self, ref_path: str) -> dict:
|
||||
return {
|
||||
"id": "c1",
|
||||
"audio_path": "x.mp3",
|
||||
"reference_path": ref_path,
|
||||
"duration_sec": 10.0,
|
||||
"entities": ENTITIES,
|
||||
}
|
||||
|
||||
def test_entity_retention_full_with_rules(self, tmp_path):
|
||||
ref = tmp_path / "ref.txt"
|
||||
ref.write_text(REF_TEXT, encoding="utf-8")
|
||||
owner = _FakeOwner(["오늘은 BLM 서버를 Kubernetes 클러스터에 배포합니다"])
|
||||
settings = Settings(_env_file=None, post_mode="rules", post_enabled=True)
|
||||
agg = _run_model(
|
||||
owner,
|
||||
[self._clip(str(ref))],
|
||||
"large-v3-turbo",
|
||||
"cpu",
|
||||
"int8",
|
||||
1,
|
||||
[],
|
||||
self._report(),
|
||||
settings,
|
||||
None,
|
||||
)
|
||||
assert agg["summary"]["entity_retention"] == 1.0
|
||||
assert agg["summary"]["failure_rate"] == 0.0
|
||||
|
||||
def test_glossary_raises_entity_retention(self, tmp_path):
|
||||
ref = tmp_path / "ref.txt"
|
||||
ref.write_text(REF_TEXT, encoding="utf-8")
|
||||
owner = _FakeOwner(["오늘은 BLM 서버를 Kubernetes 클러스터에 배포합니다"])
|
||||
settings = Settings(_env_file=None, post_mode="glossary", post_enabled=True)
|
||||
agg = _run_model(
|
||||
owner,
|
||||
[self._clip(str(ref))],
|
||||
"large-v3-turbo",
|
||||
"cpu",
|
||||
"int8",
|
||||
1,
|
||||
[],
|
||||
self._report(),
|
||||
settings,
|
||||
{"BLM": "vLLM"},
|
||||
)
|
||||
assert agg["summary"]["entity_retention"] == 1.0
|
||||
|
||||
def test_raw_text_fails_entity_retention_without_postprocess(self, tmp_path):
|
||||
# 후처리 없이 raw 전사("BLM")를 측정하면 vLLM 엔티티가 보존되지 않는다
|
||||
ref = tmp_path / "ref.txt"
|
||||
ref.write_text(REF_TEXT, encoding="utf-8")
|
||||
owner = _FakeOwner(["오늘은 BLM 서버를 Kubernetes 클러스터에 배포합니다"])
|
||||
settings = Settings(_env_file=None, post_mode="none", post_enabled=False)
|
||||
agg = _run_model(
|
||||
owner,
|
||||
[self._clip(str(ref))],
|
||||
"large-v3-turbo",
|
||||
"cpu",
|
||||
"int8",
|
||||
1,
|
||||
[],
|
||||
self._report(),
|
||||
settings,
|
||||
None,
|
||||
)
|
||||
assert agg["summary"]["entity_retention"] < 1.0
|
||||
@@ -43,6 +43,25 @@ class TestRules:
|
||||
out = apply_rules(segs)
|
||||
assert "vLLM" in out["segments"][0].text
|
||||
|
||||
def test_vllm_restored_from_blm(self):
|
||||
# GPU 실전에서 재현된 오인식: vLLM → BLM
|
||||
segs = _segments(["오늘은 BLM 서버를 배포합니다"])
|
||||
out = apply_rules(segs)
|
||||
assert "vLLM" in out["segments"][0].text
|
||||
assert "BLM" not in out["segments"][0].text
|
||||
|
||||
def test_blm_boundary_required(self):
|
||||
# 단어 경계가 없으면 교정하지 않는다 (부분 문자열 보호)
|
||||
segs = _segments(["sublm 단어"])
|
||||
out = apply_rules(segs)
|
||||
assert "sublm" in out["segments"][0].text
|
||||
assert "vLLM" not in out["segments"][0].text
|
||||
|
||||
def test_blm_case_insensitive(self):
|
||||
segs = _segments(["blm 서버"])
|
||||
out = apply_rules(segs)
|
||||
assert "vLLM" in out["segments"][0].text
|
||||
|
||||
def test_whitespace_collapse(self):
|
||||
segs = _segments(["오늘 API 서버"])
|
||||
out = apply_rules(segs)
|
||||
|
||||
Reference in New Issue
Block a user