feat: implement full-platform STT API (v2.3 consensus plan)
Batch+realtime transcription API: faster-whisper engine w/ EngineOwner (single GPU owner, OOM downgrade chain, persisted attempted_profiles), hardware-adaptive device manager (T0-T3 VRAM tiers), Redis/in-proc job queue w/ leases + crash recovery, postprocess (glossary/rules/LLM w/ egress guard), privacy-first result store (UUID keys, source deleted after transcribe), retention sweeper, API-key auth (HMAC digests, scopes, job ownership), WebSocket realtime lane (LocalAgreement), CLI (detect/transcribe/bench/serve/key), Docker, benchmark runner. 127 mock-based tests pass; ruff clean. Includes verification checklist and autoplan review notes.
This commit is contained in:
@@ -0,0 +1,84 @@
|
||||
"""벤치마크 지표 단위 테스트 — WER/K-CER/entity 보존."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from luke_scribe.benchmark.metrics import (
|
||||
character_kcer,
|
||||
clip_metrics,
|
||||
entity_retention,
|
||||
normalize_kcer,
|
||||
word_wer,
|
||||
)
|
||||
|
||||
|
||||
class TestKcer:
|
||||
def test_normalization_nfkc_punct(self):
|
||||
# NFKC: 유사 문자 통일, 문장부호/공백 제거
|
||||
assert (
|
||||
normalize_kcer("오늘 API 서버에서, vLLM을 사용해 보겠습니다.")
|
||||
== "오늘api서버에서vllm을사용해보겠습니다"
|
||||
)
|
||||
|
||||
def test_exact_match_zero(self):
|
||||
assert character_kcer("오늘 API 사용", "오늘 API 사용") == 0.0
|
||||
|
||||
def test_punct_diff_zero(self):
|
||||
# 문장부호 차이는 K-CER에서 0 (정규화됨)
|
||||
assert character_kcer("오늘 API 사용.", "오늘 API 사용") == 0.0
|
||||
|
||||
def test_one_char_diff(self):
|
||||
ref = "오늘 API 사용"
|
||||
hyp = "오늘 APT 사용"
|
||||
assert 0 < character_kcer(ref, hyp) < 0.3
|
||||
|
||||
|
||||
class TestWer:
|
||||
def test_exact(self):
|
||||
assert word_wer("a b c", "a b c") == 0.0
|
||||
|
||||
def test_one_sub(self):
|
||||
assert word_wer("a b c", "a x c") == 1 / 3
|
||||
|
||||
def test_empty_ref(self):
|
||||
assert word_wer("", "") == 0.0
|
||||
assert word_wer("", "a") == 1.0
|
||||
|
||||
|
||||
class TestEntityRetention:
|
||||
def test_preserved(self):
|
||||
ref = "오늘 API 서버에서 vLLM을 사용합니다"
|
||||
hyp = "오늘 API 서버에서 vLLM을 사용합니다"
|
||||
entities = [
|
||||
{"canonical": "API", "surface": "API", "start_char": 3, "end_char": 6},
|
||||
{"canonical": "vLLM", "surface": "vLLM", "start_char": 12, "end_char": 16},
|
||||
]
|
||||
p, t = entity_retention(ref, hyp, entities)
|
||||
assert p == 2 and t == 2
|
||||
|
||||
def test_lost(self):
|
||||
ref = "오늘 API 서버"
|
||||
hyp = "오늘 에이피아이 서버"
|
||||
entities = [{"canonical": "API", "surface": "API", "start_char": 3, "end_char": 6}]
|
||||
p, t = entity_retention(ref, hyp, entities)
|
||||
assert p == 0 and t == 1
|
||||
|
||||
def test_one_occurrence_one_match(self):
|
||||
# 같은 entity가 두 번 등장 → 2개 annotation, 둘 다 보존
|
||||
ref = "API 서버와 API 게이트웨이"
|
||||
hyp = "API 서버와 API 게이트웨이"
|
||||
entities = [
|
||||
{"canonical": "API", "surface": "API", "start_char": 0, "end_char": 3},
|
||||
{"canonical": "API", "surface": "API", "start_char": 8, "end_char": 11},
|
||||
]
|
||||
p, t = entity_retention(ref, hyp, entities)
|
||||
assert p == 2 and t == 2
|
||||
|
||||
|
||||
class TestClipMetrics:
|
||||
def test_perfect(self):
|
||||
ref = "오늘 API 서버에서 vLLM을 사용합니다"
|
||||
m = clip_metrics(
|
||||
ref, ref, [{"canonical": "API", "surface": "API", "start_char": 3, "end_char": 6}]
|
||||
)
|
||||
assert m.wer == 0.0 and m.k_cer == 0.0
|
||||
assert m.entities_preserved == 1 and m.entities_total == 1
|
||||
Reference in New Issue
Block a user