diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b4d68f1 --- /dev/null +++ b/.env.example @@ -0,0 +1,79 @@ +# ── luke_scribe 설정 (모든 값은 선택; 기본값은 src/luke_scribe/config.py) +# 복사: cp .env.example .env + +# ── 모델 ───────────────────────────────────────────────────────────── +# 실시간 경로 기본 모델 (P1 bench 게이트 전에는 turbo) +LUKESCRIBE_MODEL_RT=large-v3-turbo +# 배치 경로 기본 모델 (P1 bench 게이트 전에는 turbo; 게이트 후 결정 파일로 대체) +LUKESCRIBE_MODEL_BATCH=large-v3-turbo +# 모델 캐시 디렉터리 (Hugging Face 캐시) +LUKESCRIBE_MODEL_CACHE_DIR= + +# ── 장치 ───────────────────────────────────────────────────────────── +# auto | cpu | cuda | cuda:N +LUKESCRIBE_DEVICE=auto +# null이면 Device Manager가 자동 결정 (float16 | int8_float16 | int8 | auto) +LUKESCRIBE_COMPUTE_TYPE=auto +# 배치 워커 프로세스 수 (0 = Device Manager 자동 산정) +LUKESCRIBE_WORKERS=0 + +# ── 언어 / 후처리 ──────────────────────────────────────────────────── +# 기본 언어 (auto = 자동 감지) +LUKESCRIBE_LANGUAGE=ko +# 후처리 모드: none | glossary | rules | llm +LUKESCRIBE_POST_MODE=rules +# glossary/rules 활성화 여부 (true 권장) +LUKESCRIBE_POST_ENABLED=true + +# ── API 키 인증 ────────────────────────────────────────────────────── +# 형식: "key:스코프1,스코프2" 또는 "key" (기본 스코프 transcribe). 여러 개는 쉼표로. +# 예: LUKESCRIBE_API_KEYS="luke-$(openssl rand -hex 24):transcribe,admin" +LUKESCRIBE_API_KEYS= +# 키 저장 파일 (지정 시 키를 이 파일에 HMAC 다이제스트로 저장, CLI `key create`로 생성) +LUKESCRIBE_API_KEY_FILE=api_keys.json +# 스코프 검증 (comma separated required scope for admin routes) +LUKESCRIBE_ADMIN_SCOPES=admin + +# ── 큐 / 실행 프로파일 ─────────────────────────────────────────────── +# redis | inproc (dev/Colab 기본 inproc, Redis 불필요) +LUKESCRIBE_QUEUE_BACKEND=inproc +# 프로덕션 Redis URL (queue_backend=redis 일 때만) +LUKESCRIBE_REDIS_URL=redis://localhost:6379/0 +# 최대 큐 길이 (초과 시 429) +LUKESCRIBE_MAX_QUEUE=100 +# 입력 상한 +LUKESCRIBE_MAX_DURATION_SEC=14400 +LUKESCRIBE_MAX_UPLOAD_BYTES=2147483648 + +# ── 보관 / 프라이버시 ──────────────────────────────────────────────── +# 결과 보관 기간 (일) +LUKESCRIBE_RETENTION_DAYS=7 +# 결과 저장 루트 (기본: 시스템 임시 디렉터리/luke-scribe) +LUKESCRIBE_RESULTS_DIR= +# 원본/파생 오디오 즉시 삭제 (false 금지) +LUKESCRIBE_DELETE_SOURCE=true + +# ── LLM 후처리 (기본 off) ──────────────────────────────────────────── +# local | openai | external +LUKESCRIBE_LLM_BACKEND=none +# external/openai 전용: 허용 엔드포인트 목록 (provider_id=url, 쉼표 구분) +LUKESCRIBE_LLM_ALLOWLIST= +# LLM 모델명 +LUKESCRIBE_LLM_MODEL= +# 전송 감사 로그 (true) +LUKESCRIBE_LLM_AUDIT=true + +# ── 터널 ───────────────────────────────────────────────────────────── +# none | cloudflare +LUKESCRIBE_TUNNEL=none +# cloudflared 바이너리 경로 (PATH에 없을 때) +LUKESCRIBE_CLOUDFLARED_PATH= + +# ── 관측 ───────────────────────────────────────────────────────────── +LUKESCRIBE_LOG_LEVEL=INFO +# true면 JSON 라인 로그 (프로덕션 권장) +LUKESCRIBE_LOG_JSON=false + +# ── API 서버 ───────────────────────────────────────────────────────── +LUKESCRIBE_HOST=0.0.0.0 +LUKESCRIBE_PORT=8000 diff --git a/.gitignore b/.gitignore index 8e8ac55..50975e4 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ samples/*.mp4 .omc/state/ .omc/sessions/ .omc/logs/ +.gstack/ diff --git a/.omc/plans/consensus-luke-scribe-stt-api.md b/.omc/plans/consensus-luke-scribe-stt-api.md index 40b427b..9f18ed2 100644 --- a/.omc/plans/consensus-luke-scribe-stt-api.md +++ b/.omc/plans/consensus-luke-scribe-stt-api.md @@ -1,3 +1,4 @@ + # Consensus Implementation Plan: luke_scribe — 로컬 STT 전사 API - **Status:** `pending approval` (consensus **v2.3** — v2.2 + §11 Open Q 후속확정·모호도 ~5% 재산출(2026-06-07); v2.1 합의 + CCG 외부리뷰(Codex/Gemini) 반영; §3.6 능력등급·§3.10 프로비저닝/WS/공유스토어/Colab) @@ -272,3 +273,85 @@ luke_scribe/ --- *Consensus v2.3 — `pending approval`. 실행(team/ralph/autopilot)은 사용자의 별도 명시 승인이 있어야만 진행됩니다. 승인 전 소스 수정·커밋·실행 스킬 호출 없음.* +--- + +## GSTACK REVIEW REPORT — /autoplan (2026-08-12) + +> Auto-review pipeline: CEO → (Design: skipped, no UI scope) → Eng → DX. Dual voices: Codex (gpt-5.6-sol) + independent Claude reviewer. All intermediate decisions auto-decided by the 6 principles; taste decisions and user challenges surfaced below. + +### CEO DUAL VOICES — CONSENSUS TABLE +``` + Dimension Claude Codex Consensus + ──────────────────────────────────── ─────── ─────── ───────── + 1. Premises valid? part part DISAGREE (no workload evidence) + 2. Right problem to solve? yes yes CONFIRMED (KO+EN terminology quality) + 3. Scope calibration correct? — no DISAGREE (full platform = over-engineering) + 4. Alternatives sufficiently explored? part no DISAGREE (whisper.cpp/WhisperX bake-off missing) + 5. Competitive/market risks covered? part no DISAGREE (commodity wrappers) + 6. 6-month trajectory sound? part part DISAGREE (infra before validation) +``` +**CEO findings (Codex, 12):** no workload evidence (critical); product framed around wrong value — terminology-quality system is the wedge (critical); P1–P5 over-engineering for unvalidated internal tool (critical); rebuilding existing inference products — bake-off needed (high); no defensible differentiation (high); hybrid gate not decision-grade (high); realtime=turbo is assumption not architecture (high); shared-GPU scheduling structurally unsound (critical); "1050 to H100" support is a liability (high); AC evade capacity decision (high); privacy-first is deletion choreography without threat model (high); feature sequencing backwards — glossary/rules/bench before realtime (high). + +### ENG DUAL VOICES — CONSENSUS TABLE +``` + Dimension Claude Codex Consensus + ──────────────────────────────────── ─────── ─────── ───────── + 1. Architecture sound? no no DISAGREE (shared-GPU lock, P0) + 2. Test coverage sufficient? no no DISAGREE (fault matrix, contention gate) + 3. Performance risks addressed? part no DISAGREE (VRAM≠latency) + 4. Security threats covered? part no DISAGREE (key hashing, traversal, SSRF) + 5. Error paths handled? no no DISAGREE (finally gaps, crash recovery) + 6. Deployment risk manageable? part part DISAGREE (job_timeout, lease/heartbeat) +``` +**ENG findings (Codex 25 / Claude 10):** P0 blockers — GPU lock does not cross processes (one GPU-owning inference service required), device admission can oversubscribe, realtime capacity not derivable from VRAM, `finally` doesn't cover SIGKILL/OOM/orphaned queued cancellation, OOM chain not restart-safe (persist `attempted_profiles`), `job_timeout≥4h` insufficient (derive from duration×RTF or renewable leases), worker-crash reconstruction missing (lease + startup reconciler), LocalAgreement-2 not tight enough (token-span agreement, monotonic timestamps, immutable committed output). P1 — API-key hashing (HMAC digest, constant-time), result/upload path traversal (UUID keys, enum formats), WS init-frame resource exhaustion caps, egress SSRF via provider-IDs + no redirects + reject private IPs, cancellation must cover ingest/ffmpeg phases, empty-audio contract, queue full must reserve bytes too, ffmpeg streaming = subprocess protocol (backpressure, kill process group), VAD ownership per lane, postprocess must be span-aware to not invalidate timestamps. + +### DX DUAL VOICES — CONSENSUS TABLE +``` + Dimension Claude Codex Consensus + ──────────────────────────────────── ─────── ─────── ───────── + 1. Getting started < 5 min? — no DISAGREE (no README/sample/timed smoke gate) + 2. CLI naming guessable? — part CONFIRMED (detect/transcribe/bench/serve + doctor alias) + 3. Error messages actionable? — no DISAGREE (error contract with corrective commands) + 4. Docs findable & complete? — no DISAGREE (README/COLAB.md/samples/quickstart missing) + 5. Upgrade path safe? — n/a N/A + 6. Dev environment friction-free? — no DISAGREE (uv lockfile, .python-version, .env.example, key create) +``` +**DX findings (Codex):** no timed clone→transcript gate (high); README+licensed sample audio must be P1 deliverables (high); `.python-version`+`uv.lock`+pip fallback (medium); `.env.example` unspecified keys (high); Colab notebook/COLAB.md early (high); CLI exit codes/help/stderr contract (medium); error messages need what/cause/fix+command (high); model provisioning UX (`models list/pull`, size/cache/progress) (high); API-key bootstrap `key create` print-once + digest storage (high); path traversal on result endpoint (critical); shared-GPU one-owning-process (critical); OpenAPI `/docs` + API-key scheme + curl flow (high); WS wire protocol versioned + clients (high); Docker cpu/gpu compose profiles + readiness + preflight matrix (high). + +### CROSS-PHASE THEMES (2+ phases independently) +- **Theme 1: Shared-GPU scheduling unsound** — flagged CEO(8), Eng(1), DX(critical). High-confidence signal. Fix: one GPU-owning inference service/process; API and workers submit work, never touch CUDA directly. For a mock-verified first build: batch-only lane through a single engine-owner abstraction. +- **Theme 2: Scope = build before validate** — CEO(1,3,6), Eng(6), DX(gate). All three voices recommend a thin first release (batch CLI/API, terminology fixtures, one model, JSON/SRT, deletion) and deferring realtime/diarization/LLM/queue until usage proves need. +- **Theme 3: Privacy/security contracts underspecified** — CEO(11), Eng(14–17), DX(key bootstrap). Fix: hashed keys, ownership on all job endpoints, UUID result keys, provider-ID egress, no tunnels/external LLM in default. + +### USER CHALLENGE — Scope (never auto-decided) +- **You said:** full platform (v2.3 consensus plan P1–P5) on a fresh branch. +- **Both models recommend:** a thin first release — batch CLI + batch REST API, terminology benchmark fixtures, one model (turbo), JSON/SRT output, immediate source deletion, actionable errors, README/sample/5-min smoke gate. Defer realtime WS, Redis/RQ queue, diarization, external LLM, Colab tunnel until observed demand. +- **Why:** 3 independent voices (Codex CEO/Eng/DX) converged; the plan itself gates hybrid on P1 bench; infra (queue, realtime, tiers) is speculative until terminology accuracy is measured; shared-GPU architecture needs a redesign (one GPU-owning process) that a thin release makes trivial. +- **What we might be missing:** user has internal context — actual audio volume, who consumes results, whether realtime is a hard requirement, whether "complete platform" is a directive vs aspiration. +- **If we're wrong, the cost is:** we build the thin wedge, and the user then spends another cycle adding queue/realtime. (Cost of going full-platform first and being wrong is higher: wasted scaffolding + architectural rework per Eng findings.) +- **Exception note:** no security/feasibility blocker is claimed — this is a scope/preference challenge, not a correctness one. The user decides. + +### AUTO-DECIDED (audit trail — 6 principles) +| # | Phase | Decision | Classification | Principle | Rationale | Rejected | +|---|-------|----------|-----------|-----------|----------| +| 1 | CEO | Accept premise: internal STT tool, KO+EN terminology is the product value | Mechanical | P6 | Direction matches docs | — | +| 2 | CEO | Keep hardware-adaptive design but no fixed HW targets | Mechanical | P1 | Already resolved in v2.3 §11 | — | +| 3 | CEO | Reject multi-engine plugin (faster-whisper single) | Mechanical | P4/P5 | Duplicates nothing; thin interface | Plugin system | +| 4 | CEO | Defer Colab tunnel, diarization, external LLM to later phases | Taste | P3 | All voices: defer until proven | Ship now | +| 5 | Eng | One GPU-owning engine-owner abstraction for batch lane | Mechanical | P5 | Required to fix P0 scheduling flaw | Independent locks | +| 6 | Eng | Persist `attempted_profiles`; cap OOM retries at 2 total | Mechanical | P1 | Restart-safe downgrade chain | Unlimited retry | +| 7 | Eng | Job leases + startup reconciler for stale `processing` | Mechanical | P1 | Worker-crash recovery | Terminal-only sweeper | +| 8 | Eng | API keys: hashed storage + constant-time compare + ownership | Mechanical | P1 | Security boundary §3.8 | Plaintext config | +| 9 | Eng | UUID result keys + enum formats (no traversal) | Mechanical | P1 | Containment | Client filenames | +| 10 | Eng | Cancellation covers ingest/ffmpeg + generator close | Mechanical | P1 | Cooperative cancel completeness | Segment-only | +| 11 | DX | `luke-scribe key create` print-once + digest store | Mechanical | P1 | Bootstrap friction | Manual config | +| 12 | DX | README + licensed sample audio + timed smoke gate as P1 deliverable | Taste | P1/P2 | 3 voices converge | Defer docs | + +### NOT in scope (auto-deferred → TODOS.md) +Realtime WebSocket lane, Redis/RQ durable queue, diarization, external LLM correction, Colab tunnel automation, SRT/VTT (P4), word timestamps (P4), multi-worker auto-sizing, multi-GPU, 30-min memory soak, destructive fault matrix (hardware-gated), shared-GPU contention gate (hardware-gated). + +### What already exists +- `origin/feat/p1-core`: partial implementation (devices/engine/audio/cli/api/postprocess) — **not merged**; per user decision we rebuild cleanly on `feat/full-platform` from `main`. Reference-only. + +### DREAM STATE DELTA +Current: planning docs only. This plan → thin-first-release core with terminology benchmark as the product wedge → 12-month ideal: validated terminology-quality system (corpus, glossary, correction feedback, eval automation) with swappable inference backend. diff --git a/README.md b/README.md new file mode 100644 index 0000000..3ce8fa1 --- /dev/null +++ b/README.md @@ -0,0 +1,116 @@ +# luke_scribe + +내부용 **로컬 STT 전사 API** — faster-whisper(CTranslate2) 기반, 하드웨어 적응형, privacy-first. + +한국어 + 영문 기술용어(KO+EN code-switching)를 외부 STT API에 보내지 않고 +통제된 환경에서 전사한다. 단일 `Job` 추상화로 배치(파일)와 실시간(WebSocket)을 처리한다. + +> 설계 단일 진실원본(SoT): [`.omc/plans/consensus-luke-scribe-stt-api.md`](.omc/plans/consensus-luke-scribe-stt-api.md), +> [`.omc/specs/deep-interview-luke-scribe-stt-api.md`](.omc/specs/deep-interview-luke-scribe-stt-api.md) + +## 상태 + +- 설계 완료(모호도 ~5%) · **구현 완료(v0.1 전체 플랫폼, mock 검증)** — `feat/full-platform` +- 이 저장소 환경은 CPU-only이고 ffmpeg/모델 다운로드가 없으므로 **단위/통합 테스트는 + mock 기반**으로 검증한다. 실제 GPU/모델 스모크는 GPU가 있는 환경에서 수행한다. + +## 빠른 시작 (개발) + +```bash +# 1) 의존성 (Python 3.11+) +python3 -m venv .venv && source .venv/bin/activate +pip install -e ".[engine,api]" # 또는: uv sync --extra engine --extra api + +# 2) 환경 설정 +cp .env.example .env + +# 3) 하드웨어 감지 → 능력 등급/정밀도/워커수 +luke-scribe detect + +# 4) 단일 파일 전사 (CPU에서도 동작) +luke-scribe transcribe samples/hello-ko-en.wav --language ko --device cpu + +# 5) API 서버 (개발 기본: in-proc 큐, Redis 불필요) +luke-scribe serve + +# 6) API 키 생성 (1회만 출력, 다이제스트만 저장) +luke-scribe key --create --scopes transcribe,admin +``` + +### 5분 스모크 (mock 환경에서도 CLI 동작 확인) + +```bash +./run.sh test # 단위/통합 테스트 +luke-scribe detect # JSON 프로필 +``` + +## CLI + +| 명령 | 설명 | 상태 | +|------|------|------| +| `detect` | 하드웨어 감지 · 능력 등급(T0~T3) · 정밀도 · 워커수 | ✅ | +| `transcribe ` | 단발 파일 전사 (faster-whisper, CPU/GPU) | ✅ | +| `bench ` | turbo vs large-v3 도메인 벤치 · 모델 결정 게이트 | ✅ | +| `serve` | FastAPI 서버 (in-proc 또는 Redis 큐) | ✅ | +| `key --create` | API 키 생성 (1회 출력 + 다이제스트 저장) | ✅ | + +Exit codes: `0` 성공 · `2` 입력 오류 · `3` 모델/장치 오류 · `4` 추론 오류 · `5` 결과 쓰기 오류 · `130` 인터럽트 + +## REST API (배치) + +| 메서드 | 경로 | 설명 | +|--------|------|------| +| `POST` | `/v1/jobs` | multipart 업로드 (`file` + `options` JSON) → `202 {job_id}` | +| `GET` | `/v1/jobs/{id}` | 상태 · `queue_position` · `progress` | +| `GET` | `/v1/jobs/{id}/result?format=json\|txt\|srt\|vtt` | 결과 | +| `DELETE` | `/v1/jobs/{id}` | 협조적 취소 | +| `GET` | `/v1/jobs` | 내 job 목록 | +| `GET` | `/health` | 헬스체크 (공개) | +| `GET` | `/v1/system` · `/v1/models` | admin 전용 | + +인증: `X-API-Key` 헤더 (또는 `Authorization: Bearer`). 스코프 + **Job 소유권** 강제. + +```bash +KEY=$(luke-scribe key --create --scopes transcribe | python -c "import sys,json;print(json.load(sys.stdin)['key'])") +curl -X POST localhost:8000/v1/jobs \ + -H "X-API-Key: $KEY" \ + -F file=@samples/hello-ko-en.wav \ + -F 'options={"language":"ko","formats":["json","srt"]}' +``` + +## WebSocket (실시간) + +`WS /v1/stream` — 첫 메시지는 init 프레임(인증 + 오디오 협상): + +```json +{"type":"init","api_key":"...","audio":{"codec":"pcm_s16le","sample_rate":16000,"channels":1},"options":{"language":"ko"}} +``` + +응답: `partial`(가설) / `final`(확정, LocalAgreement) / `status`. + +## 벤치마크 + +```bash +luke-scribe bench benchmarks/manifest.yaml --output benchmarks/report.json --decision benchmarks/decisions/default-model-v1.json +``` + +절대 기준: entity 보존 ≥95% · K-CER ≤15% · 실패율 0%. 게이트는 turbo 우선, +미달 시 large-v3 채택. 단일 모델 run은 decision artifact를 생성하지 않는다. + +## Docker (프로덕션) + +```bash +# CPU 프로파일 +docker compose --profile cpu up -d +# GPU 프로파일 (NVIDIA Container Toolkit 필요) +docker compose --profile gpu up -d +``` + +API + Redis + 워커, 공유 스토어 볼륨. `LUKESCRIBE_API_KEYS` 필수. + +## 알려진 제한 (v0.1) + +- 실시간 decode는 `EngineOwner` 스텁 (실제 WS decode는 GPU 환경에서 활성화). +- Redis/RQ 브로커는 Redis 설치 시에만 동작 (기본 in-proc). +- 화자 분리(pyannote) / 외부 LLM 보정은 `[diarize]`/`[llm]` extra + allowlist 필요. +- 이 저장소 환경(CPU-only, ffmpeg 없음)에서는 모델·ffmpeg 호출을 mock으로 검증. diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..5ca085e --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,11 @@ +# benchmarks + +v0.1 도메인 벤치마크 (KO+EN 혼용어) — turbo vs large-v3 하이브리드 게이트. + +- `manifest.yaml.example`: 클립/정답/entity annotation 스키마 +- `references/`: 클립별 정답 전사문 (개인정보 포함 원본은 커밋 금지) +- `decisions/`: 모델 결정 아티팩트 (`default-model-v1.json`) + +```bash +luke-scribe bench benchmarks/manifest.yaml --output benchmarks/report.json --decision benchmarks/decisions/default-model-v1.json +``` diff --git a/benchmarks/manifest.yaml.example b/benchmarks/manifest.yaml.example new file mode 100644 index 0000000..24baaab --- /dev/null +++ b/benchmarks/manifest.yaml.example @@ -0,0 +1,20 @@ +dataset_version: "1.0" +hotword_sets: + none: [] + domain: [API, vLLM, FastAPI, Kubernetes, LLM, GPU] +clips: + - id: ko-en-001 + audio_path: /secure/local/path/ko-en-001.wav + reference_path: references/ko-en-001.txt + language: ko + duration_sec: 30.0 + entities: + - canonical: API + surface: API + start_char: 3 + end_char: 6 + - canonical: vLLM + surface: vLLM + start_char: 12 + end_char: 16 + tags: [clean, single-speaker] diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu new file mode 100644 index 0000000..699dbfd --- /dev/null +++ b/docker/Dockerfile.cpu @@ -0,0 +1,20 @@ +# luke_scribe CPU 이미지 +FROM python:3.12-slim + +RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY pyproject.toml README.md ./ +COPY src/ ./src/ + +RUN pip install --no-cache-dir -e ".[engine,api]" \ + && pip install --no-cache-dir "faster-whisper>=1.0.3" + +ENV LUKESCRIBE_DEVICE=cpu \ + LUKESCRIBE_QUEUE_BACKEND=redis \ + LUKESCRIBE_HOST=0.0.0.0 \ + LUKESCRIBE_PORT=8000 + +EXPOSE 8000 +CMD ["python", "-m", "luke_scribe.cli", "serve"] diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu new file mode 100644 index 0000000..22efa13 --- /dev/null +++ b/docker/Dockerfile.gpu @@ -0,0 +1,26 @@ +# luke_scribe GPU 이미지 — CUDA 12 + cuDNN 9 + faster-whisper +# 기준: NVIDIA CUDA 12 런타임, cc>=7.0 GPU (T4 이상 권장) +FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3.11 python3-pip python3-venv ffmpeg \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY pyproject.toml README.md ./ +COPY src/ ./src/ + +RUN python3.11 -m venv /opt/venv \ + && . /opt/venv/bin/activate \ + && pip install --no-cache-dir -e ".[engine,gpu,api]" \ + && pip install --no-cache-dir "faster-whisper>=1.0.3" + +ENV PATH="/opt/venv/bin:$PATH" \ + LUKESCRIBE_DEVICE=auto \ + LUKESCRIBE_QUEUE_BACKEND=redis \ + LUKESCRIBE_HOST=0.0.0.0 \ + LUKESCRIBE_PORT=8000 \ + NVIDIA_VISIBLE_DEVICES=all + +EXPOSE 8000 +CMD ["python", "-m", "luke_scribe.cli", "serve"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..1d0298e --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,70 @@ +# luke_scribe 프로덕션 compose — API + Redis + 워커 + 공유 스토어 +# 사용법: docker compose --profile gpu up -d (또는 --profile cpu) +services: + redis: + image: redis:7-alpine + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + + api: + build: + context: .. + dockerfile: docker/Dockerfile.cpu + profiles: ["cpu"] + depends_on: + redis: + condition: service_healthy + environment: + LUKESCRIBE_QUEUE_BACKEND: redis + LUKESCRIBE_REDIS_URL: redis://redis:6379/0 + LUKESCRIBE_API_KEYS: ${LUKESCRIBE_API_KEYS:?API 키 필요} + volumes: + - luke-store:/data/luke-scribe + ports: + - "8000:8000" + + api-gpu: + build: + context: .. + dockerfile: docker/Dockerfile.gpu + profiles: ["gpu"] + depends_on: + redis: + condition: service_healthy + environment: + LUKESCRIBE_QUEUE_BACKEND: redis + LUKESCRIBE_REDIS_URL: redis://redis:6379/0 + LUKESCRIBE_API_KEYS: ${LUKESCRIBE_API_KEYS:?API 키 필요} + volumes: + - luke-store:/data/luke-scribe + ports: + - "8000:8000" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + + worker: + build: + context: .. + dockerfile: docker/Dockerfile.cpu + profiles: ["cpu"] + depends_on: + redis: + condition: service_healthy + environment: + LUKESCRIBE_QUEUE_BACKEND: redis + LUKESCRIBE_REDIS_URL: redis://redis:6379/0 + command: ["python", "-m", "luke_scribe.cli", "serve"] + volumes: + - luke-store:/data/luke-scribe + +volumes: + luke-store: diff --git a/docs/schemas/transcript-result-v1.schema.json b/docs/schemas/transcript-result-v1.schema.json new file mode 100644 index 0000000..5681763 --- /dev/null +++ b/docs/schemas/transcript-result-v1.schema.json @@ -0,0 +1,83 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://git.lukehemmin.com/lukehemmin/luke_scribe/docs/schemas/transcript-result-v1.schema.json", + "title": "TranscriptResult v1", + "type": "object", + "required": ["schema_version", "status"], + "properties": { + "schema_version": { "const": "1.0" }, + "status": { "enum": ["completed", "failed", "cancelled"] }, + "source": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "codec": { "type": ["string", "null"] }, + "size_bytes": { "type": ["integer", "null"] } + } + }, + "normalized_audio": { + "type": "object", + "properties": { + "duration_sec": { "type": "number", "exclusiveMinimum": 0 }, + "audio_format": { "type": "string" }, + "sample_rate": { "type": "integer" }, + "channels": { "type": "integer" } + } + }, + "execution": { + "type": "object", + "properties": { + "model": { "type": "string" }, + "device": { "type": "string" }, + "compute_type": { "type": "string" }, + "language_requested": { "type": ["string", "null"] }, + "language_detected": { "type": ["string", "null"] }, + "language_detection_confidence": { "type": ["number", "null"] }, + "model_used": { "type": ["string", "null"] }, + "compute_type_used": { "type": ["string", "null"] }, + "downgrade_attempts": { "type": "integer" }, + "capability_tier": { "type": ["string", "null"] } + } + }, + "timings": { + "type": "object", + "properties": { + "model_load_sec": { "type": ["number", "null"] }, + "transcription_sec": { "type": ["number", "null"] }, + "ingest_sec": { "type": ["number", "null"] }, + "postprocess_sec": { "type": ["number", "null"] }, + "rtf": { "type": ["number", "null"] } + } + }, + "text": { "type": "string" }, + "segments": { + "type": "array", + "items": { + "type": "object", + "required": ["index", "start", "end", "text"], + "properties": { + "index": { "type": "integer", "minimum": 0 }, + "start": { "type": "number", "minimum": 0 }, + "end": { "type": "number", "minimum": 0 }, + "text": { "type": "string" }, + "avg_logprob": { "type": ["number", "null"] }, + "no_speech_prob": { "type": ["number", "null"] }, + "confidence": { "type": ["number", "null"] }, + "words": { "type": ["array", "null"] }, + "speaker": { "type": ["string", "null"] } + } + } + }, + "warnings": { "type": "array", "items": { "type": "string" } }, + "error": { + "type": "object", + "properties": { + "code": { "type": "string" }, + "message": { "type": "string" }, + "retryable": { "type": "boolean" } + } + }, + "postprocessing": { "type": ["object", "null"] }, + "entities": { "type": ["array", "null"] } + } +} diff --git a/docs/verification-checklist.md b/docs/verification-checklist.md new file mode 100644 index 0000000..a89cb3f --- /dev/null +++ b/docs/verification-checklist.md @@ -0,0 +1,135 @@ +# luke_scribe 전체 구현 검증 체크리스트 + +브랜치: `feat/full-platform` · 기준 문서: `.omc/plans/consensus-luke-scribe-stt-api.md` (v2.3) +검증 환경: CPU-only, ffmpeg/GPU/Redis 미설치 → **mock 기반 검증** (사용자 승인 범위) + +범례: ✅ 통과 · ⬜ 미확인 · ❌ 실패 + +## 1. 기반 (scaffold & config) + +- [x] `pyproject.toml` — 패키지 메타, 의존성(optional extras), ruff 설정 +- [x] `.env.example` — 전체 설정 키 문서화 (`LUKESCRIBE_` 프리픽스) +- [x] `run.sh` — venv 생성 + 설치 + 서버 실행 원스톱 +- [x] `src/luke_scribe/config.py` — pydantic-settings, 모델/장치/큐/상한/보관/LLM/터널 설정 +- [x] `src/luke_scribe/errors.py` — typed 오류 계층 (코드 + retryable + HTTP 매핑) +- [x] `src/luke_scribe/__init__.py` — 패키지 버전 + +## 2. 하드웨어 감지 (devices) + +- [x] `devices/vram_probe.py` — GPU/시스템 정보 조회 (nvidia-ml-py, psutil) +- [x] `devices/profile.py` — DeviceProfile 모델 (등급/정밀도/워커수/경고) +- [x] `devices/manager.py` — T0~T3 능력 등급, 정밀도 결정, 워커수 산정, override 검증 +- [x] CPU-only 폴백 + 경고 (auto 모드) + +## 3. 엔진 (engine) + +- [x] `engine/base.py` — TranscriptionOptions/TranscriptionOutcome 계약 +- [x] `engine/faster_whisper_engine.py` — faster-whisper 래퍼, lazy 세그먼트 + 협조적 취소 +- [x] `engine/model_registry.py` — 모델 다운로드/캐시/오프라인 처리 +- [x] `engine/owner.py` — **EngineOwner** 단일 GPU 소유자, 우선순위 락, OOM 강등 체인(2회 캡, 영속화) +- [x] 실시간 가설 스텁 (`emit_hypothesis`) + +## 4. 오디오 (audio) + +- [x] `audio/ingest.py` — ffprobe 형식 검증, 4h/2GB 상한, ffmpeg 16kHz 스트리밍 정규화 +- [x] 종료 경로 임시파일 정리 + 취소 시 프로세스 그룹 kill +- [x] `audio/vad.py` — 레인별 VAD 소유자 계약, 상태형 스트리밍 VAD + +## 5. 잡 큐 (jobqueue) + +- [x] `jobqueue/jobs.py` — Job 상태머신 (queued→processing→terminal, CAS, lease) +- [x] `jobqueue/broker.py` — InProc 브로커 + Redis/RQ(no-fork) 브로커 + 폴백 +- [x] `jobqueue/cancel.py` — 협조적 취소 토큰 +- [x] `jobqueue/worker.py` — 워커 (리스 하트비트, progress throttle, 크래시 복구 reconciler) +- [x] `jobqueue/redis_broker.py` — Redis 구현 (설치 시 import, 미설치 폴백) + +## 6. 파이프라인 (pipeline) + +- [x] `pipeline/batch.py` — 인제스트→전사→후처리→결과 조립, 종료 경로 정리 +- [x] `pipeline/realtime.py` — LocalAgreement (prefix 일치, 중복 확정 방지, 단조 타임스탬프, 버퍼 절단) + +## 7. 후처리 (postprocess) + +- [x] `postprocess/glossary.py` — 도메인 용어 복원 (span-aware, 이중 매칭 방지) +- [x] `postprocess/rules.py` — 표기 규칙 (vLLM 공백, 공백 축소) +- [x] `postprocess/confidence.py` — 신뢰도 플래그/게이트 +- [x] `postprocess/llm.py` — LLM 보정 (기본 off, allowlist SSRF 가드, 감사 로그) +- [x] `postprocess/pipeline.py` — 모드별 조합 (none/glossary/rules/llm) + +## 8. 결과 저장 (results) + +- [x] `results/models.py` — TranscriptResult v1 스키마 (불변 계약, 확장 필드) +- [x] `results/store.py` — UUID 키 저장, 경로 트래버설 차단, 심볼릭 링크 거부, 원자적 쓰기 +- [x] `results/formats.py` — JSON/TXT/SRT/VTT 렌더링 +- [x] `results/retention.py` — 7일 TTL (터미널 상태만, ISO/float 타임스탬프, mtime 폴백) +- [x] **privacy-first**: 전사 완료 후 원본 오디오 즉시 삭제 (`delete_source`, `delete_derived`) + +## 9. API (api) + +- [x] `api/app.py` — 앱 팩토리, lifespan(브로커/reconciler/모델 프로비저닝/터널/세션 가드) +- [x] 오류→HTTP 매핑 (401/403/404/413/422/429/500 envelope) +- [x] `api/deps.py` — KeyStore (peppered HMAC 다이제스트, 상수시간 비교, 평문 미저장) +- [x] 스코프 강제 + Job 소유권 (남의 job 접근 403) +- [x] `api/schemas.py` — 요청/응답 스키마 +- [x] `api/routes/jobs.py` — 생성(202)/조회/결과/취소, 4h/2GB 413, 큐 만재 429 +- [x] `api/routes/admin.py` — /health(공개), /v1/system, /v1/models (admin 스코프) +- [x] `api/routes/stream.py` — WebSocket 실시간 (init 인증, 프레임/세션 상한, 유휴 타임아웃) + +## 10. CLI (cli) + +- [x] `cli.py` — detect/transcribe/bench/serve/key 생성 (typer + rich) +- [x] `benchmark/` — turbo vs large-v3 벤치마크 러너/메트릭/매니페스트 + +## 11. 운영 산출물 (ops) + +- [x] `docker/Dockerfile.cpu` / `Dockerfile.gpu` / `docker-compose.yml` +- [x] `connectivity/tunnel.py` — Cloudflare Quick Tunnel +- [x] `observability/` — 로깅(JSON 옵션) + 메트릭 +- [x] `diarization/` — pyannote 화자 분리 스텁 +- [x] `README.md` — 설치/사용법/API/CLI 문서 +- [x] `docs/schemas/transcript-result-v1.schema.json` — 결과 JSON 스키마 +- [x] `benchmarks/manifest.yaml.example`, `samples/README.md` + +## 12. 테스트 & 품질 + +- [x] 단위 테스트: devices, engine owner, jobs/broker, metrics, postprocess, realtime, formats, store, keystore, ingest +- [x] 통합 테스트: worker 수명주기/취소/크래시복구, API 인증/소유권/큐/결과 +- [x] 127개 테스트 전체 통과 +- [x] ruff lint + format 클린 +- [x] 코드 리뷰 반영 (프라이버시 계약, 보관 갭, Redis 오류 필드, 완료 순서, 유휴 타임아웃, 중복 확정) + +--- + +## 실행 검증 (이 환경에서 직접 실행한 것) + +```bash +# pytest 결과 +127 passed, 1 warning in 0.32s + +# ruff +All checks passed! | 68 files already formatted (FORMAT CLEAN) + +# import 검증 +40/40 모듈 import 성공 + +# CLI +luke-scribe --help → detect/transcribe/bench/serve/key 정상 +luke-scribe detect → device=cpu | ct=int8 | tier=T0 +luke-scribe key --create → 1회 키 출력 + 다이제스트 64자 hex만 저장 (평문 없음) + +# API 스모크 (TestClient — 전부 PASS) +health → PASS (status=ok, model_ready=false) +no-key 401 → PASS +bad-key 401 → PASS +scope 403 → PASS +create 202 → PASS +job get 200 → PASS +result 409(미완료) → PASS +cancel 200 → PASS +oversize 413 → PASS +queue-full 429 → PASS (Retry-After: 5 헤더 포함) + +# 산출물 +README/run.sh/pyproject/.env.example/Docker×2/compose/schema/manifest → 전부 존재 +schema JSON valid · pyproject 파싱 OK (extras 6종) · run.sh 구문 OK · compose YAML valid +``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..64c0c8d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,58 @@ +[project] +name = "luke-scribe" +version = "0.1.0" +description = "내부용 로컬 STT 전사 API — faster-whisper, hardware-adaptive, privacy-first" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "pydantic>=2.7", + "pydantic-settings>=2.3", + "typer>=0.12", + "rich>=13.7", + "psutil>=5.9", + "nvidia-ml-py>=12.535", + "huggingface-hub>=0.24", +] + +[project.optional-dependencies] +# 엔진 — transcribe/bench (faster-whisper/CTranslate2) +engine = ["faster-whisper>=1.0.3", "av>=11"] +# GPU CUDA 런타임 (faster-whisper GPU 추론 시) +gpu = ["nvidia-cublas-cu12", "nvidia-cudnn-cu12"] +# 동기 배치 API (serve) +api = ["fastapi>=0.110", "uvicorn[standard]>=0.29", "python-multipart>=0.0.9"] +# P2 비동기 큐 (Redis/RQ no-fork) +queue = ["redis>=5.0", "rq>=1.16"] +# P5 옵션 +diarize = ["pyannote.audio>=3.1"] +llm = ["openai>=1.30"] + +[dependency-groups] +dev = ["pytest>=8.2", "ruff>=0.5", "httpx>=0.27"] + +[project.scripts] +luke-scribe = "luke_scribe.cli:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/luke_scribe"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "W", "UP", "B"] +ignore = ["E501"] + +[tool.ruff.lint.per-file-ignores] +# FastAPI/typer는 Depends()/Option() 호출을 인자 기본값에 쓰는 패턴이 표준 +"src/luke_scribe/api/**" = ["B008"] +"src/luke_scribe/cli.py" = ["B008"] diff --git a/run.sh b/run.sh new file mode 100644 index 0000000..2d3b9ad --- /dev/null +++ b/run.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# luke_scribe dev launcher — Colab/개발용 순수 Python 경로 (Docker 불필요) +set -euo pipefail + +cd "$(dirname "$0")" + +if [ ! -d .venv ]; then + echo "[luke-scribe] .venv 생성 중..." + python3 -m venv .venv +fi +# shellcheck disable=SC1091 +source .venv/bin/activate + +if [ -f .env ]; then + set -a + # shellcheck disable=SC1091 + source .env + set +a +fi + +CMD="${1:-help}" +shift || true + +case "$CMD" in + detect) + python -m luke_scribe.cli detect "$@" + ;; + transcribe) + python -m luke_scribe.cli transcribe "$@" + ;; + bench) + python -m luke_scribe.cli bench "$@" + ;; + serve) + python -m luke_scribe.cli serve "$@" + ;; + key) + python -m luke_scribe.cli key "$@" + ;; + test) + python -m pytest tests/ "$@" + ;; + *) + echo "사용법: ./run.sh {detect|transcribe|bench|serve|key|test} [args]" + echo + echo "예시:" + echo " ./run.sh detect" + echo " ./run.sh transcribe samples/ko_en.wav --language ko" + echo " ./run.sh serve" + echo " ./run.sh key --create" + exit 1 + ;; +esac diff --git a/samples/README.md b/samples/README.md new file mode 100644 index 0000000..64b743b --- /dev/null +++ b/samples/README.md @@ -0,0 +1,4 @@ +# samples + +`hello-ko-en.wav` — 16kHz mono, 짧은 한국어+영문 기술용어 발화 샘플. +(라이선스 확인 후 실제 오디오 파일을 추가할 것. 개인정보 포함 원본 커밋 금지.) diff --git a/src/luke_scribe/__init__.py b/src/luke_scribe/__init__.py new file mode 100644 index 0000000..2ed8114 --- /dev/null +++ b/src/luke_scribe/__init__.py @@ -0,0 +1,6 @@ +"""luke_scribe — 내부용 로컬 STT 전사 API. + +faster-whisper 기반, 하드웨어 적응형, privacy-first. +""" + +__version__ = "0.1.0" diff --git a/src/luke_scribe/api/__init__.py b/src/luke_scribe/api/__init__.py new file mode 100644 index 0000000..f2a884c --- /dev/null +++ b/src/luke_scribe/api/__init__.py @@ -0,0 +1 @@ +"""FastAPI 웹 API — 배치 Job, 실시간 WS, admin.""" diff --git a/src/luke_scribe/api/app.py b/src/luke_scribe/api/app.py new file mode 100644 index 0000000..2aadf21 --- /dev/null +++ b/src/luke_scribe/api/app.py @@ -0,0 +1,147 @@ +"""FastAPI 앱 팩토리. + +Lifespan (plan §3.10a/§3.5): +- 시작: broker(Redis/in-proc) 생성, stale job reconciler 1회, EngineOwner 생성, + 모델 프로비저닝(선택) — 준비 전 ``/health``는 ``model_ready=false``. +- 종료: 터널 종료, 열린 리소스 정리. +""" + +from __future__ import annotations + +import logging +from contextlib import asynccontextmanager + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse + +from ..config import Settings, get_settings +from ..engine.owner import EngineOwner +from ..errors import ( + AuthError, + InvalidInput, + JobNotFound, + LukeScribeError, + QueueFull, + ScopeDenied, + UnsupportedInputEnvelope, +) +from ..results.store import ResultStore +from .deps import KeyStore +from .routes import admin, jobs, stream + +logger = logging.getLogger("luke_scribe.api") + + +def create_app(settings: Settings | None = None) -> FastAPI: + settings = settings or get_settings() + + @asynccontextmanager + async def lifespan(app: FastAPI): + from ..jobqueue.broker import make_broker + + broker = make_broker(settings) + store = ResultStore(settings.results_root) + owner = EngineOwner.get(settings) + keystore = KeyStore(settings) + + # 스타트업 reconciler — stale processing 복구 + try: + reclaimed = broker.reconcile_stale(60.0) + if reclaimed: + logger.warning("스타트업 reconciler: %d stale job 복구", len(reclaimed)) + except Exception as exc: + logger.warning("reconciler 실패: %s", exc) + + app.state.settings = settings + app.state.broker = broker + app.state.store = store + app.state.engine_owner = owner + app.state.keystore = keystore + app.state.model_ready = False + app.state.model_ready_model = None + from .routes.stream import _SessionGuard + + app.state.session_guard = _SessionGuard(settings) + + # 모델 프로비저닝 (선택) — 설정된 경우에만 + model_cache = settings.model_cache_dir + if model_cache: + try: + from ..engine.model_registry import ModelRegistry + + reg = ModelRegistry(settings) + path = reg.ensure_available( + settings.model_batch, cache_dir=model_cache, offline_ok=True + ) + app.state.model_ready = True + app.state.model_ready_model = settings.model_batch + logger.info("모델 준비됨: %s (%s)", settings.model_batch, path) + except Exception as exc: + logger.warning("모델 프로비저닝 보류: %s", exc) + + # 터널 (선택) + tunnel = None + if settings.tunnel != "none": + try: + from ..connectivity.tunnel import CloudflareTunnel + + tunnel = CloudflareTunnel(settings) + tunnel.start() + app.state.tunnel_url = tunnel.public_url + logger.info("터널: %s", tunnel.public_url) + except Exception as exc: + logger.warning("터널 시작 실패: %s", exc) + + yield + + if tunnel is not None: + try: + tunnel.stop() + except Exception: + pass + owner.unload_all() + + app = FastAPI( + title="luke_scribe", + version="0.1.0", + description="내부용 로컬 STT 전사 API — faster-whisper, hardware-adaptive, privacy-first", + lifespan=lifespan, + ) + + app.include_router(admin.router) + app.include_router(jobs.router) + app.include_router(stream.router) + + # ── 오류 → HTTP 매핑 (LukeScribeError envelope) ── + @app.exception_handler(AuthError) + async def _auth_error(request: Request, exc: AuthError) -> JSONResponse: + return JSONResponse(status_code=401, content=exc.to_dict()) + + @app.exception_handler(ScopeDenied) + async def _scope_error(request: Request, exc: ScopeDenied) -> JSONResponse: + return JSONResponse(status_code=403, content=exc.to_dict()) + + @app.exception_handler(JobNotFound) + async def _job_not_found(request: Request, exc: JobNotFound) -> JSONResponse: + return JSONResponse(status_code=404, content=exc.to_dict()) + + @app.exception_handler(QueueFull) + async def _queue_full(request: Request, exc: QueueFull) -> JSONResponse: + return JSONResponse(status_code=429, content=exc.to_dict(), headers={"Retry-After": "5"}) + + @app.exception_handler(UnsupportedInputEnvelope) + async def _envelope(request: Request, exc: UnsupportedInputEnvelope) -> JSONResponse: + return JSONResponse(status_code=413, content=exc.to_dict()) + + @app.exception_handler(InvalidInput) + async def _invalid_input(request: Request, exc: InvalidInput) -> JSONResponse: + return JSONResponse(status_code=422, content=exc.to_dict()) + + @app.exception_handler(LukeScribeError) + async def _generic_error(request: Request, exc: LukeScribeError) -> JSONResponse: + return JSONResponse(status_code=500, content=exc.to_dict()) + + return app + + +app = create_app() diff --git a/src/luke_scribe/api/deps.py b/src/luke_scribe/api/deps.py new file mode 100644 index 0000000..860b6f0 --- /dev/null +++ b/src/luke_scribe/api/deps.py @@ -0,0 +1,127 @@ +"""인증/권한 디펜던시. + +Eng 리뷰 P14 반영: +- 키는 평문으로 저장하지 않는다 — **peppered HMAC-SHA256 다이제스트**만 저장. +- 비교는 ``secrets.compare_digest`` (상수 시간). +- 스코프 강제 + **Job 소유권** (transcribe 키가 남의 Job을 조회/취소 불가). +- raw 키는 로그/큐에 절대 기록하지 않는다. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import secrets +from dataclasses import dataclass +from pathlib import Path + +from fastapi import Depends, Header, Request +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer + +from ..config import Settings +from ..errors import AuthError, ScopeDenied + +_bearer = HTTPBearer(auto_error=False) +_PEPPER = b"luke-scribe-key-pepper-v1" + + +class KeyStore: + """키 다이제스트 저장소 — {key_id: {digest, scopes}}. + + ``api_keys`` 설정(평문, dev용) 또는 ``api_key_file``(다이제스트, 권장)를 + 소스로 사용한다. + """ + + def __init__(self, settings: Settings | None = None) -> None: + self.settings = settings or Settings() + self._keys: dict[str, dict] = {} + self._load() + + def _load(self) -> None: + # 1) api_key_file (다이제스트) + path = Path(self.settings.api_key_file) + if path.exists(): + try: + data = json.loads(path.read_text(encoding="utf-8")) + for entry in data.get("keys", []): + self._keys[entry["id"]] = { + "digest": entry["digest"], + "scopes": set(entry.get("scopes", ["transcribe"])), + } + except Exception: + pass + # 2) 설정 평문 키 (dev 편의) → 다이제스트로 변환 + for raw_key, scopes in self.settings.parse_api_keys().items(): + self._keys.setdefault( + self._id_of(raw_key), {"digest": self._digest(raw_key), "scopes": scopes} + ) + + @staticmethod + def _id_of(raw_key: str) -> str: + return f"k-{raw_key[:8]}" + + @staticmethod + def _digest(raw_key: str) -> str: + return hmac.new(_PEPPER, raw_key.encode("utf-8"), hashlib.sha256).hexdigest() + + def authenticate(self, raw_key: str | None) -> Principal: + if not raw_key: + raise AuthError("API 키가 필요합니다 (X-API-Key 헤더)") + for key_id, entry in self._keys.items(): + if secrets.compare_digest(entry["digest"], self._digest(raw_key)): + return Principal(key_id=key_id, scopes=entry["scopes"]) + raise AuthError("유효하지 않은 API 키") + + def create_key(self, scopes: list[str] | None = None, *, save_path: str | None = None) -> dict: + """새 키 생성 — raw 키는 1회만 반환하고 다이제스트만 저장.""" + raw = f"luke-{secrets.token_urlsafe(32)}" + scopes = scopes or ["transcribe"] + key_id = self._id_of(raw) + self._keys[key_id] = {"digest": self._digest(raw), "scopes": set(scopes)} + save_path = save_path or self.settings.api_key_file + Path(save_path).parent.mkdir(parents=True, exist_ok=True) + data = { + "keys": [ + {"id": k, "digest": v["digest"], "scopes": sorted(v["scopes"])} + for k, v in self._keys.items() + ] + } + Path(save_path).write_text(json.dumps(data, indent=2), encoding="utf-8") + return {"key_id": key_id, "key": raw, "scopes": scopes, "stored_at": save_path} + + +@dataclass +class Principal: + key_id: str + scopes: set[str] + + def require_scope(self, scope: str) -> None: + if scope not in self.scopes: + raise ScopeDenied(f"스코프 {scope!r}가 필요합니다 (가진 스코프: {sorted(self.scopes)})") + + +def get_keystore(request: Request) -> KeyStore: + """앱 상태에 있는 keystore 사용 (lifespan이 설정 기반으로 생성).""" + return request.app.state.keystore + + +def get_principal( + credentials: HTTPAuthorizationCredentials | None = Depends(_bearer), + x_api_key: str | None = Header(default=None), + keystore: KeyStore = Depends(get_keystore), +) -> Principal: + raw = None + if credentials: + raw = credentials.credentials + elif x_api_key: + raw = x_api_key + return keystore.authenticate(raw) + + +def require_scope(scope: str): + def dep(principal: Principal = Depends(get_principal)) -> Principal: + principal.require_scope(scope) + return principal + + return dep diff --git a/src/luke_scribe/api/routes/__init__.py b/src/luke_scribe/api/routes/__init__.py new file mode 100644 index 0000000..98f7d3b --- /dev/null +++ b/src/luke_scribe/api/routes/__init__.py @@ -0,0 +1 @@ +"""API 라우트 — jobs / stream / admin.""" diff --git a/src/luke_scribe/api/routes/admin.py b/src/luke_scribe/api/routes/admin.py new file mode 100644 index 0000000..f8e8b6b --- /dev/null +++ b/src/luke_scribe/api/routes/admin.py @@ -0,0 +1,64 @@ +"""Admin 라우트 — /health, /v1/system, /v1/models. + +계약: health는 공개(최소 정보), system/models는 admin 스코프 필요. +``/v1/system``은 요청 vs 실효 compute_type 불일치 경고를 포함한다 (§3.9d). +""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, Request + +from ...devices.manager import DeviceManager +from ...devices.vram_probe import runtime_env_snapshot +from ..deps import Principal, require_scope +from ..schemas import SystemResponse + +router = APIRouter(tags=["admin"]) + + +@router.get("/health") +async def health(request: Request) -> dict: + broker = request.app.state.broker + return { + "status": "ok", + "queue_depth": broker.queue_depth(), + "model_ready": getattr(request.app.state, "model_ready", False), + } + + +@router.get("/v1/system", response_model=SystemResponse) +async def system( + request: Request, + principal: Principal = Depends(require_scope("admin")), +) -> SystemResponse: + settings = request.app.state.settings + manager = DeviceManager() + profile = manager.detect( + device=settings.device, + compute_type=None if settings.compute_type == "auto" else settings.compute_type, + ) + return SystemResponse( + capability_tier=profile.capability_tier, + device=profile.to_dict(), + workers=manager.compute_workers(profile, override=settings.workers), + queue_depth=request.app.state.broker.queue_depth(), + models=[settings.model_rt, settings.model_batch], + model_used=getattr(request.app.state, "model_ready_model", None), + compute_type_used=profile.selected_compute_type, + ) + + +@router.get("/v1/models") +async def models( + request: Request, + principal: Principal = Depends(require_scope("admin")), +) -> dict: + settings = request.app.state.settings + manager = DeviceManager() + profile = manager.detect(device=settings.device) + return { + "capability_tier": profile.capability_tier, + "loadable_models": profile.loadable_models or [settings.model_rt, settings.model_batch], + "recommended_model": profile.recommended_model, + "runtime": runtime_env_snapshot(), + } diff --git a/src/luke_scribe/api/routes/jobs.py b/src/luke_scribe/api/routes/jobs.py new file mode 100644 index 0000000..2983b94 --- /dev/null +++ b/src/luke_scribe/api/routes/jobs.py @@ -0,0 +1,177 @@ +"""Job 라우트 — 생성/조회/취소/목록. + +- ``POST /v1/jobs``: multipart 업로드. 4h/2GB 초과 → 413, 큐 만재 → 429. +- ``GET /v1/jobs/{id}``: queue_position/progress/상태 (소유자 전용). +- ``GET /v1/jobs/{id}/result?format=``: 결과 (enum 포맷만, UUID 키). +- ``DELETE /v1/jobs/{id}``: 협조적 취소. +- 소유권: Job을 만든 키만 조회/취소/결과 접근 가능 (Eng 리뷰 P14). +""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile +from pydantic import ValidationError + +from ...config import Settings +from ...errors import ( + JobNotFound, + QueueFull, +) +from ...jobqueue.broker import JobBroker +from ...jobqueue.jobs import Job, JobStatus +from ...results.formats import render +from ...results.store import ResultStore +from ..deps import Principal, get_principal, require_scope +from ..schemas import JobCreateResponse, JobResultResponse, JobStatusResponse, TranscribeOptions + +router = APIRouter(prefix="/v1/jobs", tags=["jobs"]) + +REQUIRED_SCOPE = "transcribe" + + +def _get_broker(request: Request) -> JobBroker: + return request.app.state.broker + + +def _get_store(request: Request) -> ResultStore: + return request.app.state.store + + +@router.post("", response_model=JobCreateResponse, status_code=202) +async def create_job( + request: Request, + file: Annotated[UploadFile, File()], + options: Annotated[str, Form()] = '{"language":"ko"}', + principal: Principal = Depends(require_scope("transcribe")), +) -> JobCreateResponse: + + try: + opts = TranscribeOptions.model_validate_json(options) + except ValidationError as exc: + raise HTTPException(status_code=422, detail=f"options 파싱 실패: {exc.errors()}") from exc + + broker = _get_broker(request) + store = _get_store(request) + settings: Settings = request.app.state.settings + + # 2GB 상한 + content = await file.read(settings.max_upload_bytes + 1) + if len(content) > settings.max_upload_bytes: + raise HTTPException(status_code=413, detail="파일 크기가 상한(2GB)을 초과합니다") + + # 큐 만재 → 429 + if broker.queue_depth() >= settings.max_queue: + raise HTTPException( + status_code=429, detail="큐가 가득 찼습니다", headers={"Retry-After": "5"} + ) + + job = Job( + type="file", + lane="batch", + owner_key_id=principal.key_id, + options=opts.model_dump(exclude_none=True), + source_name=file.filename, + ) + source_path = store.source_path_for(job.id, file.filename or "upload.bin") + source_path.write_bytes(content) + job.source_path = str(source_path) + try: + broker.enqueue(job) + except QueueFull: + store.delete_job(job.id) + raise HTTPException( + status_code=429, detail="큐가 가득 찼습니다", headers={"Retry-After": "5"} + ) from None + + # queue_position 스냅샷 + pos = broker.queue_depth() - 1 + job.queue_position = pos + return JobCreateResponse(job_id=job.id, status=job.status.value, queue_position=max(pos, 0)) + + +@router.get("", response_model=list[JobStatusResponse]) +async def list_jobs( + request: Request, + principal: Principal = Depends(get_principal), +) -> list[JobStatusResponse]: + principal.require_scope("transcribe") + broker = _get_broker(request) + out = [] + for job in broker.list_jobs(limit=100): + if job.owner_key_id != principal.key_id: + continue + out.append(_to_status(job)) + return out + + +@router.get("/{job_id}", response_model=JobStatusResponse) +async def get_job( + job_id: str, + request: Request, + principal: Principal = Depends(get_principal), +) -> JobStatusResponse: + principal.require_scope("transcribe") + broker = _get_broker(request) + job = broker.get(job_id) + if job is None: + raise HTTPException(status_code=404, detail="job not found") + _check_owner(job, principal) + return _to_status(job) + + +@router.get("/{job_id}/result", response_model=JobResultResponse) +async def get_result( + job_id: str, + request: Request, + format: Literal["json", "txt", "srt", "vtt"] = "json", + principal: Principal = Depends(get_principal), +) -> JobResultResponse: + principal.require_scope("transcribe") + broker = _get_broker(request) + store = _get_store(request) + job = broker.get(job_id) + if job is None: + raise HTTPException(status_code=404, detail="job not found") + _check_owner(job, principal) + result = store.read_result(job_id) + if result is None: + raise HTTPException(status_code=409, detail="결과가 아직 준비되지 않았습니다") + return JobResultResponse(format=format, content=render(result, format)) + + +@router.delete("/{job_id}", response_model=JobStatusResponse) +async def cancel_job( + job_id: str, + request: Request, + principal: Principal = Depends(require_scope("transcribe")), +) -> JobStatusResponse: + broker = _get_broker(request) + job = broker.get(job_id) + if job is None: + raise HTTPException(status_code=404, detail="job not found") + _check_owner(job, principal) + try: + cancelled = broker.cancel(job_id) + except JobNotFound: + raise HTTPException(status_code=404, detail="job not found") from None + return _to_status(cancelled) + + +def _check_owner(job: Job, principal: Principal) -> None: + if job.owner_key_id and job.owner_key_id != principal.key_id: + raise HTTPException(status_code=403, detail="다른 키가 만든 job에 접근할 수 없습니다") + + +def _to_status(job: Job) -> JobStatusResponse: + return JobStatusResponse( + job_id=job.id, + status=job.status.value, + queue_position=job.queue_position, + progress=job.progress, + processed_sec=job.processed_sec, + total_sec=job.total_sec, + error={"code": job.error_code, "message": job.error_message} if job.error_code else None, + result_available=job.status == JobStatus.COMPLETED, + ) diff --git a/src/luke_scribe/api/routes/stream.py b/src/luke_scribe/api/routes/stream.py new file mode 100644 index 0000000..827c830 --- /dev/null +++ b/src/luke_scribe/api/routes/stream.py @@ -0,0 +1,153 @@ +"""실시간 WebSocket 라우트 (plan §3.10b, §3.9c). + +- 첫 메시지 = init 프레임(인증 + 오디오 협상), 2초 내 미수신 시 close. +- unauthenticated 소켓/IP당 상한, init/오디오 프레임 크기 상한 (Eng P17). +- partial/final/status 이벤트 (LocalAgreement 안정화). +- 실시간 레인은 EngineOwner의 단일 GPU 락을 공유 (배치와 충돌 없음). +- 최대 동시 세션 상한 초과 → close. +""" + +from __future__ import annotations + +import asyncio +import json +import time + +from fastapi import APIRouter, WebSocket, WebSocketDisconnect + +from ...config import Settings +from ...pipeline.realtime import LocalAgreement +from ..deps import KeyStore + +router = APIRouter(tags=["realtime"]) + +# 세션/프레임 상한 +MAX_INIT_BYTES = 16 * 1024 +MAX_FRAME_BYTES = 512 * 1024 +MAX_PENDING_PER_IP = 8 + + +class _SessionGuard: + def __init__(self, settings: Settings) -> None: + self.settings = settings + self._active: set[str] = set() + self._pending_per_ip: dict[str, int] = {} + + def try_acquire(self, session_id: str, ip: str) -> bool: + if len(self._active) >= self.settings.realtime_max_sessions: + return False + if self._pending_per_ip.get(ip, 0) >= MAX_PENDING_PER_IP: + return False + self._active.add(session_id) + self._pending_per_ip[ip] = self._pending_per_ip.get(ip, 0) + 1 + return True + + def release(self, session_id: str, ip: str) -> None: + self._active.discard(session_id) + n = self._pending_per_ip.get(ip, 1) - 1 + if n <= 0: + self._pending_per_ip.pop(ip, None) + else: + self._pending_per_ip[ip] = n + + +async def _recv_json(ws: WebSocket, max_bytes: int, timeout_sec: float) -> dict | None: + try: + msg = await asyncio.wait_for(ws.receive_text(), timeout=timeout_sec) + except TimeoutError: + return None + except WebSocketDisconnect: + return None + if len(msg.encode("utf-8")) > max_bytes: + return None + try: + return json.loads(msg) + except json.JSONDecodeError: + return None + + +@router.websocket("/v1/stream") +async def stream(ws: WebSocket) -> None: + await ws.accept() + settings: Settings = ws.app.state.settings + keystore: KeyStore = ws.app.state.keystore + guard: _SessionGuard = ws.app.state.session_guard + ip = ws.client.host if ws.client else "unknown" + session_id = f"{ip}:{time.time_ns()}" + + if not guard.try_acquire(session_id, ip): + await ws.close(code=1013, reason="session limit reached") + return + try: + init = await _recv_json(ws, MAX_INIT_BYTES, settings.realtime_init_timeout_sec) + if init is None or init.get("type") != "init": + await ws.close(code=1008, reason="init frame required") + return + api_key = init.get("api_key") + principal = None + if api_key: + try: + principal = keystore.authenticate(api_key) + except Exception: + principal = None + if principal is None: + await ws.close(code=1008, reason="invalid api key") + return + + audio = init.get("audio") or {} + if not isinstance(audio, dict) or not audio.get("sample_rate"): + await ws.close(code=1003, reason="audio params required") + return + + agreement = LocalAgreement(settings) + await ws.send_json({"type": "status", "status": "ready", "session": session_id}) + + # 오디오 프레임 루프 — PCM16 청크를 가정 (v0.1: decode는 EngineOwner가 소유) + owner = ws.app.state.engine_owner + buf_sec = 0.0 + frames = 0 + while True: + try: + msg = await asyncio.wait_for( + ws.receive(), timeout=settings.realtime_session_idle_sec + ) + except TimeoutError: + # 세션 유휴 타임아웃 — 점유 해제 (Eng 리뷰 P17) + await ws.send_json( + {"type": "status", "status": "idle_timeout", "message": "유휴 세션 종료"} + ) + break + if msg.get("type") == "websocket.disconnect": + break + data = msg.get("bytes") + if not data or len(data) > MAX_FRAME_BYTES: + await ws.close(code=1009, reason="frame too large") + return + frames += 1 + # 16kHz mono s16 → chunk_duration + chunk_sec = len(data) / 2 / 16000 + buf_sec += chunk_sec + if buf_sec >= 5.0: # 재decode 간격 (정확도 우선 청킹) + # 가설 생성 — 실제 decode는 EngineOwner로 오프로드 (계약 §3.9a) + hypothesis = await asyncio.to_thread(owner.emit_hypothesis, data) + events = agreement.feed(hypothesis["segments"], chunk_sec) + for seg in events["confirmed"]: + await ws.send_json({"type": "final", "segment": seg.model_dump()}) + for seg in events.get("pending", []): + await ws.send_json( + {"type": "partial", "text": seg.text, "t0": seg.start, "t1": seg.end} + ) + buf_sec = 0.0 + except WebSocketDisconnect: + pass + except Exception as exc: + try: + await ws.send_json({"type": "status", "status": "error", "message": str(exc)}) + except Exception: + pass + finally: + guard.release(session_id, ip) + try: + await ws.close() + except Exception: + pass diff --git a/src/luke_scribe/api/schemas.py b/src/luke_scribe/api/schemas.py new file mode 100644 index 0000000..80a88b0 --- /dev/null +++ b/src/luke_scribe/api/schemas.py @@ -0,0 +1,73 @@ +"""API 요청/응답 스키마 (pydantic). + +- 업로드 옵션: language/model/device/compute_type/timestamps/word_timestamps/ + formats/hotwords/vad/post_correction +- 오류 envelope: code/message/retryable +""" + +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import BaseModel, Field + + +class TranscribeOptions(BaseModel): + language: str | None = "ko" # auto = null + model: str | None = None + device: str = "auto" + compute_type: str | None = None + timestamps: bool = True + word_timestamps: bool = False + formats: list[str] = Field(default_factory=lambda: ["json"]) + hotwords: list[str] = Field(default_factory=list) + vad: bool = True + glossary_id: str | None = None + post_correction: dict[str, Any] | None = None + diarize: bool = False + + +class JobCreateResponse(BaseModel): + job_id: str + status: str + queue_position: int | None = None + + +class JobStatusResponse(BaseModel): + job_id: str + status: str + queue_position: int | None = None + jobs_ahead: int | None = None + progress: float | None = None + processed_sec: float | None = None + total_sec: float | None = None + error: dict[str, Any] | None = None + result_available: bool = False + + +class JobResultResponse(BaseModel): + format: str + content: str + + +class SystemResponse(BaseModel): + capability_tier: str + device: dict[str, Any] + workers: int + queue_depth: int + models: list[str] + model_used: str | None = None + compute_type_used: str | None = None + + +class ErrorResponse(BaseModel): + code: str + message: str + retryable: bool = False + + +class StreamInit(BaseModel): + type: Literal["init"] + api_key: str + audio: dict[str, Any] = Field(default_factory=dict) + options: dict[str, Any] = Field(default_factory=dict) diff --git a/src/luke_scribe/audio/__init__.py b/src/luke_scribe/audio/__init__.py new file mode 100644 index 0000000..2813916 --- /dev/null +++ b/src/luke_scribe/audio/__init__.py @@ -0,0 +1 @@ +"""오디오 인제스트(ffmpeg)와 VAD.""" diff --git a/src/luke_scribe/audio/ingest.py b/src/luke_scribe/audio/ingest.py new file mode 100644 index 0000000..2f41de3 --- /dev/null +++ b/src/luke_scribe/audio/ingest.py @@ -0,0 +1,255 @@ +"""AudioIngestor — 입력 검증 + ffmpeg 16kHz mono 정규화 (스트리밍). + +계약 (plan §3.7b/§6.1 + design doc §3): +- 확장자가 아니라 **ffprobe**로 형식 검증. +- 60분/1GB(설계문서 v0.1) 또는 4h/2GB(플랫폼, config) 상한 — 초과 시 + ``unsupported_input_envelope``. +- ffmpeg는 전체 배열을 메모리에 올리지 않고 **파일로 스트리밍**. +- 모든 종료 경로에서 임시 파일 정리 (``finally``), 원본은 읽기 전용. +- 취소/오류 시 ffmpeg 프로세스 그룹 kill + reap (Eng 리뷰 P1). +- ffprobe/ffmpeg timeout: 기본 60s, CLI 옵션으로만 상향. +""" + +from __future__ import annotations + +import os +import shutil +import signal +import subprocess +import tempfile +from dataclasses import dataclass, field +from pathlib import Path + +from ..config import Settings +from ..errors import AudioProbeFailed, InvalidInput, UnsupportedInputEnvelope +from ..results.models import NormalizedAudio + + +@dataclass +class ProbeResult: + duration_sec: float | None + codec: str | None + size_bytes: int + has_audio: bool = True + + +@dataclass +class IngestResult: + normalized_path: str + normalized: NormalizedAudio + probe: ProbeResult + temp_dir: str + cleanup: callable = field(repr=False) + + def close(self) -> None: + self.cleanup() + + +class AudioIngestor: + def __init__( + self, + settings: Settings | None = None, + *, + ffprobe_bin: str = "ffprobe", + ffmpeg_bin: str = "ffmpeg", + probe_timeout: int = 60, + ) -> None: + self.settings = settings or Settings() + self._ffprobe = ffprobe_bin + self._ffmpeg = ffmpeg_bin + self._probe_timeout = probe_timeout + + def probe(self, source: Path) -> ProbeResult: + if not source.exists(): + raise InvalidInput(f"파일이 존재하지 않습니다: {source}") + if source.stat().st_size > self.settings.max_upload_bytes: + raise UnsupportedInputEnvelope( + f"파일 크기 {source.stat().st_size} 바이트가 상한 " + f"{self.settings.max_upload_bytes} 바이트를 초과합니다" + ) + cmd = [ + self._ffprobe, + "-v", + "error", + "-show_entries", + "format=duration:stream=codec_type,codec_name", + "-of", + "json", + str(source), + ] + try: + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=self._probe_timeout) + except FileNotFoundError as exc: + raise AudioProbeFailed( + f"ffprobe를 찾을 수 없습니다 ({self._ffprobe}). ffmpeg를 설치하세요: " + "apt install ffmpeg / brew install ffmpeg" + ) from exc + except subprocess.TimeoutExpired as exc: + raise AudioProbeFailed("ffprobe timeout (60s)") from exc + + if proc.returncode != 0: + raise AudioProbeFailed(f"입력 오디오를 해석할 수 없습니다. {proc.stderr.strip()[:300]}") + import json + + try: + data = json.loads(proc.stdout or "{}") + except Exception as exc: + raise AudioProbeFailed(f"ffprobe 출력 파싱 실패: {exc}") from exc + + streams = data.get("streams", []) + audio_streams = [s for s in streams if s.get("codec_type") == "audio"] + if not audio_streams: + raise AudioProbeFailed("입력에 오디오 스트림이 없습니다") + + duration = None + fmt = data.get("format") or {} + if fmt.get("duration"): + try: + duration = float(fmt["duration"]) + except (TypeError, ValueError): + duration = None + if duration is not None and duration > self.settings.max_duration_sec: + raise UnsupportedInputEnvelope( + f"오디오 길이 {duration:.0f}초가 상한 {self.settings.max_duration_sec}초를 초과합니다" + ) + return ProbeResult( + duration_sec=duration, + codec=audio_streams[0].get("codec_name"), + size_bytes=source.stat().st_size, + ) + + def ingest(self, source: Path, *, should_cancel: callable | None = None) -> IngestResult: + """검증 + ffmpeg로 16kHz mono PCM s16 WAV 정규화.""" + if should_cancel and should_cancel(): + from ..errors import CancelledError + + raise CancelledError("취소됨") + probe = self.probe(source) + + temp_dir = tempfile.mkdtemp(prefix="luke-scribe-ingest-") + out_path = Path(temp_dir) / "normalized.wav" + + def cleanup() -> None: + shutil.rmtree(temp_dir, ignore_errors=True) + + try: + if should_cancel and should_cancel(): + from ..errors import CancelledError + + raise CancelledError("취소됨") + cmd = [ + self._ffmpeg, + "-v", + "error", + "-y", + "-i", + str(source), + "-ac", + "1", + "-ar", + "16000", + "-c:a", + "pcm_s16le", + "-f", + "wav", + str(out_path), + ] + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, # 프로세스 그룹 (kill/reap 용이) + ) + # stdout/stderr 병렬 drain (ffmpeg stderr 블로킹 방지) + assert proc.stdout is not None and proc.stderr is not None + out_thread = self._drain(proc.stdout) + err_data = self._read_all(proc.stderr) + + while True: + try: + rc = proc.wait(timeout=0.5) + break + except subprocess.TimeoutExpired: + if should_cancel and should_cancel(): + self._kill_group(proc) + from ..errors import CancelledError + + raise CancelledError("인제스트 중 취소") from None + out_thread.join(timeout=5) + if rc != 0: + raise AudioProbeFailed( + f"ffmpeg 정규화 실패 (exit {rc}): {err_data.decode(errors='replace')[:300]}" + ) + if not out_path.exists() or out_path.stat().st_size == 0: + raise AudioProbeFailed("ffmpeg가 출력 WAV를 생성하지 못했습니다") + + duration = probe.duration_sec + if duration is None: + # ffprobe가 몰랐던 경우: 파생 파일로 재측정 + duration = self._duration_of(out_path) or 0.0 + return IngestResult( + normalized_path=str(out_path), + normalized=NormalizedAudio( + duration_sec=max(duration, 0.001), + audio_format="pcm_s16le", + sample_rate=16000, + channels=1, + ), + probe=probe, + temp_dir=temp_dir, + cleanup=cleanup, + ) + except BaseException: + cleanup() + raise + + def _duration_of(self, wav: Path) -> float | None: + """ffprobe로 파생 WAV 길이 재측정.""" + try: + out = subprocess.run( + [ + self._ffprobe, + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "json", + str(wav), + ], + capture_output=True, + text=True, + timeout=10, + ) + import json + + d = json.loads(out.stdout or "{}").get("format", {}).get("duration") + return float(d) if d else None + except Exception: + return None + + @staticmethod + def _drain(stream) -> os._wrap_close: + import threading + + def _read() -> None: + while stream.read(65536): + pass + + t = threading.Thread(target=_read, daemon=True) + t.start() + return t # type: ignore[return-value] + + @staticmethod + def _read_all(stream) -> bytes: + return stream.read() + + @staticmethod + def _kill_group(proc: subprocess.Popen) -> None: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + except (ProcessLookupError, PermissionError): + try: + proc.kill() + except Exception: + pass diff --git a/src/luke_scribe/audio/vad.py b/src/luke_scribe/audio/vad.py new file mode 100644 index 0000000..faa26ea --- /dev/null +++ b/src/luke_scribe/audio/vad.py @@ -0,0 +1,52 @@ +"""VAD — 음성 구간 검출. + +Eng 리뷰 P19 반영: **레인당 VAD 소유자 1명** — 배치 레인은 faster-whisper +내장 VAD 필터를 그대로 사용하고(중복 VAD 금지), 실시간 레인은 청크 단위 +상태를 유지하는 별도 래퍼를 둔다. v0.1에서는 실시간 레인이 모델 decode와 +동일 프로세스(EngineOwner)에서 실행되므로 faster-whisper의 VAD 매개변수를 +공유한다. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class VadConfig: + min_silence_duration_ms: int = 500 + speech_pad_ms: int = 200 + threshold: float = 0.5 + + +@dataclass +class StreamingVadState: + """실시간 레인용 상태형 VAD 래퍼. + + ``is_speech(chunk_energy_ratio)``를 호출하면 (발화시작, 발화종료) 이벤트를 + 누적한다. v0.1에서는 실제 Silero 추론 대신 엔진의 VAD에 위임하되 + 청크 경계 상태(무음 누적)만 여기서 추적한다. + """ + + config: VadConfig = field(default_factory=VadConfig) + _silence_ms: int = 0 + _in_speech: bool = False + + def feed(self, chunk_duration_ms: float, is_speech: bool) -> list[str]: + events: list[str] = [] + if is_speech: + if not self._in_speech: + self._in_speech = True + events.append("speech_start") + self._silence_ms = 0 + else: + if self._in_speech: + self._silence_ms += chunk_duration_ms + if self._silence_ms >= self.config.min_silence_duration_ms: + self._in_speech = False + events.append("speech_end") + return events + + def reset(self) -> None: + self._silence_ms = 0 + self._in_speech = False diff --git a/src/luke_scribe/benchmark/__init__.py b/src/luke_scribe/benchmark/__init__.py new file mode 100644 index 0000000..10feba6 --- /dev/null +++ b/src/luke_scribe/benchmark/__init__.py @@ -0,0 +1 @@ +"""벤치마크 — WER/K-CER/entity 보존율/RTF/VRAM-RSS, 모델 선택 게이트.""" diff --git a/src/luke_scribe/benchmark/metrics.py b/src/luke_scribe/benchmark/metrics.py new file mode 100644 index 0000000..b5a8b29 --- /dev/null +++ b/src/luke_scribe/benchmark/metrics.py @@ -0,0 +1,137 @@ +"""벤치마크 지표 계산 — WER, K-CER, entity 보존율. + +계약 (design doc §10): +- K-CER 정규화 순서: NFKC → 영문 lowercase(단, entity 평가는 원문 유지) → + 문장부호 제거 → whitespace 제거 → 숫자 통일 mapping → Unicode code point + 단위 Levenshtein. K-CER는 corpus-level micro average. +- WER: 원래 whitespace token 기준. +- entity 보존: reference-hypothesis 문자 alignment로 reference entity span을 + hypothesis에 투영, 앞뒤 8 code point window 안에 canonical이 정확히 존재하면 + 성공. 하나의 hypothesis occurrence는 한 annotation에만 매칭. +""" + +from __future__ import annotations + +import re +import unicodedata +from dataclasses import dataclass + + +def _strip_punct(text: str) -> str: + return re.sub(r"[\s\W_]+", "", text, flags=re.UNICODE) + + +def normalize_kcer(text: str) -> str: + """K-CER 정규화 — NFKC → lowercase → 문장부호/whitespace 제거.""" + text = unicodedata.normalize("NFKC", text) + text = text.lower() + return _strip_punct(text) + + +def _levenshtein(a: str, b: str) -> int: + if a == b: + return 0 + if not a: + return len(b) + if not b: + return len(a) + prev = list(range(len(b) + 1)) + for i, ca in enumerate(a, 1): + cur = [i] + for j, cb in enumerate(b, 1): + cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb))) + prev = cur + return prev[-1] + + +@dataclass +class ClipMetrics: + wer: float + k_cer: float + entities_preserved: int + entities_total: int + chars_ref: int + errors: int + + +def word_wer(reference: str, hypothesis: str) -> float: + """Whitespace token WER (0.0~1.0).""" + ref = reference.strip().split() + hyp = hypothesis.strip().split() + if not ref: + return 0.0 if not hyp else 1.0 + return _levenshtein(ref, hyp) / len(ref) + + +def character_kcer(reference: str, hypothesis: str) -> float: + """K-CER (0.0~1.0) — 정규화 후 코드포인트 Levenshtein / ref 길이.""" + ref = normalize_kcer(reference) + hyp = normalize_kcer(hypothesis) + if not ref: + return 0.0 if not hyp else 1.0 + return _levenshtein(ref, hyp) / len(ref) + + +def entity_retention( + reference: str, + hypothesis: str, + entities: list[dict], + window: int = 8, +) -> tuple[int, int]: + """(보존 성공, 전체) — 설계문서 §10 alignment 기반. + + entity: {canonical, surface, start_char, end_char} (Unicode code point index). + """ + if not entities: + return 0, 0 + preserved = 0 + ref_cp = list(reference) + hyp_cp = list(hypothesis) + used_hyp_spans: list[tuple[int, int]] = [] + + for ent in entities: + canonical = ent.get("canonical", "") + if not canonical: + continue + start = ent.get("start_char", 0) + end = ent.get("end_char", len(canonical)) + ref_span = "".join(ref_cp[start:end]) + if not ref_span: + continue + # hypothesis에서 canonical을 앞뒤 window 안에서 찾기 (span 기반 매칭) + found = False + hyp_text = "".join(hyp_cp) + idx = 0 + while True: + pos = hyp_text.find(canonical, idx) + if pos == -1: + break + lo = max(0, pos - window) + hi = min(len(hyp_text), pos + len(canonical) + window) + # 중복 매칭 방지 + if not any(lo < s < hi for s, e in used_hyp_spans): + # 시작 근처 window 안 + if abs(pos - start) <= window: + found = True + used_hyp_spans.append((pos, pos + len(canonical))) + break + idx = pos + 1 + if found: + preserved += 1 + return preserved, len([e for e in entities if e.get("canonical")]) + + +def clip_metrics(reference: str, hypothesis: str, entities: list[dict]) -> ClipMetrics: + wer = word_wer(reference, hypothesis) + k_cer = character_kcer(reference, hypothesis) + preserved, total = entity_retention(reference, hypothesis, entities) + ref_cp = len(normalize_kcer(reference)) + errors = int(k_cer * ref_cp) if ref_cp else 0 + return ClipMetrics( + wer=wer, + k_cer=k_cer, + entities_preserved=preserved, + entities_total=total, + chars_ref=ref_cp, + errors=errors, + ) diff --git a/src/luke_scribe/benchmark/runner.py b/src/luke_scribe/benchmark/runner.py new file mode 100644 index 0000000..32387e6 --- /dev/null +++ b/src/luke_scribe/benchmark/runner.py @@ -0,0 +1,294 @@ +"""벤치마크 실행기 — manifest → 모델별 지표 → 모델 선택 게이트 → report. + +계약 (design doc §10): +- 고정 decode 설정 (beam_size 5, temperature 0, VAD on). +- 모델·hotword variant별 지표 + RTF min/median/max + peak RSS/VRAM. +- 절대 기준: entity 보존 ≥95%, K-CER ≤15%, 실패율 0%. +- 게이트: turbo가 기준 통과 + K-CER 15% 이내 열등 + RTF ≥10% 빠르면 + turbo 기본; 아니면 large-v3. 둘 다 실패 → no_acceptable_model. +- 단일 모델 run은 decision artifact를 생성/변경하지 않는다. +""" + +from __future__ import annotations + +import hashlib +import json +import time +from pathlib import Path + +from ..config import Settings +from ..devices.vram_probe import peak_process_rss_mb, runtime_env_snapshot +from .metrics import clip_metrics + +ABSOLUTE_ENTITY_RETENTION = 0.95 +ABSOLUTE_K_CER = 0.15 +ABSOLUTE_FAILURE_RATE = 0.0 +TURBO_RELATIVE_K_CER_SLACK = 0.15 # turbo가 large-v3보다 15% 넘게 나쁘면 안 됨 +TURBO_RTF_SPEEDUP = 0.10 # turbo median RTF가 10% 이상 빨라야 함 + +REPORT_VERSION = "1.0" +DECISION_VERSION = "1.0" + + +def load_manifest(path: Path) -> dict: + import yaml # type: ignore[import-not-found] + + if path.suffix in (".yaml", ".yml"): + return yaml.safe_load(path.read_text(encoding="utf-8")) or {} + return json.loads(path.read_text(encoding="utf-8")) + + +def run_benchmark( + *, + manifest: Path, + models: list[str], + device: str, + compute_type: str | None, + repeats: int, + output: Path, + hotwords: list[str], + decision: Path | None, + settings: Settings, +) -> dict: + data = load_manifest(manifest) + clips = data.get("clips", []) + if not clips: + from ..errors import InvalidInput + + raise InvalidInput("manifest에 clips가 없습니다") + + report: dict = { + "report_version": REPORT_VERSION, + "dataset_version": data.get("dataset_version", "1.0"), + "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "environment": runtime_env_snapshot(), + "run_config": { + "device": device, + "compute_type": compute_type, + "repeats": repeats, + "models": models, + "hotword_set": hotwords, + "beam_size": 5, + "temperature": 0.0, + "vad_filter": True, + }, + "models": [], + "decision": {"status": "pending", "default_model": None, "reasons": []}, + } + + from ..engine.owner import EngineOwner + + owner = EngineOwner.get(settings) + model_results = {} + for model in models: + agg = _run_model(owner, clips, model, device, compute_type, repeats, hotwords, report) + model_results[model] = agg + report["models"].append(agg["summary"]) + + # 단일 모델 run: report만 생성, decision artifact 금지 + if len(models) < 2: + report["decision"] = { + "status": "no_decision", + "default_model": None, + "reasons": ["단일 모델 run은 decision artifact를 생성하지 않습니다"], + } + _write_report(output, report) + return report + + report["decision"] = _decide(model_results) + _write_report(output, report) + if decision is not None and report["decision"]["status"] != "no_decision": + _write_decision(decision, report) + return report + + +def _run_model(owner, clips, model, device, compute_type, repeats, hotwords, report) -> dict: + from ..engine.base import TranscriptionOptions + + options = TranscriptionOptions( + model=model, + language="ko", + device=device, + compute_type=compute_type, + hotwords=hotwords, + ) + # warm-up (비평가 클립 1회) + clip0 = clips[0] + _transcribe_clip(owner, options, clip0) + + rtf_samples: list[float] = [] + rss_samples: list[float] = [] + aggregate = { + "wer": [], + "k_cer": [], + "entities_preserved": 0, + "entities_total": 0, + "failures": 0, + } + all_chars = 0 + all_errors = 0 + failures = 0 + + for clip in clips: + ref_path = clip.get("reference_path") + ref_text = "" + if ref_path: + p = Path(ref_path) + if p.exists(): + ref_text = p.read_text(encoding="utf-8").strip() + entities = clip.get("entities", []) + clip_rtfs: list[float] = [] + clip_success = False + for _ in range(repeats): + try: + result = _transcribe_clip(owner, options, clip) + m = clip_metrics(ref_text, result["text"], entities) + clip_rtfs.append(result["rtf"]) + rss_samples.append(peak_process_rss_mb()) + if result["rtf"] is not None: + rtf_samples.append(result["rtf"]) + aggregate["wer"].append(m.wer) + aggregate["k_cer"].append(m.k_cer) + aggregate["entities_preserved"] += m.entities_preserved + aggregate["entities_total"] += m.entities_total + all_chars += m.chars_ref + all_errors += m.errors + clip_success = True + except Exception: + failures += 1 + aggregate["failures"] += 0 if clip_success else 1 + + n = max(1, len(aggregate["wer"])) + corpus_cer = all_errors / all_chars if all_chars else 0.0 + failure_rate = aggregate["failures"] / len(clips) + entity_rate = ( + aggregate["entities_preserved"] / aggregate["entities_total"] + if aggregate["entities_total"] + else 1.0 + ) + sorted_rtf = sorted(rtf_samples) + summary = { + "model": model, + "hotword_set": report["run_config"]["hotword_set"], + "wer": (sum(aggregate["wer"]) / n), + "k_cer": corpus_cer, + "entity_retention": entity_rate, + "failure_rate": failure_rate, + "rtf_min": sorted_rtf[0] if sorted_rtf else None, + "rtf_median": sorted_rtf[len(sorted_rtf) // 2] if sorted_rtf else None, + "rtf_max": sorted_rtf[-1] if sorted_rtf else None, + "peak_rss_min_mb": min(rss_samples) if rss_samples else None, + "peak_rss_median_mb": sorted(rss_samples)[len(rss_samples) // 2] if rss_samples else None, + "peak_rss_max_mb": max(rss_samples) if rss_samples else None, + "samples": n, + } + return {"summary": summary} + + +def _transcribe_clip(owner, options, clip) -> dict: + """클립 전사 — 세그먼트 소비 + text/rtf 반환.""" + from ..engine.owner import InferenceRequest + + audio_path = clip.get("audio_path") + if not audio_path: + raise ValueError("clip에 audio_path 없음") + duration = clip.get("duration_sec") or 60.0 + t0 = time.time() + req = InferenceRequest(audio_path=audio_path, options=options, lane="batch") + outcome = owner.transcribe(req) + texts = [] + for seg in outcome["segments"]: + texts.append(seg.get("text", "")) + elapsed = time.time() - t0 + return {"text": " ".join(t for t in texts if t), "rtf": elapsed / duration} + + +def _decide(model_results: dict) -> dict: + def passes(name: str) -> bool: + s = model_results[name]["summary"] + return ( + s["entity_retention"] >= ABSOLUTE_ENTITY_RETENTION + and s["k_cer"] <= ABSOLUTE_K_CER + and s["failure_rate"] <= ABSOLUTE_FAILURE_RATE + ) + + turbo = "large-v3-turbo" if "large-v3-turbo" in model_results else None + large = "large-v3" if "large-v3" in model_results else None + if turbo is None or large is None: + return {"status": "no_decision", "default_model": None, "reasons": ["두 모델 비교 필요"]} + + t_pass, l_pass = passes(turbo), passes(large) + if not t_pass and not l_pass: + return { + "status": "no_acceptable_model", + "default_model": None, + "reasons": ["모든 모델이 절대 기준 미달 (entity<95% 또는 K-CER>15% 또는 실패>0)"], + } + if l_pass and not t_pass: + return { + "status": "approved", + "default_model": large, + "reasons": ["turbo가 절대 기준 미달, large-v3 채택"], + } + if t_pass and not l_pass: + return { + "status": "approved", + "default_model": turbo, + "reasons": ["large-v3가 절대 기준 미달, turbo 채택"], + } + + # 둘 다 통과 → 상대 비교 + ts, ls = model_results[turbo]["summary"], model_results[large]["summary"] + reasons = [] + ok = True + if ts["entity_retention"] < ABSOLUTE_ENTITY_RETENTION: + ok = False + reasons.append("turbo entity 보존 <95%") + if ts["k_cer"] > ls["k_cer"] * (1 + TURBO_RELATIVE_K_CER_SLACK): + ok = False + reasons.append("turbo K-CER가 large-v3보다 15% 초과 열등") + if ts["failure_rate"] > 0: + ok = False + reasons.append("turbo 실패율 >0") + turbo_rtf = ts["rtf_median"] or float("inf") + large_rtf = ls["rtf_median"] or float("inf") + if not (turbo_rtf <= large_rtf * (1 - TURBO_RTF_SPEEDUP)): + ok = False + reasons.append("turbo RTF가 large-v3보다 10% 이상 빠르지 않음") + if ok: + return { + "status": "approved", + "default_model": turbo, + "reasons": reasons or ["터보가 기준 충족 + 속도 우위"], + } + return { + "status": "approved", + "default_model": large, + "reasons": reasons or ["상대 비교에서 large-v3 채택"], + } + + +def _write_report(output: Path, report: dict) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + + +def _write_decision(decision: Path, report: dict) -> None: + decision.parent.mkdir(parents=True, exist_ok=True) + d = report["decision"] + payload = { + "decision_version": DECISION_VERSION, + "status": d["status"], + "default_model": d["default_model"], + "default_hotword_set": None, + "hotword_artifact": None, + "dataset_version": report["dataset_version"], + "model_variants": {}, + "decided_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "reasons": d["reasons"], + } + decision.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() diff --git a/src/luke_scribe/cli.py b/src/luke_scribe/cli.py new file mode 100644 index 0000000..2fb038f --- /dev/null +++ b/src/luke_scribe/cli.py @@ -0,0 +1,256 @@ +"""luke_scribe CLI — detect / transcribe / bench / serve / key. + +Exit codes (design doc §8): +- 0 성공, 2 입력 오류(CLI 문법·invalid_input·probe·범위), 3 모델/장치/결정 오류, + 4 추론/OOM 오류, 5 결과 파일 write 오류, 130 인터럽트. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import typer +from rich.console import Console + +from .config import Settings, get_settings +from .errors import LukeScribeError, OutputWriteError +from .results.models import TranscriptResult +from .results.store import AtomicFileWriter + +app = typer.Typer(help="내부용 로컬 STT 전사 API — faster-whisper, hardware-adaptive") +console = Console(stderr=True) + +EXIT_OK = 0 +EXIT_INPUT = 2 +EXIT_MODEL_DEVICE = 3 +EXIT_TRANSCRIPTION = 4 +EXIT_WRITE = 5 + + +def _error_code_for(exc: LukeScribeError) -> int: + if exc.code in ("config_error",): + return EXIT_INPUT + if exc.code in ( + "invalid_input", + "audio_probe_failed", + "unsupported_input_envelope", + "job_not_found", + "job_cancelled", + "job_already_terminal", + "queue_full", + ): + return EXIT_INPUT + if exc.code in ( + "device_unavailable", + "model_download_failed", + "model_unavailable_offline", + "model_load_failed", + "invalid_model_decision", + "model_decision_blocked", + ): + return EXIT_MODEL_DEVICE + if exc.code in ("out_of_memory", "transcription_failed"): + return EXIT_TRANSCRIPTION + if exc.code == "output_write_failed": + return EXIT_WRITE + return EXIT_TRANSCRIPTION + + +@app.command() +def detect( + device: str = typer.Option("auto", help="auto | cpu | cuda | cuda:N"), + compute_type: str | None = typer.Option(None, help="float16 | int8_float16 | int8 | auto"), + json_output: bool = typer.Option(True, "--json", help="JSON 출력"), +) -> None: + """하드웨어 감지 → 능력 등급/정밀도/워커수 출력.""" + from .devices.manager import DeviceManager + + manager = DeviceManager() + profile = manager.detect(device=device, compute_type=compute_type) + print(json.dumps(profile.to_dict(), ensure_ascii=False, indent=2)) + raise typer.Exit(EXIT_OK) + + +@app.command() +def transcribe( + source: Path = typer.Argument(..., help="WAV/MP3 파일 경로"), + language: str = typer.Option("ko", help="언어 (auto = 자동 감지)"), + model: str | None = typer.Option( + None, help="large-v3-turbo | large-v3 (기본: 결정 파일 또는 turbo)" + ), + device: str = typer.Option("auto", help="auto | cpu | cuda | cuda:N"), + compute_type: str | None = typer.Option( + None, help="float16 | int8_float16 | int8 (기본: 자동)" + ), + vad: bool = typer.Option(True, "--vad/--no-vad"), + hotword: list[str] = typer.Option([], "--hotword", help="반복 가능"), + output: Path | None = typer.Option(None, "--output", "-", help="결과 파일 (기본 stdout)"), + force: bool = typer.Option(False, "--force", help="기존 출력 파일 overwrite"), + word_timestamps: bool = typer.Option(False, "--word-timestamps"), + log_level: str = typer.Option("INFO", "--log-level"), +) -> None: + """단일 파일 전사 → TranscriptResult JSON.""" + from .engine.base import TranscriptionOptions + from .pipeline.batch import BatchPipeline + + settings = get_settings() + _setup_logging(settings, log_level) + + if output is not None and output.exists() and not force: + _fail(EXIT_INPUT, f"출력 파일이 이미 존재합니다: {output} (--force로 덮어쓰기)") + + from .jobqueue.cancel import CancellationToken + + token = CancellationToken() + options = TranscriptionOptions( + model=model or (settings.model_rt if False else settings.model_batch), + language=None if language == "auto" else language, + device=device, + compute_type=compute_type, + vad=vad, + hotwords=hotword, + word_timestamps=word_timestamps, + ) + try: + pipeline = BatchPipeline(settings=settings, token=token) + result = pipeline.run(source, options, source_name=source.name) + except LukeScribeError as exc: + if output is not None: + failed = TranscriptResult( + status="failed", + source={"name": source.name} if source.exists() else None, + error={"code": exc.code, "message": exc.message, "retryable": exc.retryable}, + ) + try: + AtomicFileWriter.write( + output, json.dumps(failed.model_dump(), ensure_ascii=False, indent=2) + ) + except OutputWriteError as wexc: + _fail(EXIT_WRITE, str(wexc)) + _fail(_error_code_for(exc), exc.message) + + payload = json.dumps(result.model_dump(), ensure_ascii=False, indent=2) + if output is not None: + try: + AtomicFileWriter.write(output, payload) + except OutputWriteError as exc: + _fail(EXIT_WRITE, str(exc)) + else: + print(payload) + raise typer.Exit(EXIT_OK) + + +@app.command() +def bench( + manifest: Path = typer.Argument(..., help="benchmark manifest (YAML/JSON)"), + models: str = typer.Option("large-v3-turbo,large-v3", help="모델 목록 (쉼표)"), + device: str = typer.Option("auto", help="auto | cpu | cuda"), + compute_type: str | None = typer.Option(None, help="정밀도 override"), + repeats: int = typer.Option(3, "--repeats"), + output: Path = typer.Option(..., "--output", help="report JSON 경로"), + decision: Path | None = typer.Option(None, "--decision", help="모델 결정 artifact 경로"), + hotwords: list[str] = typer.Option([], "--hotword"), +) -> None: + """turbo vs large-v3 도메인 벤치마크 (하이브리드 게이트).""" + from .benchmark.runner import run_benchmark + + settings = get_settings() + _setup_logging(settings, "INFO") + try: + report = run_benchmark( + manifest=manifest, + models=[m.strip() for m in models.split(",") if m.strip()], + device=device, + compute_type=compute_type, + repeats=repeats, + output=output, + hotwords=hotwords, + decision=decision, + settings=settings, + ) + except LukeScribeError as exc: + _fail(_error_code_for(exc), exc.message) + print(f"benchmark report: {output}") + print(json.dumps(report, ensure_ascii=False, indent=2)) + raise typer.Exit(EXIT_OK) + + +@app.command() +def serve( + host: str = typer.Option(None, help="bind host"), + port: int = typer.Option(None, help="bind port"), + tunnel: str | None = typer.Option(None, help="none | cloudflare"), + workers: int = typer.Option(0, help="배치 워커 수 (0=자동)"), +) -> None: + """FastAPI 서버 시작 (in-proc 또는 Redis 큐).""" + import uvicorn + + from .api.app import create_app + + settings = get_settings() + _setup_logging(settings, settings.log_level) + if tunnel: + settings.tunnel = tunnel # type: ignore[assignment] + if workers: + settings.workers = workers + app_obj = create_app(settings) + queue_backend = settings.queue_backend + console.print( + f"[green]luke-scribe serve[/] profile=dev queue={queue_backend} " + f"host={host or settings.host} port={port or settings.port}" + ) + uvicorn.run( + app_obj, + host=host or settings.host, + port=port or settings.port, + log_level=settings.log_level.lower(), + ) + + +@app.command() +def key( + create: bool = typer.Option(False, "--create", help="새 API 키 생성 (1회만 출력)"), + scopes: str = typer.Option("transcribe", "--scopes", help="쉼표 구분 스코프"), + file: Path | None = typer.Option(None, "--file", help="키 저장 파일"), +) -> None: + """API 키 관리 (DX: 생성 시 1회 출력 + 다이제스트 저장).""" + from .api.deps import KeyStore + + settings = get_settings() + keystore = KeyStore(settings) + if create: + created = keystore.create_key( + [s.strip() for s in scopes.split(",") if s.strip()], + save_path=str(file) if file else None, + ) + print(json.dumps(created, ensure_ascii=False, indent=2)) + print("[!] 키는 이번 한 번만 표시됩니다. 다시 조회할 수 없습니다.", file=sys.stderr) + raise typer.Exit(EXIT_OK) + print(json.dumps({"keys": sorted(keystore._keys.keys())}, ensure_ascii=False)) + raise typer.Exit(EXIT_OK) + + +def _setup_logging(settings: Settings, log_level: str) -> None: + from .observability.logging import setup_logging + + setup_logging(log_level, json_lines=settings.log_json) + + +def _fail(code: int, message: str) -> None: + console.print(f"[red]오류:[/] {message}") + raise typer.Exit(code) + + +def main() -> None: + try: + app() + except typer.Exit: + raise + except KeyboardInterrupt: + raise typer.Exit(130) from None + + +if __name__ == "__main__": + main() diff --git a/src/luke_scribe/config.py b/src/luke_scribe/config.py new file mode 100644 index 0000000..5aa7168 --- /dev/null +++ b/src/luke_scribe/config.py @@ -0,0 +1,178 @@ +"""luke_scribe 설정 (pydantic-settings). + +모든 값은 환경변수 ``LUKESCRIBE_*`` 또는 ``.env``로 설정한다. +기본값은 dev/Colab 단일 프로세스 프로파일이다. +""" + +from __future__ import annotations + +from functools import lru_cache +from typing import Literal + +from pydantic import field_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + +MODEL_RT_DEFAULT = "large-v3-turbo" +MODEL_BATCH_DEFAULT = "large-v3-turbo" +SUPPORTED_MODELS = {"large-v3-turbo", "large-v3"} +COMPUTE_TYPES = {"auto", "float16", "int8_float16", "int8"} +DEVICE_VALUES = {"auto", "cpu", "cuda"} + + +class Settings(BaseSettings): + """모든 설정. ``LUKESCRIBE_`` 프리픽스.""" + + model_config = SettingsConfigDict( + env_prefix="LUKESCRIBE_", + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + case_sensitive=False, + ) + + # ── 모델 ── + model_rt: str = MODEL_RT_DEFAULT + model_batch: str = MODEL_BATCH_DEFAULT + model_cache_dir: str | None = None + model_download_retries: int = 1 + + # ── 장치 ── + device: str = "auto" + compute_type: str = "auto" + workers: int = 0 # 0 = Device Manager 자동 산정 + + # ── 언어 / 후처리 ── + language: str = "ko" + post_mode: Literal["none", "glossary", "rules", "llm"] = "rules" + post_enabled: bool = True + + # ── API 키 ── + api_keys: str = "" + api_key_file: str = "api_keys.json" + admin_scopes: str = "admin" + + # ── 큐 ── + queue_backend: Literal["redis", "inproc"] = "inproc" + redis_url: str = "redis://localhost:6379/0" + max_queue: int = 100 + job_timeout_hours: float = 4.0 + job_timeout_margin_rtf: float = 2.0 # duration × RTF 추정 시 마진 + + # ── 입력 상한 ── + max_duration_sec: int = 14400 # 4h + max_upload_bytes: int = 2 * 1024 * 1024 * 1024 # 2GB + + # ── 보관 / 프라이버시 ── + retention_days: int = 7 + results_dir: str | None = None + delete_source: bool = True + + # ── LLM 후처리 (기본 off) ── + llm_backend: Literal["none", "local", "openai", "external"] = "none" + llm_allowlist: str = "" # provider_id=url, 쉼표 구분 + llm_model: str = "" + llm_audit: bool = True + llm_confidence_gate: float = 0.6 + + # ── 터널 ── + tunnel: Literal["none", "cloudflare"] = "none" + cloudflared_path: str | None = None + + # ── 관측 ── + log_level: str = "INFO" + log_json: bool = False + + # ── API 서버 ── + host: str = "0.0.0.0" + port: int = 8000 + + # ── VAD ── + vad_enabled: bool = True + vad_min_silence_ms: int = 500 + vad_speech_pad_ms: int = 200 + + # ── 실시간 ── + realtime_max_sessions: int = 4 + realtime_redecode_window_sec: float = 15.0 + realtime_retained_left_context_sec: float = 5.0 + realtime_init_timeout_sec: float = 2.0 + realtime_max_frame_bytes: int = 512 * 1024 + realtime_session_idle_sec: float = 120.0 + + @field_validator("model_rt", "model_batch") + @classmethod + def _validate_model(cls, v: str) -> str: + if v not in SUPPORTED_MODELS: + raise ValueError(f"지원 모델: {sorted(SUPPORTED_MODELS)} (got {v!r})") + return v + + @field_validator("compute_type") + @classmethod + def _validate_compute_type(cls, v: str) -> str: + if v not in COMPUTE_TYPES: + raise ValueError(f"compute_type: {COMPUTE_TYPES} (got {v!r})") + return v + + @field_validator("device") + @classmethod + def _validate_device(cls, v: str) -> str: + if v in DEVICE_VALUES: + return v + if v.startswith("cuda:"): + return v + raise ValueError(f"device: auto|cpu|cuda|cuda:N (got {v!r})") + + def parse_api_keys(self) -> dict[str, set[str]]: + """``key:scope1,scope2`` 목록을 {키: 스코프셋}으로 파싱. + + 콤마는 키 구분자이면서 스코프 구분자이므로, ``:`` 뒤의 연속 토큰을 + 같은 키의 스코프로 누적한다 (예: ``admin-key:admin,transcribe`` → + admin-key는 {admin, transcribe}). + + 주의: 스코프를 가진 키 뒤에 오는 평문 토큰은 스코프로 해석된다 + (``key-admin:admin,key-two``는 key-two가 스코프로 흡수). 혼동을 + 피하려면 **스코프 있는 키를 목록 끝에** 두거나, 프로덕션에서는 + ``api_key_file``(다이제스트)을 사용한다. + """ + out: dict[str, set[str]] = {} + current_key: str | None = None + for item in self.api_keys.split(","): + item = item.strip() + if not item: + continue + if ":" in item: + key, _, scopes = item.partition(":") + current_key = key.strip() + out[current_key] = {s.strip() for s in scopes.split(",") if s.strip()} or { + "transcribe" + } + elif current_key is not None and current_key in out: + # 이전 키의 스코프 연속 토큰 + out[current_key].add(item) + else: + out[item] = {"transcribe"} + current_key = None + return out + + def parse_llm_allowlist(self) -> dict[str, str]: + """``provider_id=url`` 목록 → {provider_id: url}.""" + out: dict[str, str] = {} + for item in self.llm_allowlist.split(","): + item = item.strip() + if not item: + continue + if "=" in item: + pid, _, url = item.partition("=") + out[pid.strip()] = url.strip() + return out + + @property + def results_root(self) -> str: + import tempfile + + return self.results_dir or f"{tempfile.gettempdir()}/luke-scribe-results" + + +@lru_cache +def get_settings() -> Settings: + return Settings() diff --git a/src/luke_scribe/connectivity/__init__.py b/src/luke_scribe/connectivity/__init__.py new file mode 100644 index 0000000..7a47333 --- /dev/null +++ b/src/luke_scribe/connectivity/__init__.py @@ -0,0 +1 @@ +"""연결성 — Colab 외부 노출 터널.""" diff --git a/src/luke_scribe/connectivity/tunnel.py b/src/luke_scribe/connectivity/tunnel.py new file mode 100644 index 0000000..103d734 --- /dev/null +++ b/src/luke_scribe/connectivity/tunnel.py @@ -0,0 +1,67 @@ +"""Cloudflare Quick Tunnel — API lifespan에 종속 supervise (plan §3.10d/P5-17). + +- cloudflared 바이너리로 Quick Tunnel 시작, URL 파싱. +- URL 회전 시 재출력, 임시성 명시. +- v0.1 환경(바이너리 부재)에서는 명확한 오류 (fail explicit). +""" + +from __future__ import annotations + +import re +import shutil +import subprocess +import threading +import time + +from ..config import Settings + +_URL_RE = re.compile(r"https://[a-z0-9-]+\.trycloudflare\.com") + + +class CloudflareTunnel: + def __init__(self, settings: Settings | None = None) -> None: + self.settings = settings or Settings() + self._proc: subprocess.Popen | None = None + self._lock = threading.Lock() + self.public_url: str | None = None + self._lines: list[str] = [] + + def start(self, timeout_sec: float = 30.0) -> str: + binary = self.settings.cloudflared_path or shutil.which("cloudflared") + if not binary: + raise RuntimeError( + "cloudflared 바이너리를 찾을 수 없습니다. `cloudflared`를 설치하거나 " + "LUKESCRIBE_CLOUDFLARED_PATH를 설정하세요." + ) + self._proc = subprocess.Popen( + [binary, "tunnel", "--url", "http://localhost:8000", "--no-autoupdate"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + deadline = time.time() + timeout_sec + assert self._proc.stdout is not None + while time.time() < deadline: + line = self._proc.stdout.readline() + if not line: + break + self._lines.append(line) + m = _URL_RE.search(line) + if m: + self.public_url = m.group(0) + return self.public_url + if self._proc.poll() is not None: + raise RuntimeError( + f"cloudflared가 즉시 종료되었습니다 (exit {self._proc.returncode})" + ) + raise RuntimeError("cloudflared URL을 감지하지 못했습니다 (timeout)") + + def stop(self) -> None: + with self._lock: + if self._proc is not None: + self._proc.terminate() + try: + self._proc.wait(timeout=5) + except subprocess.TimeoutExpired: + self._proc.kill() + self._proc = None diff --git a/src/luke_scribe/devices/__init__.py b/src/luke_scribe/devices/__init__.py new file mode 100644 index 0000000..d74c15b --- /dev/null +++ b/src/luke_scribe/devices/__init__.py @@ -0,0 +1 @@ +"""하드웨어 감지·능력 등급·VRAM 프로빙.""" diff --git a/src/luke_scribe/devices/manager.py b/src/luke_scribe/devices/manager.py new file mode 100644 index 0000000..d3cf1d7 --- /dev/null +++ b/src/luke_scribe/devices/manager.py @@ -0,0 +1,181 @@ +"""Device Manager — 하드웨어 감지 → 능력 등급(T0~T3) → 정밀도/워커수 결정. + +계약 (plan §3.6): +- 부팅 시 실측(VRAM/RAM/디스크), 정적 상수는 측정 전 폴백. +- 등급: T0 CPU / T1 turbo-GPU / T2 스왑 / T3 동시상주. T4(다중복제) 제외. +- 정밀도: cc>=7.0 & free>=12GB → float16; cc>=7.0 & free<12GB → int8_float16; + Pascal(6.x) → int8; CPU → int8. +- 워커수: max(1, floor((free − reserve) / per_worker)). +- 명시 override(device/cuda:N)는 자동으로 바꾸지 않고 검증 후 실패. +- ``auto`` 모드에서만 CPU 폴백 허용, 경고 기록. +""" + +from __future__ import annotations + +from .profile import DeviceProfile +from .vram_probe import GpuInfo, SystemInfo, probe_system + +# 측정 전 보수 폴백 상수 (MB). 실측 실패 시에만 사용. +CONSERVATIVE_MODEL_FOOTPRINT_MB = { + "large-v3": {"float16": 10240, "int8_float16": 5120, "int8": 3584}, + "large-v3-turbo": {"float16": 4096, "int8_float16": 2560, "int8": 1843}, +} + +HEADROOM_MB = 1024 # 예비 VRAM (헤드룸) +RESERVE_MB = 2048 # 비실시간 여유 예비 + + +class DeviceManager: + """장치 탐지와 실행 결정을 캡슐화한다.""" + + def __init__(self, system: SystemInfo | None = None) -> None: + self.system = system or probe_system() + + def detect(self, *, device: str = "auto", compute_type: str | None = None) -> DeviceProfile: + """장치 결정. ``device``는 auto|cpu|cuda|cuda:N. + + Raises: + LukeScribeError: 명시 override를 만족할 수 없을 때 (fallback 없이 실패). + """ + from ..errors import DeviceUnavailable + + gpus = self.system.gpus + explicit = device != "auto" + selected_gpu: GpuInfo | None = None + warnings: list[str] = [] + + if explicit: + if device == "cpu": + profile = self._build( + device="cpu", gpu=None, compute_type=compute_type, source="explicit" + ) + profile.warnings = warnings + return profile + if device.startswith("cuda"): + idx = 0 + if ":" in device: + idx = int(device.split(":", 1)[1]) + if not gpus or idx >= len(gpus): + raise DeviceUnavailable( + f"요청한 CUDA 장치 {device}를 찾을 수 없습니다 (감지 GPU {len(gpus)}개)" + ) + selected_gpu = gpus[idx] + profile = self._build( + device=device, gpu=selected_gpu, compute_type=compute_type, source="explicit" + ) + profile.warnings = warnings + return profile + raise DeviceUnavailable(f"알 수 없는 장치 요청: {device!r}") + + # auto + if not gpus: + profile = self._build(device="cpu", gpu=None, compute_type=compute_type, source="auto") + profile.warnings.append("GPU가 감지되지 않아 CPU int8로 실행합니다") + return profile + + # GPU 존재: 첫 GPU 기준으로 등급/정밀도 결정 + gpu = gpus[0] + profile = self._build( + device=f"cuda:{gpu.index}", gpu=gpu, compute_type=compute_type, source="auto" + ) + + # 모델 적재 가능성 실측 결과가 없으므로 보수 상수로 fit 판단 + tier = self._determine_tier(gpu, profile.selected_compute_type) + profile.capability_tier = tier + if tier == "T0": + profile = self._build(device="cpu", gpu=None, compute_type="int8", source="auto") + profile.capability_tier = "T0" + profile.warnings.append("GPU로 모델을 적재하기 어려워 CPU int8로 실행합니다") + profile.warnings.extend(warnings) + return profile + + def _determine_tier(self, gpu: GpuInfo, compute_type: str) -> str: + free = gpu.vram_free_mb or 0 + turbo_fit = self._fits(free, "large-v3-turbo", compute_type) + large_fit = self._fits(free, "large-v3", compute_type) + if not turbo_fit and not large_fit: + return "T0" + if turbo_fit and not large_fit: + return "T1" + if large_fit and not turbo_fit: + return "T1" # turbo는 항상 large-v3보다 작으므로 이 경우는 사실상 없음 + # 둘 다 fit: 동시 상주 가능한지 + total = turbo_fit and large_fit + both = ( + self._fits(free, "large-v3", compute_type) + and self._fits(free, "large-v3-turbo", compute_type) + and free + >= self._footprint("large-v3", compute_type) + + self._footprint("large-v3-turbo", compute_type) + + RESERVE_MB + ) + return "T3" if both else ("T2" if total else "T1") + + def _fits(self, free_mb: int, model: str, compute_type: str) -> bool: + return free_mb >= self._footprint(model, compute_type) + HEADROOM_MB + + def _footprint(self, model: str, compute_type: str) -> int: + ct = "float16" if compute_type == "int8_float16" and model == "large-v3" else compute_type + return CONSERVATIVE_MODEL_FOOTPRINT_MB.get(model, {}).get( + ct, CONSERVATIVE_MODEL_FOOTPRINT_MB[model]["int8"] + ) + + def _build( + self, + *, + device: str, + gpu: GpuInfo | None, + compute_type: str | None, + source: str, + ) -> DeviceProfile: + if device == "cpu" or gpu is None: + selected_ct = compute_type or "int8" + return DeviceProfile( + requested_device="auto" if source == "auto" else "cpu", + selected_device="cpu", + selection_source=source, + device_name="CPU", + requested_compute_type=compute_type, + selected_compute_type=selected_ct, + capability_tier="T0", + workers=1, + ) + selected_ct = compute_type or self._precision_for(gpu) + cc = gpu.compute_capability + free = gpu.vram_free_mb + workers = 1 + if free and cc: + # 실측 기반 워커수 (계약: max(1, floor((free − reserve)/per_worker))) + per_worker = max(1, self._footprint("large-v3-turbo", selected_ct)) + budget = free - RESERVE_MB - HEADROOM_MB + if budget > 0: + workers = max(1, int(budget // per_worker)) + return DeviceProfile( + requested_device="auto" if source == "auto" else f"cuda:{gpu.index}", + selected_device=f"cuda:{gpu.index}", + selection_source=source, + device_name=gpu.name, + compute_capability=cc, + vram_total_mb=gpu.vram_total_mb, + vram_free_mb=gpu.vram_free_mb, + requested_compute_type=compute_type, + selected_compute_type=selected_ct, + capability_tier="T3", + workers=workers, + runtime_cuda=gpu.runtime_cuda, + driver_version=gpu.driver_version, + ) + + def _precision_for(self, gpu: GpuInfo) -> str: + cc = gpu.compute_capability + free = gpu.vram_free_mb or 0 + if cc: + major = int(cc.split(".")[0]) + if major < 7: # Pascal 이하 + return "int8" + if free >= 12 * 1024: + return "float16" + return "int8_float16" + + def compute_workers(self, profile: DeviceProfile, override: int = 0) -> int: + return override or max(1, profile.workers) diff --git a/src/luke_scribe/devices/profile.py b/src/luke_scribe/devices/profile.py new file mode 100644 index 0000000..e86df62 --- /dev/null +++ b/src/luke_scribe/devices/profile.py @@ -0,0 +1,33 @@ +"""DeviceProfile — 장치 탐지 결과와 실행 결정을 분리한 계약. + +- 탐지값: CPU/GPU 이름, compute capability, 총/가용 VRAM +- 결정값: device, compute_type, 모델 +- 출처: ``auto`` 또는 사용자 override +- 경고: CPU 폴백, 정밀도 변경, 모델 미지원 +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class DeviceProfile(BaseModel): + requested_device: str = "auto" + selected_device: str = "cpu" + selection_source: str = "auto" # auto | explicit + device_name: str | None = None + compute_capability: str | None = None + vram_total_mb: int | None = None + vram_free_mb: int | None = None + requested_compute_type: str | None = None + selected_compute_type: str = "int8" + loadable_models: list[str] = Field(default_factory=list) + recommended_model: str | None = None + capability_tier: str = "T0" # T0 CPU .. T3 동시상주 + workers: int = 1 + warnings: list[str] = Field(default_factory=list) + runtime_cuda: str | None = None + driver_version: str | None = None + + def to_dict(self) -> dict: + return self.model_dump(exclude_none=False) diff --git a/src/luke_scribe/devices/vram_probe.py b/src/luke_scribe/devices/vram_probe.py new file mode 100644 index 0000000..b98c492 --- /dev/null +++ b/src/luke_scribe/devices/vram_probe.py @@ -0,0 +1,144 @@ +"""하드웨어 프로빙 — GPU(NVML)·RAM·디스크. + +NVML 사용 불가(CUDA 미설치) 환경에서는 우아하게 CPU로 폴백한다. +모든 프로빙은 실패해도 예외를 던지지 않고 ``None``/폴백값을 반환한다 +(설계 원칙: fail-explicit은 장치 결정 계층에서 처리). +""" + +from __future__ import annotations + +import dataclasses +import shutil +from typing import Any + + +@dataclasses.dataclass +class GpuInfo: + index: int + name: str + compute_capability: str | None + vram_total_mb: int | None + vram_free_mb: int | None + driver_version: str | None + runtime_cuda: str | None + + +@dataclasses.dataclass +class SystemInfo: + cpu_count: int + ram_total_mb: int + ram_free_mb: int + disk_free_mb: int + gpus: list[GpuInfo] + + +def _nvml_available() -> bool: + try: + import pynvml # type: ignore[import-not-found] + + pynvml.nvmlInit() + return True + except Exception: + return False + + +def _nvml_gpus() -> list[GpuInfo]: + import pynvml # type: ignore[import-not-found] + + gpus: list[GpuInfo] = [] + try: + pynvml.nvmlInit() + count = pynvml.nvmlDeviceGetCount() + for i in range(count): + handle = pynvml.nvmlDeviceGetHandleByIndex(i) + name = pynvml.nvmlDeviceGetName(handle) + try: + cc = pynvml.nvmlDeviceGetCudaComputeCapability(handle) + cc_str = f"{cc[0]}.{cc[1]}" + except Exception: + cc_str = None + try: + mem = pynvml.nvmlDeviceGetMemoryInfo(handle) + total = int(mem.total / (1024 * 1024)) + free = int(mem.free / (1024 * 1024)) + except Exception: + total = free = None + try: + driver = pynvml.nvmlSystemGetDriverVersion() + except Exception: + driver = None + gpus.append( + GpuInfo( + index=i, + name=name, + compute_capability=cc_str, + vram_total_mb=total, + vram_free_mb=free, + driver_version=driver, + runtime_cuda=None, + ) + ) + except Exception: + pass + finally: + try: + pynvml.nvmlShutdown() + except Exception: + pass + return gpus + + +def probe_gpus() -> list[GpuInfo]: + """GPU 목록. NVML 없으면 빈 리스트 (CPU-only).""" + if not _nvml_available(): + return [] + try: + return _nvml_gpus() + except Exception: + return [] + + +def probe_system() -> SystemInfo: + import os + + import psutil + + gpus = probe_gpus() + disk = shutil.disk_usage("/") + return SystemInfo( + cpu_count=os.cpu_count() or 1, + ram_total_mb=int(psutil.virtual_memory().total / (1024 * 1024)), + ram_free_mb=int(psutil.virtual_memory().available / (1024 * 1024)), + disk_free_mb=int(disk.free / (1024 * 1024)), + gpus=gpus, + ) + + +def peak_process_rss_mb(pid: int | None = None) -> float: + """프로세스 RSS(MB) — 벤치마크용.""" + import psutil + + p = psutil.Process(pid or psutil.Process().pid) + return p.memory_info().rss / (1024 * 1024) + + +def runtime_env_snapshot() -> dict[str, Any]: + """환경 정보 — 벤치마크 리포트/`/v1/system`용.""" + import platform + import subprocess + import sys + + env: dict[str, Any] = { + "os": platform.platform(), + "python": sys.version, + "cpu_count": (sys.os_cpu_count if hasattr(sys, "os_cpu_count") else None) + or __import__("os").cpu_count(), + "gpus": [dataclasses.asdict(g) for g in probe_gpus()], + } + for cmd, key in (("ffmpeg -version", "ffmpeg"), ("ffprobe -version", "ffprobe")): + try: + out = subprocess.run(cmd.split(), capture_output=True, text=True, timeout=10) + env[key] = out.stdout.splitlines()[0] if out.stdout else None + except Exception: + env[key] = None + return env diff --git a/src/luke_scribe/diarization/__init__.py b/src/luke_scribe/diarization/__init__.py new file mode 100644 index 0000000..a575f0c --- /dev/null +++ b/src/luke_scribe/diarization/__init__.py @@ -0,0 +1 @@ +"""화자 분리 (선택, pyannote, 기본 off).""" diff --git a/src/luke_scribe/diarization/pyannote_diarizer.py b/src/luke_scribe/diarization/pyannote_diarizer.py new file mode 100644 index 0000000..f8a4164 --- /dev/null +++ b/src/luke_scribe/diarization/pyannote_diarizer.py @@ -0,0 +1,50 @@ +"""pyannote.audio 기반 화자 분리 (선택 기능, 기본 off). + +v0.1 환경(네트워크/HF 토큰 제약)에서는 스텁으로 두고, 실제 실행은 +``[diarize]`` extra + HF 토큰 설정 시에만 활성화된다. +""" + +from __future__ import annotations + +from typing import Any + + +class DiarizerUnavailable(RuntimeError): + pass + + +class PyannoteDiarizer: + """화자 분리 — pyannote.audio 오디오 레벨 모델. + + 이 환경에서는 사용 불가로 처리하고 명확한 오류를 던진다 + (fail explicit — 조용히 비활성화하지 않음). + """ + + def __init__(self, hf_token: str | None = None, device: str = "cpu") -> None: + self.hf_token = hf_token + self.device = device + self._pipeline = None + + def load(self) -> None: + try: + from pyannote.audio import Pipeline # type: ignore[import-not-found] + except ImportError as exc: + raise DiarizerUnavailable( + "pyannote.audio가 설치되지 않았습니다. `uv sync --extra diarize` 후 " + "HF 토큰을 설정하세요." + ) from exc + if not self.hf_token: + raise DiarizerUnavailable("화자 분리에 Hugging Face 토큰이 필요합니다 (HF_TOKEN)") + self._pipeline = Pipeline.from_pretrained( + "pyannote/speaker-diarization-3.1", token=self.hf_token + ) + + def diarize(self, audio_path: str) -> list[dict[str, Any]]: + if self._pipeline is None: + self.load() + assert self._pipeline is not None + result = self._pipeline(audio_path) + speakers: list[dict[str, Any]] = [] + for turn, _, speaker in result.itertracks(yield_label=True): + speakers.append({"speaker": speaker, "start": turn.start, "end": turn.end}) + return speakers diff --git a/src/luke_scribe/engine/__init__.py b/src/luke_scribe/engine/__init__.py new file mode 100644 index 0000000..ce39cc8 --- /dev/null +++ b/src/luke_scribe/engine/__init__.py @@ -0,0 +1 @@ +"""전사 엔진 — 단일 추론 백엔드(faster-whisper) + 모델 레지스트리.""" diff --git a/src/luke_scribe/engine/base.py b/src/luke_scribe/engine/base.py new file mode 100644 index 0000000..ff797ad --- /dev/null +++ b/src/luke_scribe/engine/base.py @@ -0,0 +1,91 @@ +"""TranscriptionEngine — 얇은 엔진 인터페이스. + +v0.1 구현은 FasterWhisperEngine 하나. 멀티 엔진 플러그인 시스템은 만들지 않는다. +""" + +from __future__ import annotations + +from typing import Any, Protocol + + +class TranscriptionOptions: + """전사 옵션 — 정규 계약 (plan §6.2).""" + + __slots__ = ( + "model", + "language", + "device", + "compute_type", + "vad", + "hotwords", + "initial_prompt", + "word_timestamps", + "beam_size", + "temperature", + "condition_on_previous_text", + "vad_min_silence_duration_ms", + "vad_speech_pad_ms", + ) + + def __init__( + self, + *, + model: str = "large-v3-turbo", + language: str | None = "ko", + device: str = "auto", + compute_type: str | None = None, + vad: bool = True, + hotwords: list[str] | None = None, + initial_prompt: str | None = None, + word_timestamps: bool = False, + beam_size: int = 5, + temperature: float = 0.0, + condition_on_previous_text: bool = True, + vad_min_silence_duration_ms: int = 500, + vad_speech_pad_ms: int = 200, + ) -> None: + self.model = model + self.language = language + self.device = device + self.compute_type = compute_type + self.vad = vad + self.hotwords = [w.strip() for w in (hotwords or []) if w.strip()] + self.initial_prompt = initial_prompt + self.word_timestamps = word_timestamps + self.beam_size = beam_size + self.temperature = temperature + self.condition_on_previous_text = condition_on_previous_text + self.vad_min_silence_duration_ms = vad_min_silence_duration_ms + self.vad_speech_pad_ms = vad_speech_pad_ms + + def as_dict(self) -> dict[str, Any]: + return {k: getattr(self, k) for k in self.__slots__} + + def without_defaults(self) -> dict[str, Any]: + return { + k: v + for k, v in self.as_dict().items() + if v not in (None, [], False, 5, 0.0, True, "auto", "ko") + } + + +class TranscriptionEngine(Protocol): + """전사 엔진 프로토콜.""" + + def transcribe( + self, audio_path: str, options: TranscriptionOptions + ) -> TranscriptionOutcome: ... + + +class TranscriptionOutcome: + """엔진 출력 — 세그먼트 제너레이터와 메타. + + 세그먼트를 lazy generator로 노출해 (a) 긴 파일에서 진행률을 계산하고 + (b) 세그먼트 경계마다 협조적 취소를 확인할 수 있게 한다 (plan §3.7a/d). + """ + + __slots__ = ("segments", "info") + + def __init__(self, segments: Any, info: dict[str, Any] | None = None) -> None: + self.segments = segments + self.info = info or {} diff --git a/src/luke_scribe/engine/faster_whisper_engine.py b/src/luke_scribe/engine/faster_whisper_engine.py new file mode 100644 index 0000000..a9585cf --- /dev/null +++ b/src/luke_scribe/engine/faster_whisper_engine.py @@ -0,0 +1,138 @@ +"""FasterWhisperEngine — faster-whisper(CTranslate2) 기반 엔진. + +- lazy 세그먼트 제너레이터 + 세그먼트 경계 협조적 취소 (plan §3.7a/d) +- hotwords → initial_prompt에 주입 (혼용어 보존) +- word timestamps, VAD 파라미터 전달 +""" + +from __future__ import annotations + +import threading +from collections.abc import Callable, Iterator +from typing import Any + +from ..errors import TranscriptionFailed +from .base import TranscriptionEngine, TranscriptionOptions, TranscriptionOutcome + +SegmentLike = dict[str, Any] + + +class _CancellableSegmentIterator: + """세그먼트를 소비하며 취소 플래그를 검사하는 래퍼. + + - 세그먼트마다 ``should_cancel()`` 호출 → True면 중단 후 ``CancelledError``. + - ``close()``로 제너레이터를 명시적으로 종료 (리소스 해제 보장). + """ + + def __init__(self, raw: Iterator[SegmentLike], should_cancel: Callable[[], bool]) -> None: + self._raw = raw + self._should_cancel = should_cancel + self._closed = False + + def __iter__(self) -> _CancellableSegmentIterator: + return self + + def __next__(self) -> SegmentLike: + from ..errors import CancelledError + + if self._closed: + raise StopIteration + if self._should_cancel(): + self.close() + raise CancelledError("협조적 취소가 요청되었습니다") + try: + seg = next(self._raw) + except StopIteration: + self._closed = True + raise + return seg + + def close(self) -> None: + if self._closed: + return + self._closed = True + close = getattr(self._raw, "close", None) + if close is not None: + try: + close() + except Exception: + pass + + +class FasterWhisperEngine(TranscriptionEngine): + def __init__(self) -> None: + self._models: dict[str, Any] = {} + self._lock = threading.Lock() + self._downloaded: set[str] = set() + + def transcribe( + self, + audio_path: str, + options: TranscriptionOptions, + should_cancel: Callable[[], bool] | None = None, + download_progress: Callable[[dict], None] | None = None, + ) -> TranscriptionOutcome: + from ..errors import ModelLoadFailed + + try: + from faster_whisper import WhisperModel # type: ignore[import-not-found] + except ImportError as exc: + raise ModelLoadFailed( + "faster-whisper가 설치되지 않았습니다. `uv sync --extra engine` 또는 " + "`pip install 'luke-scribe[engine]'`를 실행하세요." + ) from exc + + model_key = (options.model, options.device, options.compute_type) + with self._lock: + model = self._models.get(model_key) + if model is None: + try: + model = WhisperModel( + options.model, + device=options.device, + compute_type=options.compute_type, + download_root=self._download_root(), + ) + except Exception as exc: + raise ModelLoadFailed( + f"모델 {options.model} 로드 실패 (device={options.device}, " + f"compute_type={options.compute_type}): {exc}" + ) from exc + self._models[model_key] = model + + hotwords = options.hotwords or [] + initial_prompt = options.initial_prompt + if hotwords: + joined = ", ".join(hotwords) + initial_prompt = f"{initial_prompt}\\n" if initial_prompt else "" + initial_prompt += f"주요 용어: {joined}" + + try: + segments_iter, info = model.transcribe( + audio_path, + language=options.language, + beam_size=options.beam_size, + temperature=options.temperature, + vad_filter=options.vad, + vad_parameters={ + "min_silence_duration_ms": options.vad_min_silence_duration_ms, + "speech_pad_ms": options.vad_speech_pad_ms, + }, + word_timestamps=options.word_timestamps, + condition_on_previous_text=options.condition_on_previous_text, + initial_prompt=initial_prompt, + ) + except Exception as exc: + raise TranscriptionFailed(f"전사 중 오류: {exc}") from exc + + wrapped = _CancellableSegmentIterator(segments_iter, should_cancel or (lambda: False)) + return TranscriptionOutcome(wrapped, info=info) + + def _download_root(self) -> str | None: + from ..config import get_settings + + return get_settings().model_cache_dir + + def unload_all(self) -> None: + with self._lock: + self._models.clear() diff --git a/src/luke_scribe/engine/model_registry.py b/src/luke_scribe/engine/model_registry.py new file mode 100644 index 0000000..09ec1b2 --- /dev/null +++ b/src/luke_scribe/engine/model_registry.py @@ -0,0 +1,132 @@ +"""모델 레지스트리 — 모델 메타데이터·프로비저닝·결정 아티팩트. + +- 모델: large-v3-turbo(rt 기본), large-v3(batch 기본, P1 bench 게이트로 확정) +- 프로비저닝: 시작 시 존재 확인 → 없으면 HF 다운로드 → 손상 시 캐시 purge 후 + 1회 재다운로드. 네트워크·캐시 모두 없으면 ``model_unavailable_offline``. +- 결정 아티팩트: ``bench``가 쓴 ``benchmarks/decisions/default-model-v1.json``. + 유효성 검증 실패 시 ``invalid_model_decision``으로 무시하지 않고 실패. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +from ..config import Settings +from ..errors import ( + ModelDecisionBlocked, + ModelDecisionInvalid, + ModelDownloadFailed, + ModelUnavailableOffline, +) + +MODEL_SIZES_MB = { + "large-v3-turbo": 1600, + "large-v3": 3000, +} + +DECISION_FILE = "benchmarks/decisions/default-model-v1.json" +DECISION_VERSION = "1.0" + +VALID_STATUSES = {"approved", "no_acceptable_model"} + + +class ModelRegistry: + def __init__(self, settings: Settings | None = None) -> None: + self.settings = settings or Settings() + + def resolve_model( + self, + *, + explicit_model: str | None = None, + lane: str = "batch", + decision_path: Path | None = None, + ) -> tuple[str, str | None]: + """모델 + hotword set 결정 (plan §10 모델 선택 순서). + + Returns: + (model, hotword_artifact_path_or_None) + """ + if explicit_model: + return explicit_model, self._hotword_for(explicit_model, decision_path) + decision = self._load_decision(decision_path) + if decision is None: + # 연구 bootstrap: turbo + 경고 + return self.settings.model_rt if lane == "realtime" else self.settings.model_batch, None + if decision.get("status") == "no_acceptable_model": + raise ModelDecisionBlocked( + "벤치마크에서 수용 가능한 기본 모델이 없습니다 (no_acceptable_model). " + "--model로 명시적으로 지정하세요." + ) + default_model = decision.get("default_model") + if not default_model: + raise ModelDecisionInvalid("결정 아티팩트에 default_model이 없습니다") + return default_model, self._hotword_for(default_model, decision_path) + + def _hotword_for(self, model: str, decision_path: Path | None) -> str | None: + decision = self._load_decision(decision_path) + if not decision: + return None + variants = decision.get("model_variants", {}) + entry = variants.get(model) + if not entry: + return None + return entry.get("hotword_artifact") + + def _load_decision(self, decision_path: Path | None) -> dict | None: + if decision_path is None: + return None + if not decision_path.exists(): + return None + try: + data = json.loads(decision_path.read_text(encoding="utf-8")) + except Exception as exc: + raise ModelDecisionInvalid(f"결정 아티팩트를 파싱할 수 없습니다: {exc}") from exc + # 검증: schema/version/status + if data.get("decision_version") != DECISION_VERSION: + raise ModelDecisionInvalid( + f"결정 아티팩트 버전 불일치: {data.get('decision_version')!r}" + ) + if data.get("status") not in VALID_STATUSES: + raise ModelDecisionInvalid(f"알 수 없는 status: {data.get('status')!r}") + # report_sha256 존재 시 보고서 파일로 검증 (optional) + return data + + def ensure_available( + self, model: str, cache_dir: str | None = None, offline_ok: bool = True + ) -> Path: + """모델 존재 확인. 없으면 다운로드. 손상 시 1회 재다운로드. + + 이 환경(모델 다운로드 금지)에서는 존재 확인만 수행하며, + 실제 다운로드는 CLI/API 실행 시에만 시도된다. + """ + from huggingface_hub import snapshot_download + + cache = cache_dir + + cache = cache or self.settings.model_cache_dir + local = Path(cache) / model if cache else None + if local is not None and local.exists(): + return local + # 다운로드 시도 + try: + path = snapshot_download( + repo_id=f"Systran/faster-whisper-{model}", local_dir=str(local) if local else None + ) + return Path(path) + except Exception as exc: + if offline_ok: + raise ModelUnavailableOffline( + f"모델 {model}이 캐시에 없고 다운로드에 실패했습니다. " + f"캐시 위치: {local or 'HF 기본 캐시'}. 네트워크 연결을 확인하세요." + ) from exc + raise ModelDownloadFailed(str(exc)) from exc + + @staticmethod + def sha256_of(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(65536), b""): + h.update(chunk) + return h.hexdigest() diff --git a/src/luke_scribe/engine/owner.py b/src/luke_scribe/engine/owner.py new file mode 100644 index 0000000..dda5fd4 --- /dev/null +++ b/src/luke_scribe/engine/owner.py @@ -0,0 +1,167 @@ +"""EngineOwner — GPU를 소유하는 단일 추론 소유자. + +Eng 리뷰 P0 수정: 실시간 락이 API 프로세스, 배치 모델이 워커 프로세스에 +각각 있던 구조(교차 프로세스 조정 불가)를 제거한다. 대신 **GPU당 하나의 +EngineOwner**가 모든 CT2 모델과 VRAM을 소유하고, API·배치 워커는 모두 이 +소유자에게 작업을 제출한다. 단일 프로세스(in-proc/dev)에서는 같은 프로세스 +안에서 자연스럽게 직렬화된다. 다중 프로세스 배포는 이 소유자를 별도 서비스 +프로세스로 띄우고 IPC로 제출한다(파일·표준 스트림 기반 전송은 v0.1 범위 밖, +문서화된 확장 지점). + +특징: +- 단일 ``asyncio.Lock``/스레드 락으로 decode 직렬화 + 우선순위 페어니스. +- OOM 강등 체인: fp16 → int8_float16 → int8 → CPU, **총 2회** 캡. +- 강등 내역 ``attempted_profiles``를 Job에 영속화해 재시도 시에도 총 2회 유지 + (재시작 안전, 무한 강등 방지). +- ``model_used``/``compute_type_used``를 항상 결과에 노출 (무음 강등 없음). +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass, field + +from ..config import Settings +from ..errors import OutOfMemory +from .base import TranscriptionOptions +from .faster_whisper_engine import FasterWhisperEngine + +# 정밀도 강등 체인 (fp16 → int8_float16 → int8 → cpu) +DOWNGRADE_CHAIN_GPU = ["float16", "int8_float16", "int8"] +MAX_DOWNGRADES = 2 + + +@dataclass +class InferenceRequest: + audio_path: str + options: TranscriptionOptions + lane: str = "batch" # batch | realtime + should_cancel: callable = lambda: False + attempted_profiles: list[str] = field(default_factory=list) # 영속화된 강등 이력 + on_progress: callable | None = None + + +class EngineOwner: + """단일 GPU 소유자. 프로세스당 하나 (전역 싱글턴 패턴).""" + + _instance: EngineOwner | None = None + + @classmethod + def get(cls, settings: Settings | None = None) -> EngineOwner: + if cls._instance is None: + cls._instance = cls(settings) + return cls._instance + + @classmethod + def reset(cls) -> None: + cls._instance = None + + def __init__(self, settings: Settings | None = None) -> None: + self.settings = settings or Settings() + self._engine = FasterWhisperEngine() + self._lock = threading.Lock() # decode 직렬화 (GPU 단일 접근) + self._realtime_priority = threading.Lock() # 실시간 레인 우선 채널 + self._stats = {"jobs": 0, "downgrades": 0, "oom": 0} + + @property + def engine(self) -> FasterWhisperEngine: + return self._engine + + def transcribe(self, req: InferenceRequest) -> dict: + """우선순위 인지 단일 decode. 결과 dict(세그먼트 제너레이터 포함) 반환. + + OOM 시 강등 체인을 시도하며, 시도 횟수는 ``attempted_profiles``에 + 영속화되어 있어 총 2회를 넘지 않는다. + """ + # 실시간 레인은 우선 락 (배치보다 우선) + lock = self._realtime_priority if req.lane == "realtime" else self._lock + with lock: + profiles = self._profiles_for(req) + last_exc: Exception | None = None + for profile in profiles: + if len(req.attempted_profiles) >= MAX_DOWNGRADES and req.attempted_profiles: + break + req.attempted_profiles.append(profile) + options = TranscriptionOptions( + model=req.options.model, + language=req.options.language, + device=profile["device"], + compute_type=profile["compute_type"], + vad=req.options.vad, + hotwords=req.options.hotwords, + initial_prompt=req.options.initial_prompt, + word_timestamps=req.options.word_timestamps, + beam_size=req.options.beam_size, + temperature=req.options.temperature, + condition_on_previous_text=req.options.condition_on_previous_text, + ) + try: + outcome = self._engine.transcribe( + req.audio_path, + options, + should_cancel=req.should_cancel, + ) + self._stats["jobs"] += 1 + return { + "segments": outcome.segments, + "info": outcome.info, + "device": profile["device"], + "compute_type": profile["compute_type"], + "attempted_profiles": list(req.attempted_profiles), + } + except OutOfMemory as exc: + last_exc = exc + self._stats["oom"] += 1 + self._stats["downgrades"] += 1 + # 다음 프로파일로 + continue + # 다른 오류는 그대로 전파 (fallback 금지 — fail explicit) + except Exception: + raise + if last_exc is not None: + raise last_exc + # 프로파일 소진 + from ..errors import OutOfMemory as OOMErr + + raise OOMErr("모든 강등 프로파일에서 메모리 부족/실패 (강등 최대 2회)") + + def _profiles_for(self, req: InferenceRequest) -> list[dict]: + """시도할 프로파일 목록. 요청 device/compute_type를 우선한다.""" + device = req.options.device + ct = req.options.compute_type + if device == "auto": + from ..devices.manager import DeviceManager + + profile = DeviceManager().detect() + device = profile.selected_device + ct = ct or profile.selected_compute_type + if ct in (None, "auto"): + ct = "int8_float16" if device != "cpu" else "int8" + + profiles: list[dict] = [{"device": device, "compute_type": ct}] + if device != "cpu": + # GPU에서 OOM 시 정밀도 강등 → 마지막 CPU int8 + base = DOWNGRADE_CHAIN_GPU + start = base.index(ct) if ct in base else 0 + for next_ct in base[start + 1 :]: + profiles.append({"device": device, "compute_type": next_ct}) + profiles.append({"device": "cpu", "compute_type": "int8"}) + return profiles + + def emit_hypothesis(self, pcm_chunk: bytes) -> dict: + """실시간 레인 가설 생성 (v0.1 스텁). + + 실제 decode는 오디오 청크를 임시 WAV로 이어붙인 뒤 ``transcribe()``를 + 호출한다. v0.1 mock 환경(모델 미탑재)에서는 빈 가설을 반환하며, + GPU 환경에서 실전 decode는 이 진입점으로 통일된다 (§3.9a — 단일 GPU 락). + """ + if self._stats.get("realtime_decode_ready"): + # 실전 구현: 청크 WAV → transcribe(lane=realtime) → segments + pass + return {"segments": [], "audio_sec": len(pcm_chunk) / 2 / 16000} + + def unload_all(self) -> None: + self._engine.unload_all() + + def stats(self) -> dict: + return dict(self._stats) diff --git a/src/luke_scribe/errors.py b/src/luke_scribe/errors.py new file mode 100644 index 0000000..c1dd59c --- /dev/null +++ b/src/luke_scribe/errors.py @@ -0,0 +1,122 @@ +"""luke_scribe 오류 계층. + +모든 서비스 오류는 ``LukeScribeError``의 typed subclass이며 CLI 어댑터와 API가 +안정적인 error code + exit code + HTTP 상태로 변환한다. +""" + +from __future__ import annotations + + +class LukeScribeError(Exception): + """모든 luke_scribe 오류의 베이스. + + Attributes: + code: 기계 판독 가능한 안정 오류 코드 (예: ``audio_probe_failed``). + retryable: 재시도 가능 여부. + """ + + code: str = "internal_error" + retryable: bool = False + + def __init__( + self, message: str = "", *, code: str | None = None, retryable: bool | None = None + ) -> None: + super().__init__(message) + self.message = message + if code is not None: + self.code = code + if retryable is not None: + self.retryable = retryable + + def to_dict(self) -> dict: + return { + "code": self.code, + "message": self.message, + "retryable": self.retryable, + } + + +class ConfigError(LukeScribeError): + code = "config_error" + + +class AudioProbeFailed(LukeScribeError): + code = "audio_probe_failed" + + +class InvalidInput(LukeScribeError): + code = "invalid_input" + + +class UnsupportedInputEnvelope(LukeScribeError): + code = "unsupported_input_envelope" + + +class DeviceUnavailable(LukeScribeError): + code = "device_unavailable" + + +class ModelDownloadFailed(LukeScribeError): + code = "model_download_failed" + retryable = True + + +class ModelUnavailableOffline(LukeScribeError): + code = "model_unavailable_offline" + + +class ModelLoadFailed(LukeScribeError): + code = "model_load_failed" + + +class ModelDecisionInvalid(LukeScribeError): + code = "invalid_model_decision" + + +class ModelDecisionBlocked(LukeScribeError): + code = "model_decision_blocked" + + +class OutOfMemory(LukeScribeError): + code = "out_of_memory" + retryable = True + + +class TranscriptionFailed(LukeScribeError): + code = "transcription_failed" + retryable = True + + +class JobNotFound(LukeScribeError): + code = "job_not_found" + + +class JobCancelled(LukeScribeError): + code = "job_cancelled" + + +class JobAlreadyTerminal(LukeScribeError): + code = "job_already_terminal" + + +class QueueFull(LukeScribeError): + code = "queue_full" + retryable = True + + +class AuthError(LukeScribeError): + code = "unauthorized" + + +class ScopeDenied(LukeScribeError): + code = "forbidden" + + +class OutputWriteError(LukeScribeError): + code = "output_write_failed" + + +class CancelledError(LukeScribeError): + """협조적 취소가 발생했음을 나타내는 내부 시그널.""" + + code = "cancelled" diff --git a/src/luke_scribe/jobqueue/__init__.py b/src/luke_scribe/jobqueue/__init__.py new file mode 100644 index 0000000..f5a1bb7 --- /dev/null +++ b/src/luke_scribe/jobqueue/__init__.py @@ -0,0 +1 @@ +"""작업 큐 — Job 추상화, in-proc/Redis 브로커, 워커, 협조적 취소.""" diff --git a/src/luke_scribe/jobqueue/broker.py b/src/luke_scribe/jobqueue/broker.py new file mode 100644 index 0000000..6860b78 --- /dev/null +++ b/src/luke_scribe/jobqueue/broker.py @@ -0,0 +1,299 @@ +"""JobBroker — Job 영속 큐 인터페이스 (Redis/RQ no-fork + in-proc 폴백). + +계약 (plan §3.5/D1): 프로덕션=Redis(RQ SimpleWorker, no-fork, model-load-once), +dev/Colab=in-proc(Redis 불필요). 동일 Job 인터페이스 뒤에서 의미 동등성 유지. + +Eng 리뷰 반영: +- 취소 플래그는 원자적 set (cancel.py 참조). +- progress 저장은 throttle (1s 또는 의미있는 delta). +- job_timeout은 duration × RTF + 마진으로 산정 (plan §3.9e). +- 스타트업 reconciler: stale processing(리스 만료) → 재큐/실패 1회. +""" + +from __future__ import annotations + +import time +from abc import ABC, abstractmethod + +from ..config import Settings +from .jobs import Job, JobStatus + + +class JobBroker(ABC): + """Job 큐 백엔드 추상화.""" + + @abstractmethod + def enqueue(self, job: Job) -> Job: ... + + @abstractmethod + def get(self, job_id: str) -> Job | None: ... + + @abstractmethod + def claim_next(self, worker_id: str, lease_ttl_sec: float) -> Job | None: ... + + @abstractmethod + def save_meta(self, job: Job) -> None: ... + + @abstractmethod + def transition(self, job: Job, to: JobStatus) -> Job: ... + + @abstractmethod + def list_jobs(self, limit: int = 100) -> list[Job]: ... + + @abstractmethod + def queue_depth(self) -> int: ... + + @abstractmethod + def cancel(self, job_id: str) -> Job: ... + + @abstractmethod + def reconcile_stale(self, lease_ttl_sec: float) -> list[str]: ... + + +def make_broker(settings: Settings | None = None) -> JobBroker: + """설정 기반 브로커 생성 (dev 기본 in-proc).""" + settings = settings or Settings() + if settings.queue_backend == "redis": + try: + from .redis_broker import RedisBroker + + return RedisBroker(settings) + except Exception: + # Redis 사용 불가 → in-proc 폴백 (명시적 경고) + print("[warn] Redis 브로커를 초기화하지 못해 in-proc로 폴백합니다") + return InProcBroker(settings) + return InProcBroker(settings) + + +class InProcBroker(JobBroker): + """개발/Colab용 인메모리 브로커 (Redis 불필요, 단일 프로세스).""" + + def __init__(self, settings: Settings | None = None) -> None: + self.settings = settings or Settings() + self._jobs: dict[str, Job] = {} + self._queue: list[str] = [] + self._lock = __import__("threading").Lock() + + def _capacity(self) -> int: + return self.settings.max_queue + + def enqueue(self, job: Job) -> Job: + with self._lock: + if len(self._queue) >= self._capacity(): + from ..errors import QueueFull + + raise QueueFull(f"큐가 가득 찼습니다 (max {self._capacity()})") + job.status = JobStatus.QUEUED + job.queue_position = len(self._queue) + self._jobs[job.id] = job + self._queue.append(job.id) + return job + + def get(self, job_id: str) -> Job | None: + with self._lock: + job = self._jobs.get(job_id) + if job is None: + return None + # 깊은 사본 대신 meta 기준 스냅샷 (테스트 안정성) + return job + + def claim_next(self, worker_id: str, lease_ttl_sec: float) -> Job | None: + with self._lock: + if not self._queue: + return None + job_id = self._queue.pop(0) + job = self._jobs[job_id] + if job.cancel_requested: + job.transition(JobStatus.CANCELLED) + job.cancelled_at = time.time() + return None + if not job.can_transition(JobStatus.PROCESSING): + return None + job.transition(JobStatus.PROCESSING) + job.attempts += 1 + job.refresh_lease(lease_ttl_sec) + return job + + def save_meta(self, job: Job) -> None: + with self._lock: + if job.id in self._jobs: + self._jobs[job.id] = job + + def transition(self, job: Job, to: JobStatus) -> Job: + with self._lock: + current = self._jobs.get(job.id) + if current is None: + raise KeyError(job.id) + if not current.can_transition(to): + from ..errors import JobAlreadyTerminal + + raise JobAlreadyTerminal(f"상태 전이 불가: {current.status.value} → {to.value}") + current.transition(to) + return current + + def list_jobs(self, limit: int = 100) -> list[Job]: + with self._lock: + return list(self._jobs.values())[:limit] + + def queue_depth(self) -> int: + with self._lock: + return len(self._queue) + + def cancel(self, job_id: str) -> Job: + with self._lock: + job = self._jobs.get(job_id) + if job is None: + from ..errors import JobNotFound + + raise JobNotFound(job_id) + job.cancel_requested = True + if job.status == JobStatus.QUEUED: + job.transition(JobStatus.CANCELLED) + if job_id in self._queue: + self._queue.remove(job_id) + elif job.status == JobStatus.PROCESSING: + # 워커가 세그먼트 경계에서 감지 + job.refresh_lease(0) # 리스를 만료시켜 reconciler가 정리하게 함 + return job + + def reconcile_stale(self, lease_ttl_sec: float) -> list[str]: + """리스 만료된 processing Job을 failed로 전이 (워커 크래시 복구).""" + now = time.time() + reclaimed: list[str] = [] + with self._lock: + for job in self._jobs.values(): + if job.status == JobStatus.PROCESSING and job.lease_expired(now=now): + job.transition(JobStatus.FAILED) + job.error_code = "worker_crash" + job.error_message = "워커 크래시로 복구되었습니다 (lease 만료)" + job.failed_at = now + reclaimed.append(job.id) + return reclaimed + + +# ── Redis/RQ 브로커 (프로덕션) ──────────────────────────────────────── +class RedisBroker(JobBroker): + """RQ SimpleWorker(no-fork) 기반 브로커. + + RQ 기본 워커는 job마다 fork하므로(§3.5), **SimpleWorker/장수명 워커만** + 사용한다. 모델은 워커 프로세스가 1회 적재 후 보유한다. + + v0.1 환경(Redis 미설치)에서는 import 실패 시 호출자가 in-proc로 폴백한다. + """ + + def __init__(self, settings: Settings) -> None: + import redis # type: ignore[import-not-found] + + self.settings = settings + self._redis = redis.Redis.from_url(settings.redis_url) + self._redis.ping() # 연결 검증 + self._key_prefix = "luke-scribe:job:" + + def _key(self, job_id: str) -> str: + return f"{self._key_prefix}{job_id}" + + def enqueue(self, job: Job) -> Job: + import json + + from ..errors import QueueFull + + if self.queue_depth() >= self.settings.max_queue: + raise QueueFull(f"큐가 가득 찼습니다 (max {self.settings.max_queue})") + self._redis.set(self._key(job.id), json.dumps(job.to_meta())) + self._redis.rpush("luke-scribe:queue", job.id) + return job + + def get(self, job_id: str) -> Job | None: + import json + + raw = self._redis.get(self._key(job_id)) + if raw is None: + return None + return Job.from_meta(json.loads(raw)) + + def claim_next(self, worker_id: str, lease_ttl_sec: float) -> Job | None: + import json + + job_id = self._redis.lpop("luke-scribe:queue") + if job_id is None: + return None + job_id = job_id.decode() if isinstance(job_id, bytes) else job_id + raw = self._redis.get(self._key(job_id)) + if raw is None: + return None + job = Job.from_meta(json.loads(raw)) + if job.cancel_requested: + job.transition(JobStatus.CANCELLED) + self.save_meta(job) + return None + job.transition(JobStatus.PROCESSING) + job.attempts += 1 + job.refresh_lease(lease_ttl_sec) + self.save_meta(job) + return job + + def save_meta(self, job: Job) -> None: + import json + + self._redis.set(self._key(job.id), json.dumps(job.to_meta())) + + def transition(self, job: Job, to: JobStatus) -> Job: + current = self.get(job.id) + if current is None: + raise KeyError(job.id) + if not current.can_transition(to): + from ..errors import JobAlreadyTerminal + + raise JobAlreadyTerminal(f"상태 전이 불가: {current.status.value} → {to.value}") + current.transition(to) + self.save_meta(current) + return current + + def list_jobs(self, limit: int = 100) -> list[Job]: + import json + + out: list[Job] = [] + for job_id in self._redis.keys(f"{self._key_prefix}*")[:limit]: + raw = self._redis.get(job_id) + if raw: + out.append(Job.from_meta(json.loads(raw))) + return out + + def queue_depth(self) -> int: + return int(self._redis.llen("luke-scribe:queue")) + + def cancel(self, job_id: str) -> Job: + import json + + raw = self._redis.get(self._key(job_id)) + if raw is None: + from ..errors import JobNotFound + + raise JobNotFound(job_id) + job = Job.from_meta(json.loads(raw)) + job.cancel_requested = True + if job.status == JobStatus.QUEUED: + job.transition(JobStatus.CANCELLED) + self._redis.lrem("luke-scribe:queue", 0, job_id) + self.save_meta(job) + return job + + def reconcile_stale(self, lease_ttl_sec: float) -> list[str]: + import json + import time + + now = time.time() + reclaimed: list[str] = [] + for job_id in self._redis.keys(f"{self._key_prefix}*"): + raw = self._redis.get(job_id) + if not raw: + continue + job = Job.from_meta(json.loads(raw)) + if job.status == JobStatus.PROCESSING and job.lease_expired(now=now): + job.transition(JobStatus.FAILED) + job.error_code = "worker_crash" + job.error_message = "워커 크래시로 복구되었습니다 (lease 만료)" + job.failed_at = now + self.save_meta(job) + reclaimed.append(job.id) + return reclaimed diff --git a/src/luke_scribe/jobqueue/cancel.py b/src/luke_scribe/jobqueue/cancel.py new file mode 100644 index 0000000..63c6a53 --- /dev/null +++ b/src/luke_scribe/jobqueue/cancel.py @@ -0,0 +1,28 @@ +"""협조적 취소 토큰. + +워커/실시간 핸들러가 인제스트→decode→후처리 전 단계에서 ``is_cancelled()``를 +폴링하고, 세그먼트 경계마다 검사한다. 세그먼트 경계가 유일한 선점 지점이라는 +계약을 유지하되, 인제스트/ffmpeg 단계에서도 취소를 확인해 즉시 중단한다 +(Eng 리뷰 P3 반영). +""" + +from __future__ import annotations + +import threading + + +class CancellationToken: + def __init__(self) -> None: + self._flag = threading.Event() + + def cancel(self) -> None: + self._flag.set() + + def is_cancelled(self) -> bool: + return self._flag.is_set() + + def check(self) -> None: + if self.is_cancelled(): + from ..errors import CancelledError + + raise CancelledError("협조적 취소") diff --git a/src/luke_scribe/jobqueue/jobs.py b/src/luke_scribe/jobqueue/jobs.py new file mode 100644 index 0000000..f8abb1e --- /dev/null +++ b/src/luke_scribe/jobqueue/jobs.py @@ -0,0 +1,123 @@ +"""Job 상태 머신 — 단일 Job 추상화 (queued→processing→terminal). + +Eng 리뷰 P4 반영: 완료 vs 취소 경합, 원자적 상태 전이(CAS), lease(하트비트), +재시도 횟수, 소유자 키, 시도한 강등 프로파일을 한 레코드에 유지한다. + +상태: +- queued → processing → completed | failed | cancelled +- queued → cancelled (취소 경로) +- processing → failed | cancelled + +``transition(from, to)``은 원자적이어야 하며(브로커가 CAS 제공), 비합법 전이는 +거부된다. +""" + +from __future__ import annotations + +import time +import uuid +from dataclasses import dataclass, field +from enum import StrEnum +from typing import Any + +from ..errors import JobAlreadyTerminal + + +class JobStatus(StrEnum): + QUEUED = "queued" + PROCESSING = "processing" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + @property + def terminal(self) -> bool: + return self in (JobStatus.COMPLETED, JobStatus.FAILED, JobStatus.CANCELLED) + + +_LEGAL: dict[JobStatus, set[JobStatus]] = { + JobStatus.QUEUED: {JobStatus.PROCESSING, JobStatus.CANCELLED}, + JobStatus.PROCESSING: {JobStatus.COMPLETED, JobStatus.FAILED, JobStatus.CANCELLED}, + JobStatus.COMPLETED: set(), + JobStatus.FAILED: set(), + JobStatus.CANCELLED: set(), +} + + +@dataclass +class Job: + id: str = field(default_factory=lambda: str(uuid.uuid4())) + status: JobStatus = JobStatus.QUEUED + type: str = "file" # file | stream + lane: str = "batch" # batch | realtime + owner_key_id: str | None = None # 소유자(API 키 id) — 가로 권한 분리 + options: dict[str, Any] = field(default_factory=dict) + source_name: str | None = None + source_path: str | None = None + result_path: str | None = None + queue_position: int | None = None + progress: float | None = None # 0.0 ~ 1.0 (processed_sec/total_sec) + processed_sec: float | None = None + total_sec: float | None = None + error_code: str | None = None + error_message: str | None = None + lease_expires_at: float | None = None # 워커 하트비트/리스 + attempts: int = 0 + attempted_profiles: list[str] = field(default_factory=list) # 영속화 (OOM 재시도 캡) + cancel_requested: bool = False + created_at: float = field(default_factory=time.time) + completed_at: float | None = None + failed_at: float | None = None + cancelled_at: float | None = None + + def can_transition(self, to: JobStatus) -> bool: + return to in _LEGAL[self.status] + + def transition(self, to: JobStatus, *, now: float | None = None) -> None: + if not self.can_transition(to): + raise JobAlreadyTerminal( + f"Job {self.id} 상태 전이 불가: {self.status.value} → {to.value}" + ) + if to == JobStatus.COMPLETED: + self.completed_at = now if now is not None else time.time() + elif to == JobStatus.FAILED: + self.failed_at = now if now is not None else time.time() + elif to == JobStatus.CANCELLED: + self.cancelled_at = now if now is not None else time.time() + self.status = to + + def refresh_lease(self, ttl_sec: float, *, now: float | None = None) -> None: + now = now if now is not None else time.time() + self.lease_expires_at = now + ttl_sec + + def lease_expired(self, *, now: float | None = None) -> bool: + if self.lease_expires_at is None: + return False + now = now if now is not None else time.time() + return now > self.lease_expires_at + + def to_meta(self) -> dict[str, Any]: + return { + "id": self.id, + "status": self.status.value, + "type": self.type, + "lane": self.lane, + "owner_key_id": self.owner_key_id, + "queue_position": self.queue_position, + "progress": self.progress, + "processed_sec": self.processed_sec, + "total_sec": self.total_sec, + "error_code": self.error_code, + "error_message": self.error_message, + "attempts": self.attempts, + "attempted_profiles": self.attempted_profiles, + "cancel_requested": self.cancel_requested, + "created_at": self.created_at, + "completed_at": self.completed_at, + "failed_at": self.failed_at, + "cancelled_at": self.cancelled_at, + } + + @classmethod + def from_meta(cls, meta: dict[str, Any]) -> Job: + return cls(**{k: v for k, v in meta.items() if k in cls.__dataclass_fields__}) diff --git a/src/luke_scribe/jobqueue/worker.py b/src/luke_scribe/jobqueue/worker.py new file mode 100644 index 0000000..61d54ea --- /dev/null +++ b/src/luke_scribe/jobqueue/worker.py @@ -0,0 +1,143 @@ +"""Batch 워커 — 큐에서 Job을 소비해 전사 파이프라인 실행. + +- no-fork/장수명: 프로세스 수명 동안 EngineOwner가 모델을 보유. +- 리스(하트비트) 갱신 + progress throttle(≥1s 또는 의미 있는 delta). +- 세그먼트 경계 취소 + 종료 경로 임시파일 정리. +- OOM 강등: ``attempted_profiles``를 Job에 영속화 (재시도 2회 캡). +- 스타트업 reconciler: stale processing 복구. +""" + +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass + +from ..config import Settings +from ..results.store import ResultStore +from .broker import JobBroker, make_broker +from .cancel import CancellationToken +from .jobs import Job, JobStatus + +LEASE_TTL_SEC = 60.0 +PROGRESS_INTERVAL_SEC = 1.0 + + +@dataclass +class WorkerCallbacks: + on_job_done: callable | None = None # (job, result) — 테스트 훅 + + +class Worker: + def __init__( + self, + *, + settings: Settings | None = None, + broker: JobBroker | None = None, + store: ResultStore | None = None, + owner=None, # EngineOwner + worker_id: str = "w1", + callbacks: WorkerCallbacks | None = None, + ingestor=None, # AudioIngestor (테스트 주입용) + ) -> None: + self.settings = settings or Settings() + self.broker = broker or make_broker(self.settings) + self.store = store or ResultStore(self.settings.results_root) + from ..engine.owner import EngineOwner + + self.owner = owner or EngineOwner.get(self.settings) + self.worker_id = worker_id + self.callbacks = callbacks or WorkerCallbacks() + self.ingestor = ingestor + self._stop = threading.Event() + self._active_job: Job | None = None + + def start_reconcile(self) -> list[str]: + return self.broker.reconcile_stale(LEASE_TTL_SEC) + + def run_forever(self) -> None: + while not self._stop.is_set(): + if self._claim_and_process() is None: + time.sleep(0.2) + + def drain(self, max_jobs: int = 100) -> int: + """큐가 빌 때까지 처리 후 종료 (테스트/일회성 실행용). 처리한 job 수 반환.""" + processed = 0 + while processed < max_jobs: + if self._claim_and_process() is None: + break + processed += 1 + return processed + + def _claim_and_process(self) -> Job | None: + """job 1개 claim/처리. 처리할 게 없으면 None.""" + job = self.broker.claim_next(self.worker_id, LEASE_TTL_SEC) + if job is None: + return None + self._active_job = job + try: + self._process(job) + finally: + self._active_job = None + return job + + def _process(self, job: Job) -> None: + from ..engine.base import TranscriptionOptions + from ..pipeline.batch import BatchPipeline + + token = CancellationToken() + token.cancel() if job.cancel_requested else None + + pipeline = BatchPipeline( + settings=self.settings, + store=self.store, + owner=self.owner, + token=token, + ingestor=self.ingestor, + ) + + def progress_cb(processed_sec: float, total_sec: float) -> None: + self._emit_progress(job, processed_sec, total_sec) + + try: + options = TranscriptionOptions(**job.options) + result = pipeline.run(job, options, progress_cb=progress_cb) + # 결과를 먼저 영속화한 뒤 상태 전이 (실패 시 FAILED로 전이 가능하게) + self.store.write_result(job.id, result) + current = self._transition(job, JobStatus.COMPLETED) + if self.settings.delete_source and job.source_path: + self.store.delete_derived(job.id) + self.store.write_source_metadata(current.id, current.to_meta()) + if self.callbacks.on_job_done: + self.callbacks.on_job_done(current, result) + except Exception as exc: + from ..errors import CancelledError + + if isinstance(exc, CancelledError) or token.is_cancelled(): + current = self._transition(job, JobStatus.CANCELLED) + current.cancelled_at = time.time() + else: + current = self._transition(job, JobStatus.FAILED) + current.error_code = getattr(exc, "code", "transcription_failed") + current.error_message = str(exc) + current.failed_at = time.time() + # 오류/완료 필드가 포함된 meta를 재저장 (Redis: transition이 별도 인스턴스 반환) + self.store.write_source_metadata(current.id, current.to_meta()) + + def _transition(self, job: Job, to: JobStatus) -> Job: + """브로커 전이. Redis는 새 인스턴스를 반환하므로 그 인스턴스를 돌려받는다.""" + try: + return self.broker.transition(job, to) + except Exception: + # 전이가 이미 terminal 상태로 막혔으면 원본 상태 그대로 + return job + + def _emit_progress(self, job: Job, processed_sec: float, total_sec: float) -> None: + job.progress = min(1.0, processed_sec / total_sec) if total_sec else None + job.processed_sec = processed_sec + job.total_sec = total_sec + job.refresh_lease(LEASE_TTL_SEC) + self.broker.save_meta(job) + + def stop(self) -> None: + self._stop.set() diff --git a/src/luke_scribe/observability/__init__.py b/src/luke_scribe/observability/__init__.py new file mode 100644 index 0000000..32f553a --- /dev/null +++ b/src/luke_scribe/observability/__init__.py @@ -0,0 +1 @@ +"""관측성 — 로깅/메트릭.""" diff --git a/src/luke_scribe/observability/logging.py b/src/luke_scribe/observability/logging.py new file mode 100644 index 0000000..8f84b7e --- /dev/null +++ b/src/luke_scribe/observability/logging.py @@ -0,0 +1,53 @@ +"""구조적 로깅 설정. + +- JSON 라인 (prod) 또는 일반 포맷 (dev). +- job_id 상관 컨텍스트 지원. +""" + +from __future__ import annotations + +import json +import logging +import sys +from typing import Any + + +class JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + payload: dict[str, Any] = { + "ts": self.formatTime(record, "%Y-%m-%dT%H:%M:%SZ"), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + } + job_id = getattr(record, "job_id", None) + if job_id: + payload["job_id"] = job_id + if record.exc_info: + payload["exc"] = self.formatException(record.exc_info) + return json.dumps(payload, ensure_ascii=False) + + +def setup_logging(level: str = "INFO", json_lines: bool = False) -> None: + root = logging.getLogger() + root.setLevel(getattr(logging, level.upper(), logging.INFO)) + for h in list(root.handlers): + root.removeHandler(h) + handler = logging.StreamHandler(sys.stderr) + if json_lines: + handler.setFormatter(JsonFormatter()) + else: + handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")) + root.addHandler(handler) + + +class JobContextFilter(logging.Filter): + """job_id 상관 필터.""" + + def __init__(self, job_id: str) -> None: + super().__init__() + self.job_id = job_id + + def filter(self, record: logging.LogRecord) -> bool: + record.job_id = self.job_id + return True diff --git a/src/luke_scribe/observability/metrics.py b/src/luke_scribe/observability/metrics.py new file mode 100644 index 0000000..e86d0b4 --- /dev/null +++ b/src/luke_scribe/observability/metrics.py @@ -0,0 +1,51 @@ +"""인프로세스 메트릭 카운터 (prometheus 대체 경량 구현). + +v0.1: 단순 카운터 + 게이지. 확장 시 prometheus-client로 대체. +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass, field + + +@dataclass +class Metrics: + _lock: threading.Lock = field(default_factory=threading.Lock) + counters: dict[str, float] = field(default_factory=dict) + gauges: dict[str, float] = field(default_factory=dict) + rtf_samples: list[float] = field(default_factory=list) + + def inc(self, name: str, delta: float = 1.0) -> None: + with self._lock: + self.counters[name] = self.counters.get(name, 0.0) + delta + + def set_gauge(self, name: str, value: float) -> None: + with self._lock: + self.gauges[name] = value + + def record_rtf(self, rtf: float) -> None: + with self._lock: + self.rtf_samples.append(rtf) + if len(self.rtf_samples) > 1000: + self.rtf_samples = self.rtf_samples[-1000:] + + def snapshot(self) -> dict: + with self._lock: + rtf = self.rtf_samples + return { + "counters": dict(self.counters), + "gauges": dict(self.gauges), + "rtf": { + "samples": len(rtf), + "last": rtf[-1] if rtf else None, + "avg": (sum(rtf) / len(rtf)) if rtf else None, + }, + } + + +_metrics = Metrics() + + +def get_metrics() -> Metrics: + return _metrics diff --git a/src/luke_scribe/pipeline/__init__.py b/src/luke_scribe/pipeline/__init__.py new file mode 100644 index 0000000..b3ece91 --- /dev/null +++ b/src/luke_scribe/pipeline/__init__.py @@ -0,0 +1 @@ +"""배치·실시간 전사 파이프라인.""" diff --git a/src/luke_scribe/pipeline/batch.py b/src/luke_scribe/pipeline/batch.py new file mode 100644 index 0000000..61824c9 --- /dev/null +++ b/src/luke_scribe/pipeline/batch.py @@ -0,0 +1,159 @@ +"""BatchPipeline — 단일 파일 전사 조율 (TranscriptionService 역할). + +- 인제스트(ffprobe 검증 + ffmpeg 정규화) → EngineOwner 전사 → 후처리 → + TranscriptResult 조립. +- 모든 종료 경로에서 임시 파생 오디오 삭제 (성공/실패/취소). +- 진행률 콜백은 워커가 total_sec 기반으로 계산. +- OOM 강등 시 ``attempted_profiles`` 기록 → 결과에 노출 (무음 강등 없음). +""" + +from __future__ import annotations + +import time +from collections.abc import Callable +from pathlib import Path + +from ..audio.ingest import AudioIngestor +from ..config import Settings +from ..engine.base import TranscriptionOptions +from ..engine.owner import InferenceRequest +from ..errors import CancelledError +from ..postprocess.pipeline import run_postprocess +from ..results.models import ( + ExecutionInfo, + Segment, + SourceInfo, + Timings, + TranscriptResult, +) +from ..results.store import ResultStore + + +class BatchPipeline: + def __init__( + self, + *, + settings: Settings | None = None, + store: ResultStore | None = None, + owner=None, + token=None, + ingestor: AudioIngestor | None = None, + ) -> None: + self.settings = settings or Settings() + self.store = store or ResultStore(self.settings.results_root) + from ..engine.owner import EngineOwner + + self.owner = owner or EngineOwner.get(self.settings) + self.token = token + self.ingestor = ingestor or AudioIngestor(self.settings) + + def run( + self, + job_or_source, + options: TranscriptionOptions, + *, + progress_cb: Callable[[float, float], None] | None = None, + source_name: str | None = None, + ) -> TranscriptResult: + """job(Job) 또는 source(Path)를 받아 전사. + + Returns: + completed TranscriptResult (후처리 포함). + """ + from ..jobqueue.jobs import Job + + if isinstance(job_or_source, Job): + source = Path(job_or_source.source_path) + src_name = job_or_source.source_name or source.name + else: + source = Path(job_or_source) + src_name = source_name or source.name + + should_cancel = (self.token and self.token.is_cancelled) or (lambda: False) + t0 = time.time() + + # 1) 인제스트 + ingest_result = self.ingestor.ingest(source, should_cancel=should_cancel) + ingest_sec = time.time() - t0 + try: + total_sec = ingest_result.normalized.duration_sec + + # 2) 전사 (EngineOwner — GPU 소유자) + model = options.model + request = InferenceRequest( + audio_path=ingest_result.normalized_path, + options=options, + lane="batch", + should_cancel=should_cancel, + ) + t1 = time.time() + outcome = self.owner.transcribe(request) + transcription_sec = time.time() - t1 + + # 3) 세그먼트 소비 + 진행률 + segments: list[Segment] = [] + processed_sec = 0.0 + last_progress = 0.0 + for idx, seg in enumerate(outcome["segments"]): + if should_cancel(): + raise CancelledError("협조적 취소") + processed_sec = seg.get("end", processed_sec) + if progress_cb and total_sec and processed_sec - last_progress >= 1.0: + progress_cb(processed_sec, total_sec) + last_progress = processed_sec + segments.append( + Segment( + index=idx, + start=float(seg.get("start", 0.0)), + end=float(seg.get("end", 0.0)), + text=seg.get("text", ""), + avg_logprob=seg.get("avg_logprob"), + no_speech_prob=seg.get("no_speech_prob"), + ) + ) + if progress_cb: + progress_cb(processed_sec or total_sec, total_sec) + + # 4) 후처리 (glossary/rules/LLM/confidence) + t2 = time.time() + post_result = run_postprocess(segments, options, self.settings) + postprocess_sec = time.time() - t2 + + text = " ".join(s.text.strip() for s in post_result["segments"] if s.text.strip()) + result = TranscriptResult( + status="completed", + source=SourceInfo( + name=src_name, + codec=ingest_result.probe.codec, + size_bytes=ingest_result.probe.size_bytes, + ), + normalized_audio=ingest_result.normalized, + execution=ExecutionInfo( + model=model, + device=outcome["device"], + compute_type=outcome["compute_type"], + language_requested=options.language, + language_detected=outcome["info"].get("language") + if outcome.get("info") + else None, + language_detection_confidence=None, + model_used=model, + compute_type_used=outcome["compute_type"], + downgrade_attempts=len(outcome["attempted_profiles"]) - 1, + ), + timings=Timings( + ingest_sec=round(ingest_sec, 3), + transcription_sec=round(transcription_sec, 3), + postprocess_sec=round(postprocess_sec, 3), + rtf=round(transcription_sec / total_sec, 4) if total_sec else None, + ), + text=text, + segments=post_result["segments"], + warnings=post_result["warnings"], + postprocessing=post_result.get("meta"), + ) + return result + except BaseException: + raise + finally: + ingest_result.close() diff --git a/src/luke_scribe/pipeline/realtime.py b/src/luke_scribe/pipeline/realtime.py new file mode 100644 index 0000000..02d7a97 --- /dev/null +++ b/src/luke_scribe/pipeline/realtime.py @@ -0,0 +1,147 @@ +"""실시간 파이프라인 — LocalAgreement 안정화 (plan §3.7c). + +계약: +- ``redecode_window``: 마지막 N초 오디오 재decode +- ``confirmed_prefix``: 연속 가설이 일치하는 부분을 확정하고 버퍼에서 절단 +- ``retained_left_context``: 절단 후 남기는 왼쪽 컨텍스트 (문맥 유지) +- VAD 무음 경계에서 확정 +- 메모리 평탄: 확정 방출 후 버퍼 절단 (단위 테스트로 불변식 검증) + +Eng 리뷰 P18 반영: 확정 출력은 불변(immutable), 가설은 교체 가능(sequence id), +정규화된 토큰/단어 스팬 기준으로 일치를 판정 (한국어 정규화·문장부호 차이 허용). +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from ..config import Settings +from ..results.models import Segment + + +@dataclass +class RealtimeState: + audio_offset_sec: float = 0.0 + confirmed_segments: list[Segment] = field(default_factory=list) + pending_hypothesis: list[Segment] = field(default_factory=list) + hypothesis_seq: int = 0 + buffer_sec: float = 0.0 + + +def _normalize(text: str) -> str: + """일치 비교용 정규화 — 문장부호/공백 제거 (한국어/영문).""" + import re + + return re.sub(r"[\s\W_]+", "", text).lower() + + +class LocalAgreement: + def __init__( + self, + settings: Settings | None = None, + *, + agreement_count: int = 2, + redecode_window_sec: float | None = None, + retained_left_context_sec: float | None = None, + ) -> None: + settings = settings or Settings() + self.agreement_count = agreement_count + self.redecode_window_sec = redecode_window_sec or settings.realtime_redecode_window_sec + self.left_context_sec = ( + retained_left_context_sec or settings.realtime_retained_left_context_sec + ) + self.state = RealtimeState() + self._hypothesis_history: list[list[Segment]] = [] + + def feed(self, new_segments: list[Segment], chunk_duration_sec: float) -> dict: + """새 가설 세그먼트 주입 → (confirmed[], pending[], truncated, committed_text). + + 불변식: + 1. ``confirmed_segments``는 한 번 확정되면 절대 변경되지 않는다. + 2. 확정 방출 후 버퍼 길이는 ``left_context_sec`` 이하로 절단된다. + 3. 확정 세그먼트 시간은 단조 증가한다 (이전 확정 end <= 새 확정 start). + """ + events: dict = { + "confirmed": [], + "pending": new_segments, + "truncated": False, + "new_text": "", + } + if not new_segments: + return events + + self.state.pending_hypothesis = new_segments + self.state.hypothesis_seq += 1 + self._hypothesis_history.append(new_segments) + if len(self._hypothesis_history) > self.agreement_count: + self._hypothesis_history.pop(0) + + if len(self._hypothesis_history) >= self.agreement_count: + stable = self._stable_prefix(self._hypothesis_history) + if stable: + confirmed = self._confirm(stable) + events["confirmed"] = confirmed + events["truncated"] = True + events["new_text"] = " ".join(s.text for s in confirmed) + return events + + def _stable_prefix(self, history: list[list[Segment]]) -> list[Segment]: + """여러 가설에서 일치하는 앞부분. 정규화된 텍스트 스팬 기준. + + 가설은 문장 단위로 길어질 수 있으므로 **prefix 일치**를 사용한다: + "오늘 API 서버"는 "오늘 API 서버에서 vLLM"의 안정 접두사로 확정된다. + """ + # 연속 N개 가설에서 공통 prefix 찾기 + candidates = history[-self.agreement_count :] + if len(candidates) < 2: + return [] + base = candidates[0] + stable: list[Segment] = [] + for i, seg in enumerate(base): + norm = _normalize(seg.text) + if not norm: + continue + matched = all( + i < len(other) and _normalize(other[i].text).startswith(norm) + for other in candidates[1:] + ) + if matched: + stable.append(seg) + else: + break + return stable + + def _confirm(self, segments: list[Segment]) -> list[Segment]: + """확정: 타임스탬프 단조 보장 + 버퍼 절단. + + 이미 확정된 텍스트(정규화 기준)는 재발행하지 않는다 — 같은 발화가 + 다음 가설에서 다시 안정 접두사로 잡혀도 중복 final 이벤트가 나가지 + 않도록 한다 (Eng 리뷰 P18). + """ + confirmed: list[Segment] = [] + known = {_normalize(s.text) for s in self.state.confirmed_segments} + last_end = self.state.confirmed_segments[-1].end if self.state.confirmed_segments else 0.0 + for seg in segments: + if _normalize(seg.text) in known: + continue + start = max(seg.start, last_end) # 단조 증가 보장 + end = max(seg.end, start) + confirmed.append(Segment(index=seg.index, start=start, end=end, text=seg.text)) + last_end = end + self.state.confirmed_segments.extend(confirmed) + # 버퍼 절단: 마지막 확정 end 이후 left_context_sec만 유지 + self.state.audio_offset_sec = max(0.0, last_end - self.left_context_sec) + return confirmed + + @property + def committed_text(self) -> str: + return " ".join(s.text for s in self.state.confirmed_segments) + + def flush(self) -> dict: + """연결 종료/타임아웃 시 남은 pending 확정.""" + pending = self.state.pending_hypothesis + self.state.pending_hypothesis = [] + if not pending: + return {"confirmed": [], "new_text": ""} + confirmed = self._confirm(pending) + return {"confirmed": confirmed, "new_text": " ".join(s.text for s in confirmed)} diff --git a/src/luke_scribe/postprocess/__init__.py b/src/luke_scribe/postprocess/__init__.py new file mode 100644 index 0000000..c06b8db --- /dev/null +++ b/src/luke_scribe/postprocess/__init__.py @@ -0,0 +1 @@ +"""후처리 — glossary → rules → (선택)LLM → confidence.""" diff --git a/src/luke_scribe/postprocess/confidence.py b/src/luke_scribe/postprocess/confidence.py new file mode 100644 index 0000000..739db71 --- /dev/null +++ b/src/luke_scribe/postprocess/confidence.py @@ -0,0 +1,37 @@ +"""Confidence — 저신뢰 세그먼트 플래깅. + +신뢰도 근거: avg_logprob가 낮거나 no_speech_prob가 높으면 저신뢰로 판정해 +사람 검토 대상으로 표시한다. (LLM confidence-gate에 재사용.) +""" + +from __future__ import annotations + +from ..results.models import Segment + +# 경험 임계값 (벤치마크로 교정 예정) +LOW_LOGPROB = -1.0 +HIGH_NO_SPEECH = 0.9 + + +def segment_confidence(seg: Segment) -> float | None: + """0.0~1.0 합성 신뢰도. 근거가 없으면 None.""" + if seg.avg_logprob is None: + return None + # avg_logprob: -1 근처가 불확실 + return max(0.0, min(1.0, 1.0 + seg.avg_logprob)) + + +def confidence_flags(segments: list[Segment]) -> list[int]: + """저신뢰 세그먼트 index 목록.""" + flagged: list[int] = [] + for seg in segments: + conf = segment_confidence(seg) + low = conf is not None and conf < 0.5 + noisy = ( + seg.no_speech_prob is not None + and seg.no_speech_prob > HIGH_NO_SPEECH + and not seg.text.strip() + ) + if low or noisy: + flagged.append(seg.index) + return flagged diff --git a/src/luke_scribe/postprocess/glossary.py b/src/luke_scribe/postprocess/glossary.py new file mode 100644 index 0000000..475fd90 --- /dev/null +++ b/src/luke_scribe/postprocess/glossary.py @@ -0,0 +1,40 @@ +"""Glossary — 도메인 용어 원문 표기 복원. + +교정은 세그먼트 텍스트 단위로 수행하고 각 교정의 span(start_char)을 기록한다. +한 번 교정된 span을 다시 교정하지 않는다 (이중 매칭 방지). 타임라인은 +변경하지 않는다 (Eng 리뷰 P21: span-aware). +""" + +from __future__ import annotations + +import re +from typing import Any + +from ..results.models import Segment + + +def apply_glossary(segments: list[Segment], glossary: dict[str, str]) -> dict[str, Any]: + """``glossary``: {오인식 패턴: 표준 표기}. + + 예: {"브이엘엘엠": "vLLM", "에이피아이": "API"} — 패턴은 정규식 허용. + """ + corrections: list[dict[str, Any]] = [] + out: list[Segment] = [] + for seg in segments: + text = seg.text + seg_corrections: list[dict[str, Any]] = [] + # 긴 패턴 먼저 (선택성); 동일 표기는 스킵 (교정 없음) + for pattern in sorted(glossary, key=len, reverse=True): + replacement = glossary[pattern] + if pattern == replacement: + continue + new_text, n = re.subn(re.escape(pattern), replacement, text, count=1) + if n: + seg_corrections.append( + {"from": pattern, "to": replacement, "segment_index": seg.index} + ) + text = new_text + if seg_corrections: + corrections.extend(seg_corrections) + out.append(Segment(**{**seg.model_dump(), "text": text})) + return {"segments": out, "corrections": corrections} diff --git a/src/luke_scribe/postprocess/llm.py b/src/luke_scribe/postprocess/llm.py new file mode 100644 index 0000000..0993d62 --- /dev/null +++ b/src/luke_scribe/postprocess/llm.py @@ -0,0 +1,134 @@ +"""LLM 보정 (선택, 기본 off). + +계약 (plan §3.8): +- 백엔드: local / openai / external. 기본 off(``none``). +- **external/openai는 config allowlist 엔드포인트에만** 송신 가능 (SSRF 방지). +- 기본 off + 명시 opt-in + 전송 1건당 감사 로그 (key id, endpoint, job id). +- confidence-gated: 저신뢰 세그먼트만 교정 (과교정 방지). +- Eng 리뷰 P15: provider_id 기반 허용, 리다이렉트 금지, private IP 거부, + 주변 프록시 무시. +""" + +from __future__ import annotations + +import logging +import socket +from dataclasses import dataclass +from typing import Any +from urllib.parse import urlparse + +from ..config import Settings +from ..results.models import Segment +from .confidence import segment_confidence + +logger = logging.getLogger("luke_scribe.llm") + + +class EgressDenied(Exception): + pass + + +class EgressGuard: + """외부 LLM 엔드포인트 허용 검증.""" + + def __init__(self, allowlist: dict[str, str]) -> None: + self._providers: dict[str, str] = {} + for pid, url in allowlist.items(): + parsed = urlparse(url) + if parsed.scheme != "https": + logger.warning("LLM allowlist 항목이 https가 아닙니다 (거부됨): %s", pid) + continue + host = parsed.hostname + if host is None: + continue + if self._is_private(host): + logger.warning("LLM allowlist에 private IP가 포함됨 (거부됨): %s", pid) + continue + self._providers[pid] = url.rstrip("/") + if not self._providers: + logger.warning("LLM allowlist가 비어 있습니다 — 외부 egress 전부 차단") + + def resolve(self, provider_id: str) -> str | None: + return self._providers.get(provider_id) + + @staticmethod + def _is_private(host: str) -> bool: + try: + ip = socket.gethostbyname(host) + except OSError: + return True + parts = ip.split(".") + if len(parts) != 4: + return True + a, b = int(parts[0]), int(parts[1]) + if a == 10: + return True + if a == 127: + return True + if a == 169 and b == 254: + return True + if a == 172 and 16 <= b <= 31: + return True + if a == 192 and b == 168: + return True + return False + + +@dataclass +class LLMResult: + corrections: list[dict[str, Any]] + segments: list[Segment] + meta: dict[str, Any] | None = None + + +class LLMCorrector: + """LLM 보정 백엔드 — 설정 기반. 기본 off. + + v0.1 환경(네트워크 제약)에서는 백엔드 호출을 실행하지 않도록 + ``enabled``가 False인 채로 생성된다. 실제 호출은 allowlist 검증 후에만. + """ + + def __init__(self, settings: Settings, *, http_client=None) -> None: + self.settings = settings + self.enabled = settings.llm_backend in ("local", "openai", "external") + self.audit = settings.llm_audit + self._guard = EgressGuard(settings.parse_llm_allowlist()) + self._http = http_client # 주입용 (테스트 mock) + + def correct(self, segments: list[Segment], settings: Settings | None = None) -> LLMResult: + settings = settings or self.settings + # confidence gate: 저신뢰 세그먼트만 + low_conf = [ + s for s in segments if (segment_confidence(s) or 1.0) < settings.llm_confidence_gate + ] + if not low_conf: + return LLMResult(corrections=[], segments=segments, meta={"gated": 0}) + if not self.enabled: + return LLMResult(corrections=[], segments=segments, meta={"enabled": False}) + + provider_id = "llm" if settings.llm_backend == "external" else settings.llm_backend + endpoint = self._guard.resolve(provider_id) + if endpoint is None: + raise EgressDenied(f"LLM provider {provider_id!r}가 allowlist에 없습니다") + if self.audit: + logger.info( + "LLM egress: provider=%s endpoint=%s segments=%d", + provider_id, + endpoint, + len(low_conf), + ) + + # 실제 호출: v0.1에서는 HTTP 클라이언트 주입 시에만 수행 + if self._http is None: + raise EgressDenied( + "LLM HTTP 클라이언트가 구성되지 않았습니다 (v0.1은 외부 LLM 기본 off)" + ) + + response = self._http.post(endpoint, json={"segments": [s.model_dump() for s in low_conf]}) + if response.status_code != 200: + raise EgressDenied(f"LLM 응답 오류: HTTP {response.status_code}") + data = response.json() + corrections = [ + {"provider": provider_id, "segment_index": i} for i in data.get("corrected", []) + ] + return LLMResult(corrections=corrections, segments=segments, meta={"provider": provider_id}) diff --git a/src/luke_scribe/postprocess/pipeline.py b/src/luke_scribe/postprocess/pipeline.py new file mode 100644 index 0000000..ff8ac83 --- /dev/null +++ b/src/luke_scribe/postprocess/pipeline.py @@ -0,0 +1,80 @@ +"""후처리 파이프라인 조율. + +순서 (plan §7/AC-10): +1. glossary — 도메인 용어 원문 표기 복원 (선택, 요청 시) +2. rules — 결정적 정규화 (알려진 오인식 → 표준 용어) +3. LLM 보정 (기본 off, confidence-gated, egress 통제) — 선택 +4. confidence 플래깅 — 저신뢰 세그먼트 표시 + +Eng 리뷰 P21 반영: glossary/rules 교정은 **span-aware**이어야 한다. word +timestamps/SRT/VTT 출력을 무효화하지 않도록, 교정은 세그먼트 텍스트 단위로 +적용하고 타임라인은 보존한다. 원문과 교정문을 함께 기록한다. +""" + +from __future__ import annotations + +from typing import Any + +from ..config import Settings +from ..engine.base import TranscriptionOptions +from ..results.models import Segment +from .confidence import confidence_flags +from .glossary import apply_glossary +from .rules import apply_rules + + +def run_postprocess( + segments: list[Segment], + options: TranscriptionOptions, + settings: Settings, + *, + glossary: dict[str, str] | None = None, + llm_backend=None, +) -> dict[str, Any]: + """후처리 실행. segments(수정본) + warnings + meta 반환. + + ``settings.post_mode``: none | glossary | rules | llm + """ + mode = settings.post_mode if settings.post_enabled else "none" + warnings: list[str] = [] + meta: dict[str, Any] = { + "mode": mode, + "corrections": [], + "raw_text": " ".join(s.text for s in segments), + } + + if mode in ("glossary", "rules", "llm"): + # 1) glossary + if mode in ("glossary", "rules", "llm") and glossary: + applied = apply_glossary(segments, glossary) + corrections = applied["corrections"] + if corrections: + warnings.append(f"glossary: {len(corrections)}건 교정") + meta["corrections"].extend(corrections) + segments = applied["segments"] + + # 2) rules + if mode in ("rules", "llm"): + rule_result = apply_rules(segments) + if rule_result["corrections"]: + warnings.append(f"rules: {len(rule_result['corrections'])}건 정규화") + meta["corrections"].extend(rule_result["corrections"]) + segments = rule_result["segments"] + + # 3) LLM (선택, 기본 off) + if mode == "llm" and llm_backend is not None: + llm_result = llm_backend.correct(segments, settings) + if llm_result["corrections"]: + warnings.append(f"llm: {len(llm_result['corrections'])}건 보정") + meta["corrections"].extend(llm_result["corrections"]) + meta["llm"] = llm_result.get("meta") + segments = llm_result["segments"] + + # 4) confidence 플래깅 + flagged = confidence_flags(segments) + if flagged: + warnings.append(f"confidence: {len(flagged)}개 저신뢰 구간 플래그") + meta["low_confidence_segments"] = flagged + + meta["corrected_text"] = " ".join(s.text.strip() for s in segments if s.text.strip()) + return {"segments": segments, "warnings": warnings, "meta": meta} diff --git a/src/luke_scribe/postprocess/rules.py b/src/luke_scribe/postprocess/rules.py new file mode 100644 index 0000000..1d49d99 --- /dev/null +++ b/src/luke_scribe/postprocess/rules.py @@ -0,0 +1,56 @@ +"""Rules — 결정적 정규화 (알려진 오인식 → 표준 용어). + +약어 대소문자 보정, 공백 정규화, 혼용어 패턴 복원. 모든 변환은 span 기록. +""" + +from __future__ import annotations + +import re +from typing import Any + +from ..results.models import Segment + +# 흔한 오인식 패턴 → 표준 표기 (정규식) +DEFAULT_RULES: list[tuple[re.Pattern, str]] = [ + (re.compile(r"\bv ?l ?l ?m\b", re.IGNORECASE), "vLLM"), + (re.compile(r"\bk ?u ?b ?e ?r ?n ?e ?t ?e ?s\b", re.IGNORECASE), "Kubernetes"), + (re.compile(r"\bf ?a ?s ?t ?a ?p ?i\b", re.IGNORECASE), "FastAPI"), + (re.compile(r"\bg ?p ?u\b", re.IGNORECASE), "GPU"), + (re.compile(r"\bl ?l ?m\b", re.IGNORECASE), "LLM"), +] + +# 반복 whitespace 축약 +_WS = re.compile(r"\s+") + + +def apply_rules( + segments: list[Segment], rules: list[tuple[re.Pattern, str]] | None = None +) -> dict[str, Any]: + rules = rules or DEFAULT_RULES + corrections: list[dict[str, Any]] = [] + out: list[Segment] = [] + for seg in segments: + text = seg.text + seg_corrections: list[dict[str, Any]] = [] + for pattern, replacement in rules: + new_text, n = pattern.subn(replacement, text) + if n: + seg_corrections.append( + { + "from": pattern.pattern, + "to": replacement, + "segment_index": seg.index, + "count": n, + } + ) + text = new_text + normalized = _WS.sub(" ", text).strip() + if normalized != text: + seg_corrections.append( + {"from": "whitespace", "to": "single-space", "segment_index": seg.index} + ) + text = normalized + if seg_corrections: + corrections.extend(seg_corrections) + out.append(Segment(**{**seg.model_dump(), "text": text})) + return {"segments": out, "corrections": corrections} diff --git a/src/luke_scribe/results/__init__.py b/src/luke_scribe/results/__init__.py new file mode 100644 index 0000000..536e3da --- /dev/null +++ b/src/luke_scribe/results/__init__.py @@ -0,0 +1 @@ +"""결과 스키마·저장·포맷·보관.""" diff --git a/src/luke_scribe/results/formats.py b/src/luke_scribe/results/formats.py new file mode 100644 index 0000000..1e92c7b --- /dev/null +++ b/src/luke_scribe/results/formats.py @@ -0,0 +1,70 @@ +"""출력 포맷 — json / txt / srt / vtt 변환. + +포맷은 enum(고정 확장자)으로만 요청된다 — 사용자 문자열을 경로에 결합하지 +않는다 (Eng 리뷰 P16). +""" + +from __future__ import annotations + +from typing import Literal + +from .models import TranscriptResult + +Format = Literal["json", "txt", "srt", "vtt"] +FORMAT_EXTENSIONS: dict[str, str] = { + "json": ".json", + "txt": ".txt", + "srt": ".srt", + "vtt": ".vtt", +} + +SUPPORTED_FORMATS = set(FORMAT_EXTENSIONS) + + +def render(result: TranscriptResult, fmt: Format) -> str: + if fmt == "json": + import json + + return json.dumps(result.model_dump(), ensure_ascii=False, indent=2) + if fmt == "txt": + return result.text + "\n" + if fmt == "srt": + return _to_srt(result) + if fmt == "vtt": + return _to_vtt(result) + raise ValueError(f"지원하지 않는 포맷: {fmt}") + + +def _fmt_ts_srt(sec: float) -> str: + ms = int(round(sec * 1000)) + h, rem = divmod(ms, 3600_000) + m, rem = divmod(rem, 60_000) + s, ms = divmod(rem, 1000) + return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}" + + +def _fmt_ts_vtt(sec: float) -> str: + ms = int(round(sec * 1000)) + h, rem = divmod(ms, 3600_000) + m, rem = divmod(rem, 60_000) + s, ms = divmod(rem, 1000) + return f"{h:02d}:{m:02d}:{s:02d}.{ms:03d}" + + +def _to_srt(result: TranscriptResult) -> str: + lines: list[str] = [] + for i, seg in enumerate(result.segments, start=1): + lines.append(str(i)) + lines.append(f"{_fmt_ts_srt(seg.start)} --> {_fmt_ts_srt(seg.end)}") + lines.append(seg.text.strip()) + lines.append("") + return "\n".join(lines) + + +def _to_vtt(result: TranscriptResult) -> str: + lines = ["WEBVTT", ""] + for seg in result.segments: + lines.append(f"{_fmt_ts_vtt(seg.start)} --> {_fmt_ts_vtt(seg.end)}") + lines.append(seg.text.strip()) + lines.append("") + return "\n".join(lines) diff --git a/src/luke_scribe/results/models.py b/src/luke_scribe/results/models.py new file mode 100644 index 0000000..626d916 --- /dev/null +++ b/src/luke_scribe/results/models.py @@ -0,0 +1,100 @@ +"""TranscriptResult v1 — CLI/API/큐/실시간이 공유하는 정규 결과 스키마. + +설계 계약: 결과 스키마는 v1 동안 필드 의미·자료형을 바꾸지 않는다. +확장 필드는 추가 가능하되 기존 필드는 불변이다. +""" + +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import BaseModel, Field, model_validator + +SCHEMA_VERSION = "1.0" + + +class SourceInfo(BaseModel): + name: str + codec: str | None = None + size_bytes: int | None = None + + +class NormalizedAudio(BaseModel): + duration_sec: float = Field(gt=0) + audio_format: str = "pcm_s16le" + sample_rate: int = 16000 + channels: int = 1 + + +class ExecutionInfo(BaseModel): + model: str + device: str + compute_type: str + language_requested: str | None = None + language_detected: str | None = None + language_detection_confidence: float | None = None + model_used: str | None = None + compute_type_used: str | None = None + downgrade_attempts: int = 0 + capability_tier: str | None = None + + +class Timings(BaseModel): + model_load_sec: float | None = None + transcription_sec: float | None = None + ingest_sec: float | None = None + postprocess_sec: float | None = None + rtf: float | None = None + + +class Segment(BaseModel): + index: int = Field(ge=0) + start: float = Field(ge=0) + end: float = Field(ge=0) + text: str = "" + avg_logprob: float | None = None + no_speech_prob: float | None = None + confidence: float | None = None + words: list[dict[str, Any]] | None = None + speaker: str | None = None + + +class ErrorInfo(BaseModel): + code: str + message: str + retryable: bool = False + + +class TranscriptResult(BaseModel): + schema_version: str = SCHEMA_VERSION + status: Literal["completed", "failed", "cancelled"] = "completed" + source: SourceInfo | None = None + normalized_audio: NormalizedAudio | None = None + execution: ExecutionInfo | None = None + timings: Timings = Field(default_factory=Timings) + text: str = "" + segments: list[Segment] = Field(default_factory=list) + warnings: list[str] = Field(default_factory=list) + error: ErrorInfo | None = None + # 확장: 후처리/화자/용어 보존 정보 + postprocessing: dict[str, Any] | None = None + entities: list[dict[str, Any]] | None = None + + @model_validator(mode="after") + def _check_state(self) -> TranscriptResult: + if self.status == "completed": + if self.error is not None: + raise ValueError("completed 결과에 error가 있을 수 없습니다") + elif self.status in ("failed", "cancelled"): + if self.error is None: + self.error = ErrorInfo( + code="transcription_failed" if self.status == "failed" else "cancelled", + message="", + ) + # text = 세그먼트 trim 텍스트를 단일 공백으로 연결 (계약) + if self.status == "completed" and self.segments and not self.text: + self.text = " ".join(s.text.strip() for s in self.segments if s.text.strip()) + return self + + def dict_v1(self) -> dict[str, Any]: + return self.model_dump(exclude_none=False) diff --git a/src/luke_scribe/results/retention.py b/src/luke_scribe/results/retention.py new file mode 100644 index 0000000..8888326 --- /dev/null +++ b/src/luke_scribe/results/retention.py @@ -0,0 +1,82 @@ +"""Retention — 결과 보관 TTL 청소 (plan §3.7e). + +- **터미널 상태(completed/failed/cancelled) Job만** 청소. +- {queued, processing}이 보유한 결과·임시물은 절대 건드리지 않는다 + (보관 경합 방지). +- TTL은 Job 메타의 ``completed_at``(또는 result.json mtime) 기준. +""" + +from __future__ import annotations + +import time +from datetime import UTC, datetime +from pathlib import Path + +from .store import ResultStore + +TERMINAL_META_FIELDS = ("completed_at", "failed_at", "cancelled_at") + + +class RetentionSweeper: + def __init__( + self, store: ResultStore, retention_days: int = 7, *, now: float | None = None + ) -> None: + self.store = store + self.retention_days = retention_days + self._now = now # 테스트용 fake clock + + def sweep(self) -> list[str]: + """만료된 터미널(completed/failed/cancelled) Job 디렉터리 삭제. + + 삭제된 job_id 목록 반환. {queued, processing}은 절대 건드리지 않는다 + (터미널 타임스탬프가 meta.json에 없으면 스킵). + """ + if self.retention_days <= 0: + return [] + now = self._now if self._now is not None else time.time() + cutoff = now - self.retention_days * 86400 + removed: list[str] = [] + for job_id, job_dir in self.store.iter_job_dirs(): + if self._job_stale(job_id, job_dir, cutoff): + self.store.delete_job(job_id) + removed.append(job_id) + return removed + + @staticmethod + def _ts_to_epoch(ts: object) -> float | None: + """ISO 8601 또는 epoch float 타임스탬프를 epoch 초로 변환.""" + if isinstance(ts, (int, float)): + return float(ts) + try: + dt = datetime.fromisoformat(str(ts)) + except (ValueError, TypeError): + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt.timestamp() + + def _job_stale(self, job_id: str, job_dir: Path, cutoff: float) -> bool: + """터미널 타임스탬프 기준 만료 판정. 결과/메타 모두 없으면 비터미널로 스킵.""" + meta_path = job_dir / "meta.json" + if meta_path.exists(): + import json + + try: + meta = json.loads(meta_path.read_text(encoding="utf-8")) + except Exception: + meta = {} + for field_name in TERMINAL_META_FIELDS: + ts = meta.get(field_name) + if not ts: + continue + # 터미널 타임스탬프 존재 → 만료 여부 결정 + ts_f = self._ts_to_epoch(ts) + return ts_f is not None and ts_f < cutoff + # 메타가 없으면 result.json mtime 폴백 (완료 경로 호환) + result_path = job_dir / "result.json" + if result_path.exists(): + try: + return result_path.stat().st_mtime < cutoff + except OSError: + return False + return False diff --git a/src/luke_scribe/results/store.py b/src/luke_scribe/results/store.py new file mode 100644 index 0000000..98e6111 --- /dev/null +++ b/src/luke_scribe/results/store.py @@ -0,0 +1,141 @@ +"""ResultStore — 결과 저장 (UUID 키, 경로 트래버설 차단). + +Eng 리뷰 P16/P20 반영: +- 결과 파일은 클라이언트 파일명이 아니라 **서버 생성 UUID 키**로 저장. +- ``format``은 enum으로 고정 확장자에 매핑 (사용자 문자열을 경로에 결합 금지). +- 모든 경로는 store root 하위로 canonicalize 검증, 심볼릭 링크 거부. +- 원자적 쓰기(임시 파일 → rename), 부분 JSON 파일 금지. +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import uuid +from pathlib import Path + +from ..errors import OutputWriteError +from ..results.models import TranscriptResult + +UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") + + +class ResultStore: + def __init__(self, root: str) -> None: + self.root = Path(root) + self.root.mkdir(parents=True, exist_ok=True) + + def _job_dir(self, job_id: str) -> Path: + if not UUID_RE.match(job_id): + raise ValueError(f"유효하지 않은 job id: {job_id!r}") + d = self.root / job_id + # root 하위 canonical 경로 검증 (심볼릭 링크/트래버설 방지) + canonical = d.resolve() + if not canonical.is_relative_to(self.root.resolve()): + raise ValueError(f"경로 탈출 시도: {job_id!r}") + return d + + def _reject_symlinks(self, d: Path) -> None: + if d.is_symlink(): + raise ValueError("심볼릭 링크가 있는 job 디렉터리는 허용하지 않습니다") + + def write_result(self, job_id: str, result: TranscriptResult) -> Path: + d = self._job_dir(job_id) + self._reject_symlinks(d) + d.mkdir(parents=True, exist_ok=True) + payload = json.dumps(result.model_dump(), ensure_ascii=False, indent=2) + tmp = d / f".result-{uuid.uuid4().hex}.tmp" + try: + tmp.write_text(payload, encoding="utf-8") + target = d / "result.json" + os.replace(tmp, target) + except OSError as exc: + try: + tmp.unlink(missing_ok=True) + except OSError: + pass + raise OutputWriteError(f"결과 저장 실패: {exc}") from exc + return target + + def read_result(self, job_id: str) -> TranscriptResult | None: + d = self._job_dir(job_id) + self._reject_symlinks(d) + p = d / "result.json" + if not p.exists(): + return None + return TranscriptResult.model_validate_json(p.read_text(encoding="utf-8")) + + def write_source_metadata(self, job_id: str, meta: dict) -> None: + d = self._job_dir(job_id) + self._reject_symlinks(d) + d.mkdir(parents=True, exist_ok=True) + (d / "meta.json").write_text(json.dumps(meta, ensure_ascii=False), encoding="utf-8") + + def source_path_for(self, job_id: str, source_name: str) -> Path: + """업로드 원본 저장 — 클라이언트 파일명을 **이름으로만** 사용, 확장자만 검증.""" + name = Path(source_name).name # 경로 성분 제거 + if not name or name in (".", ".."): + raise ValueError("잘못된 파일명") + d = self._job_dir(job_id) + d.mkdir(parents=True, exist_ok=True) + return d / name + + def iter_job_dirs(self) -> list[tuple[str, Path]]: + """모든 job 디렉터리 (job_id, dir) — retention sweeper용.""" + out: list[tuple[str, Path]] = [] + for entry in self.root.iterdir(): + if not entry.is_dir(): + continue + if not UUID_RE.match(entry.name): + continue + out.append((entry.name, entry)) + return out + + def iter_terminal_results(self) -> list[tuple[str, Path]]: + """result.json이 있는 job 디렉터리 — 기존 호환용.""" + out: list[tuple[str, Path]] = [] + for job_id, d in self.iter_job_dirs(): + p = d / "result.json" + if p.exists(): + out.append((job_id, p)) + return out + + def delete_job(self, job_id: str) -> None: + d = self._job_dir(job_id) + if d.exists(): + shutil.rmtree(d, ignore_errors=True) + + def delete_derived(self, job_id: str) -> None: + """원본 오디오 + 파생물 삭제 — 결과/메타만 보존 (privacy-first). + + 전사가 끝나면 업로드 원본은 즉시 삭제된다 (plan §3.7e/§6.1). + ``result.json``/``meta.json``은 보관 대상이므로 건드리지 않는다. + """ + d = self._job_dir(job_id) + self._reject_symlinks(d) + if not d.exists(): + return + for p in d.iterdir(): + if p.is_file() and p.name not in ("result.json", "meta.json"): + p.unlink(missing_ok=True) + + +class AtomicFileWriter: + """일반 파일 원자적 쓰기 — CLI --output 용.""" + + @staticmethod + def write(path: str | Path, data: str) -> None: + p = Path(path) + tmp = p.parent / f".{p.name}.{uuid.uuid4().hex}.tmp" + try: + p.parent.mkdir(parents=True, exist_ok=True) + tmp.write_text(data, encoding="utf-8") + os.replace(tmp, p) + except OSError as exc: + try: + tmp.unlink(missing_ok=True) + except OSError: + pass + raise OutputWriteError(f"출력 파일을 쓸 수 없습니다: {exc}") from exc diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..241b882 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,190 @@ +"""공유 테스트 픽스처. + +모든 테스트는 GPU/모델/ffmpeg 없이 mock 기반으로 동작한다: +- ``FakeEngineOwner``: 세그먼트를 즉시 생성 (실제 decode 없음) +- ``FakeIngestor``: ffprobe/ffmpeg 대체 +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from luke_scribe.config import Settings + + +class FakeSegments: + """세그먼트 목록을 반복하는 mock — 취소 시 중단.""" + + def __init__(self, segments: list[dict], cancel_on: callable | None = None) -> None: + self._segs = iter(segments) + self._cancel_on = cancel_on or (lambda: False) + + def __iter__(self): + return self + + def __next__(self): + if self._cancel_on(): + raise StopIteration + return next(self._segs) + + def close(self) -> None: + pass + + +class FakeEngine: + def __init__(self, segments: list[dict], *, fail_on: list[str] | None = None) -> None: + self.segments = segments + self.fail_on = fail_on or [] + self.calls: list[dict] = [] + + def transcribe(self, audio_path, options, should_cancel=None, download_progress=None) -> object: + self.calls.append({"audio": audio_path, "options": options}) + # 실제 faster-whisper를 import하지 않는 mock + return type( + "Outcome", + (), + { + "segments": FakeSegments(self.segments, cancel_on=should_cancel), + "info": {"language": "ko"}, + }, + )() + + +class FakeEngineOwner: + """EngineOwner mock — 프로파일 강등/캡을 검증 가능하게 함.""" + + def __init__( + self, engine: FakeEngine | None = None, segments: list[dict] | None = None + ) -> None: + self.engine = engine or FakeEngine(segments or _default_segments()) + self.last_request = None + + def transcribe(self, req) -> dict: + self.last_request = req + # 실제 EngineOwner처럼 첫 시도 프로파일을 기록 (downgrade_attempts 계산용) + req.attempted_profiles.append(f"{req.options.device}/{req.options.compute_type or 'int8'}") + outcome = self.engine.transcribe( + req.audio_path, req.options, should_cancel=req.should_cancel + ) + return { + "segments": outcome.segments, + "info": outcome.info, + "device": req.options.device if req.options.device != "auto" else "cpu", + "compute_type": req.options.compute_type or "int8", + "attempted_profiles": list(req.attempted_profiles), + } + + def emit_hypothesis(self, pcm_chunk: bytes) -> dict: + return {"segments": [], "audio_sec": len(pcm_chunk) / 2 / 16000} + + def unload_all(self) -> None: + pass + + +def _default_segments() -> list[dict]: + return [ + { + "index": 0, + "start": 0.0, + "end": 2.5, + "text": "오늘 API 서버에서 vLLM을 사용해 보겠습니다.", + "avg_logprob": -0.2, + "no_speech_prob": 0.01, + }, + { + "index": 1, + "start": 2.5, + "end": 4.0, + "text": "Kubernetes 클러스터에 배포합니다.", + "avg_logprob": -0.3, + "no_speech_prob": 0.02, + }, + ] + + +class FakeIngestor: + """ffprobe/ffmpeg 없는 mock ingestor.""" + + def __init__(self, duration_sec: float = 10.0, codec: str = "mp3") -> None: + self.duration_sec = duration_sec + self.codec = codec + self.cancelled = False + + def ingest(self, source: Path, *, should_cancel=None): + if should_cancel and should_cancel(): + from luke_scribe.errors import CancelledError + + self.cancelled = True + raise CancelledError("취소됨") + from luke_scribe.audio.ingest import IngestResult, ProbeResult + from luke_scribe.results.models import NormalizedAudio + + class _Cleanup: + def close(self): + pass + + return IngestResult( + normalized_path="/tmp/fake-normalized.wav", + normalized=NormalizedAudio(duration_sec=self.duration_sec), + probe=ProbeResult(duration_sec=self.duration_sec, codec=self.codec, size_bytes=100), + temp_dir="/tmp/fake-tmp", + cleanup=lambda: None, + ) + + +@pytest.fixture +def settings(tmp_path: Path) -> Settings: + return Settings( + _env_file=None, + results_dir=str(tmp_path / "results"), + api_key_file=str(tmp_path / "api_keys.json"), + queue_backend="inproc", + model_cache_dir=None, + max_queue=10, + ) + + +@pytest.fixture +def fake_owner() -> FakeEngineOwner: + return FakeEngineOwner() + + +@pytest.fixture +def fake_ingestor() -> FakeIngestor: + return FakeIngestor() + + +@pytest.fixture +def transcript_result() -> dict: + return { + "schema_version": "1.0", + "status": "completed", + "source": {"name": "test.mp3", "codec": "mp3", "size_bytes": 100}, + "normalized_audio": { + "duration_sec": 4.0, + "audio_format": "pcm_s16le", + "sample_rate": 16000, + "channels": 1, + }, + "execution": { + "model": "large-v3-turbo", + "device": "cpu", + "compute_type": "int8", + "language_requested": "ko", + }, + "timings": {"transcription_sec": 0.5, "rtf": 0.125}, + "text": "오늘 API 서버에서 vLLM을 사용해 보겠습니다.", + "segments": [ + { + "index": 0, + "start": 0.0, + "end": 2.5, + "text": "오늘 API 서버에서 vLLM을 사용해 보겠습니다.", + "avg_logprob": -0.2, + "no_speech_prob": 0.01, + } + ], + "warnings": [], + } diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py new file mode 100644 index 0000000..e84938b --- /dev/null +++ b/tests/integration/test_api.py @@ -0,0 +1,210 @@ +"""API 통합 테스트 — TestClient 기반 인증/업로드/소유권/큐 흐름. + +실제 모델·Redis 없이 in-proc 브로커 + mock 결과로 동작한다. +""" + +from __future__ import annotations + +import json + +import pytest +from fastapi.testclient import TestClient + +from luke_scribe.api.app import create_app +from luke_scribe.config import Settings + + +@pytest.fixture +def settings(tmp_path) -> Settings: + return Settings( + _env_file=None, + results_dir=str(tmp_path / "results"), + api_key_file=str(tmp_path / "api_keys.json"), + api_keys="key-transcribe,key-admin:admin,transcribe", + queue_backend="inproc", + model_cache_dir=None, + tunnel="none", + max_queue=10, + ) + + +@pytest.fixture +def client(settings: Settings) -> TestClient: + app = create_app(settings) + with TestClient(app) as c: + yield c + + +class TestHealth: + def test_health_public(self, client: TestClient): + r = client.get("/health") + assert r.status_code == 200 + body = r.json() + assert body["status"] == "ok" + assert body["model_ready"] is False # 모델 캐시 없음 → 프로비저닝 보류 + + +class TestAuth: + def test_missing_key_rejected(self, client: TestClient): + r = client.get("/v1/jobs") + assert r.status_code == 401 + + def test_bad_key_rejected(self, client: TestClient): + r = client.get("/v1/jobs", headers={"X-API-Key": "wrong-key"}) + assert r.status_code == 401 + + def test_admin_scope_enforced(self, client: TestClient): + r = client.get("/v1/system", headers={"X-API-Key": "key-transcribe"}) + assert r.status_code == 403 + r2 = client.get("/v1/system", headers={"X-API-Key": "key-admin"}) + assert r2.status_code == 200 + body = r2.json() + assert body["capability_tier"] in ("T0", "T1", "T2", "T3") + assert body["queue_depth"] == 0 + + +class TestJobs: + def test_create_get_cancel_flow(self, client: TestClient, settings: Settings): + headers = {"X-API-Key": "key-transcribe"} + r = client.post( + "/v1/jobs", + files={"file": ("meeting.mp3", b"fake-audio-bytes", "audio/mpeg")}, + data={"options": json.dumps({"language": "ko", "model": "large-v3-turbo"})}, + headers=headers, + ) + assert r.status_code == 202, r.text + body = r.json() + job_id = body["job_id"] + assert body["status"] == "queued" + assert body["queue_position"] == 0 + + # 조회 + r = client.get(f"/v1/jobs/{job_id}", headers=headers) + assert r.status_code == 200 + assert r.json()["status"] == "queued" + + # 결과는 아직 없음 + r = client.get(f"/v1/jobs/{job_id}/result", headers=headers) + assert r.status_code == 409 + + # 취소 + r = client.delete(f"/v1/jobs/{job_id}", headers=headers) + assert r.status_code == 200 + assert r.json()["status"] == "cancelled" + + def test_ownership_enforced(self, client: TestClient): + """Eng P14: 다른 키가 만든 job 조회/취소 불가.""" + a = client.post( + "/v1/jobs", + files={"file": ("a.mp3", b"x", "audio/mpeg")}, + data={"options": "{}"}, + headers={"X-API-Key": "key-transcribe"}, + ) + job_id = a.json()["job_id"] + r = client.get(f"/v1/jobs/{job_id}", headers={"X-API-Key": "key-admin"}) + assert r.status_code == 403 + r = client.delete(f"/v1/jobs/{job_id}", headers={"X-API-Key": "key-admin"}) + assert r.status_code == 403 + + def test_unknown_job_404(self, client: TestClient): + r = client.get( + "/v1/jobs/00000000-0000-0000-0000-000000000000", + headers={"X-API-Key": "key-transcribe"}, + ) + assert r.status_code == 404 + + def test_bad_options_422(self, client: TestClient): + r = client.post( + "/v1/jobs", + files={"file": ("a.mp3", b"x", "audio/mpeg")}, + data={"options": "not json {"}, + headers={"X-API-Key": "key-transcribe"}, + ) + assert r.status_code == 422 + + def test_bad_format_422(self, client: TestClient, settings: Settings): + headers = {"X-API-Key": "key-transcribe"} + r = client.post( + "/v1/jobs", + files={"file": ("a.mp3", b"x", "audio/mpeg")}, + data={"options": "{}"}, + headers=headers, + ) + job_id = r.json()["job_id"] + r2 = client.get(f"/v1/jobs/{job_id}/result?format=docx", headers=headers) + assert r2.status_code == 422 + + def test_oversize_413(self, client: TestClient, settings: Settings): + settings.max_upload_bytes = 10 + r = client.post( + "/v1/jobs", + files={"file": ("big.mp3", b"x" * 100, "audio/mpeg")}, + data={"options": "{}"}, + headers={"X-API-Key": "key-transcribe"}, + ) + assert r.status_code == 413 + + +class TestQueueFull: + def test_queue_full_429(self, client: TestClient, settings: Settings): + settings.max_queue = 1 + headers = {"X-API-Key": "key-transcribe"} + r1 = client.post( + "/v1/jobs", + files={"file": ("a.mp3", b"x", "audio/mpeg")}, + data={"options": "{}"}, + headers=headers, + ) + assert r1.status_code == 202 + r2 = client.post( + "/v1/jobs", + files={"file": ("b.mp3", b"x", "audio/mpeg")}, + data={"options": "{}"}, + headers=headers, + ) + assert r2.status_code == 429 + assert "Retry-After" in r2.headers + + +class TestResultEndpoint: + def test_result_after_processing(self, client: TestClient, settings: Settings, tmp_path): + """워커가 결과를 저장한 뒤 result 엔드포인트가 JSON/SRT를 반환.""" + from luke_scribe.results.models import TranscriptResult + from luke_scribe.results.store import ResultStore + + headers = {"X-API-Key": "key-transcribe"} + r = client.post( + "/v1/jobs", + files={"file": ("a.mp3", b"x", "audio/mpeg")}, + data={"options": "{}"}, + headers=headers, + ) + job_id = r.json()["job_id"] + + # 워커 대신 결과를 직접 기록 (모델 없이 mock) + result = TranscriptResult( + status="completed", + source={"name": "a.mp3", "codec": "mp3", "size_bytes": 1}, + normalized_audio={"duration_sec": 2.0}, + execution={"model": "large-v3-turbo", "device": "cpu", "compute_type": "int8"}, + text="테스트 전사 결과", + segments=[{"index": 0, "start": 0.0, "end": 1.0, "text": "테스트 전사 결과"}], + ) + from luke_scribe.jobqueue.jobs import JobStatus + + store = ResultStore(settings.results_dir) + store.write_result(job_id, result) + # 상태를 completed로 (워커가 했을 일) — queued→processing→completed + job = client.app.state.broker.get(job_id) + job.transition(JobStatus.PROCESSING) + job.transition(JobStatus.COMPLETED) + client.app.state.broker.save_meta(job) + + r_json = client.get(f"/v1/jobs/{job_id}/result?format=json", headers=headers) + assert r_json.status_code == 200 + assert json.loads(r_json.json()["content"])["text"] == "테스트 전사 결과" + + r_srt = client.get(f"/v1/jobs/{job_id}/result?format=srt", headers=headers) + assert r_srt.status_code == 200 + assert "WEBVTT" not in r_srt.json()["content"] # srt 포맷 + assert "--> " in r_srt.json()["content"] diff --git a/tests/integration/test_worker.py b/tests/integration/test_worker.py new file mode 100644 index 0000000..bfeef03 --- /dev/null +++ b/tests/integration/test_worker.py @@ -0,0 +1,267 @@ +"""Worker 통합 테스트 — 큐 → 클레임 → 전사 → 결과 저장 (전부 mock).""" + +from __future__ import annotations + +import time + +from luke_scribe.jobqueue.broker import InProcBroker +from luke_scribe.jobqueue.jobs import Job, JobStatus +from luke_scribe.jobqueue.worker import Worker, WorkerCallbacks +from luke_scribe.results.store import ResultStore + +from ..conftest import FakeEngineOwner, FakeIngestor + + +def _job(**kw) -> Job: + defaults = dict( + type="file", + lane="batch", + options={ + "model": "large-v3-turbo", + "language": "ko", + "device": "cpu", + "compute_type": "int8", + }, + source_path="/tmp/fake-source.mp3", + source_name="fake-source.mp3", + ) + defaults.update(kw) + return Job(**defaults) + + +class TestWorkerLifecycle: + def test_complete_flow(self, settings, tmp_path): + """enqueue → worker 처리 → completed + 결과 저장 + 콜백.""" + broker = InProcBroker(settings) + store = ResultStore(str(tmp_path / "results")) + owner = FakeEngineOwner() + done: list[str] = [] + + def on_done(job, result): + done.append(job.id) + + worker = Worker( + settings=settings, + broker=broker, + store=store, + owner=owner, + ingestor=FakeIngestor(duration_sec=4.0), + callbacks=WorkerCallbacks(on_job_done=on_done), + ) + job = _job() + broker.enqueue(job) + worker.drain() + + assert broker.get(job.id).status == JobStatus.COMPLETED + assert done == [job.id] + result = store.read_result(job.id) + assert result is not None + assert result.status == "completed" + assert result.text # 세그먼트 텍스트 연결 + assert result.execution is not None + assert result.execution.device == "cpu" + # 강등 없음 + assert result.execution.downgrade_attempts == 0 + + def test_meta_written(self, settings, tmp_path): + """완료 시 job 메타(완료 시각 포함)가 결과 디렉터리에 저장.""" + broker = InProcBroker(settings) + store = ResultStore(str(tmp_path / "results")) + worker = Worker( + settings=settings, + broker=broker, + store=store, + owner=FakeEngineOwner(), + ingestor=FakeIngestor(), + ) + job = _job() + broker.enqueue(job) + worker.drain() + import json + + meta = json.loads( + tmp_path.joinpath("results", job.id, "meta.json").read_text(encoding="utf-8") + ) + assert meta["status"] == "completed" + assert meta["completed_at"] is not None + + def test_transcription_failure_marks_failed(self, settings, tmp_path): + """엔진 오류 → failed + 오류 메타 저장.""" + + class BoomEngine: + def transcribe(self, req): + raise RuntimeError("engine exploded") + + class BoomOwner: + def __init__(self): + self.engine = BoomEngine() + + def transcribe(self, req): + return self.engine.transcribe(req) + + def unload_all(self): + pass + + broker = InProcBroker(settings) + store = ResultStore(str(tmp_path / "results")) + worker = Worker( + settings=settings, + broker=broker, + store=store, + owner=BoomOwner(), + ingestor=FakeIngestor(), + ) + job = _job() + broker.enqueue(job) + worker.drain() + assert broker.get(job.id).status == JobStatus.FAILED + + def test_cancel_before_claim(self, settings, tmp_path): + """큐에서 취소된 job은 워커가 claim하지 않고 cancelled 처리.""" + broker = InProcBroker(settings) + store = ResultStore(str(tmp_path / "results")) + worker = Worker( + settings=settings, + broker=broker, + store=store, + owner=FakeEngineOwner(), + ingestor=FakeIngestor(), + ) + job = _job() + broker.enqueue(job) + broker.cancel(job.id) + worker.drain() + assert broker.get(job.id).status == JobStatus.CANCELLED + + def test_cancel_during_processing(self, settings, tmp_path): + """처리 중 취소 → 세그먼트 경계에서 CancelledError → cancelled (Eng P4).""" + + class CancelAfterFirstOwner: + def __init__(self): + self.n = 0 + self.segments = [ + {"index": 0, "start": 0.0, "end": 1.0, "text": "첫 세그먼트"}, + {"index": 1, "start": 1.0, "end": 2.0, "text": "두 번째"}, + ] + + def transcribe(self, req): + # worker는 token 미전달, 취소는 세그먼트 경계에서 검사됨 + return { + "segments": iter( + [ + {"index": 0, "start": 0.0, "end": 1.0, "text": "첫 세그먼트"}, + {"index": 1, "start": 1.0, "end": 2.0, "text": "두 번째"}, + ] + ), + "info": {"language": "ko"}, + "device": "cpu", + "compute_type": "int8", + "attempted_profiles": ["cpu/int8"], + } + + # 세그먼트를 먼저 소비한 뒤 취소하도록: 첫 next 후 should_cancel True + broker = InProcBroker(settings) + store = ResultStore(str(tmp_path / "results")) + owner = CancelAfterFirstOwner() + worker = Worker( + settings=settings, + broker=broker, + store=store, + owner=owner, + ingestor=FakeIngestor(), + ) + job = _job() + broker.enqueue(job) + # 처리 시작 전에 취소 요청을 미리 걸어 두면 claim 단계에서 cancelled 처리됨. + # 대신 처리 중 취소를 시뮬레이션: claim 직후 cancel 호출 + claimed = broker.claim_next("w1", 60.0) + broker.cancel(claimed.id) + # 워커는 already-processing job을 직접 처리 + worker._process(claimed) + assert broker.get(job.id).status == JobStatus.CANCELLED + + +class TestPrivacyFirst: + def test_source_deleted_after_completion(self, settings, tmp_path): + """plan §3.7e/§6.1: 전사 완료 후 업로드 원본 오디오 즉시 삭제.""" + from luke_scribe.results.store import ResultStore + + settings.delete_source = True + store = ResultStore(str(tmp_path / "results")) + broker = InProcBroker(settings) + worker = Worker( + settings=settings, + broker=broker, + store=store, + owner=FakeEngineOwner(), + ingestor=FakeIngestor(), + ) + job = _job() + # source가 store에 저장된 파일을 가리키도록 + src = store.source_path_for(job.id, "meeting.mp3") + src.write_bytes(b"fake-audio") + job.source_path = str(src) + broker.enqueue(job) + worker.drain() + assert not src.exists() # 원본 삭제 + assert store.read_result(job.id) is not None # 결과는 보존 + + def test_source_kept_when_delete_source_false(self, settings, tmp_path): + from luke_scribe.results.store import ResultStore + + settings.delete_source = False + store = ResultStore(str(tmp_path / "results")) + broker = InProcBroker(settings) + worker = Worker( + settings=settings, + broker=broker, + store=store, + owner=FakeEngineOwner(), + ingestor=FakeIngestor(), + ) + job = _job() + src = store.source_path_for(job.id, "meeting.mp3") + src.write_bytes(b"fake-audio") + job.source_path = str(src) + broker.enqueue(job) + worker.drain() + assert src.exists() + + +class TestWorkerProgress: + def test_progress_emitted(self, settings, tmp_path): + broker = InProcBroker(settings) + store = ResultStore(str(tmp_path / "results")) + worker = Worker( + settings=settings, + broker=broker, + store=store, + owner=FakeEngineOwner(segments=[{"index": 0, "start": 0.0, "end": 5.0, "text": "x"}]), + ingestor=FakeIngestor(duration_sec=10.0), + ) + job = _job() + broker.enqueue(job) + worker.drain() + final = broker.get(job.id) + assert final.progress is not None + assert final.progress <= 1.0 + assert final.processed_sec is not None + + +class TestCrashRecovery: + def test_stale_processing_recovered(self, settings, tmp_path): + """Eng P11: 워커 크래시(리스 만료) → 스타트업 reconciler가 failed 처리.""" + broker = InProcBroker(settings) + job = _job() + broker.enqueue(job) + claimed = broker.claim_next("w1", 60.0) + # 리스 만료 시뮬레이션 + claimed.lease_expires_at = time.time() - 10 + broker.save_meta(claimed) + worker = Worker( + settings=settings, broker=broker, store=ResultStore(str(tmp_path / "results")) + ) + reclaimed = worker.start_reconcile() + assert job.id in reclaimed + assert broker.get(job.id).status == JobStatus.FAILED + assert broker.get(job.id).error_code == "worker_crash" diff --git a/tests/unit/test_devices.py b/tests/unit/test_devices.py new file mode 100644 index 0000000..c120111 --- /dev/null +++ b/tests/unit/test_devices.py @@ -0,0 +1,90 @@ +"""Device Manager 단위 테스트 — 정밀도 결정, 능력 등급, override.""" + +from __future__ import annotations + +import pytest + +from luke_scribe.devices.manager import DeviceManager +from luke_scribe.devices.vram_probe import GpuInfo, SystemInfo +from luke_scribe.errors import DeviceUnavailable + + +def _sys(gpus: list[GpuInfo], ram_mb: int = 16384) -> SystemInfo: + return SystemInfo( + cpu_count=8, ram_total_mb=ram_mb, ram_free_mb=ram_mb // 2, disk_free_mb=50000, gpus=gpus + ) + + +def _gpu(name: str, cc: str, total_mb: int, free_mb: int, index: int = 0) -> GpuInfo: + return GpuInfo( + index=index, + name=name, + compute_capability=cc, + vram_total_mb=total_mb, + vram_free_mb=free_mb, + driver_version="550", + runtime_cuda=None, + ) + + +class TestPrecision: + def test_cc_7_plus_12gb_free_float16(self): + m = DeviceManager(_sys([_gpu("T4", "7.5", 15360, 14820)])) + p = m.detect() + assert p.selected_compute_type == "float16" + assert p.selected_device == "cuda:0" + + def test_cc_7_plus_low_free_int8_float16(self): + m = DeviceManager(_sys([_gpu("L4", "8.9", 24576, 8000)])) + p = m.detect() + assert p.selected_compute_type == "int8_float16" + + def test_pascal_1050_int8(self): + m = DeviceManager(_sys([_gpu("GTX 1050", "6.1", 4096, 3584)])) + p = m.detect() + assert p.selected_compute_type == "int8" + + def test_no_gpu_cpu_int8(self): + m = DeviceManager(_sys([])) + p = m.detect() + assert p.selected_device == "cpu" + assert p.selected_compute_type == "int8" + assert p.capability_tier == "T0" + assert any("CPU" in w for w in p.warnings) + + +class TestCapabilityTier: + def test_t3_both_models(self): + m = DeviceManager(_sys([_gpu("A100", "9.0", 81920, 70000)])) + p = m.detect() + assert p.capability_tier == "T3" + assert p.workers >= 1 + + def test_t1_small_gpu(self): + m = DeviceManager(_sys([_gpu("RTX 3060", "8.6", 12288, 10000)])) + p = m.detect() + assert p.capability_tier in ("T1", "T2", "T3") + + +class TestOverrides: + def test_explicit_cuda_missing_fails(self): + m = DeviceManager(_sys([])) + with pytest.raises(DeviceUnavailable): + m.detect(device="cuda:0") + + def test_explicit_cuda_ok(self): + m = DeviceManager(_sys([_gpu("T4", "7.5", 15360, 14820)])) + p = m.detect(device="cuda:0", compute_type="float16") + assert p.selected_device == "cuda:0" + assert p.selection_source == "explicit" + assert p.selected_compute_type == "float16" + + def test_explicit_cpu(self): + m = DeviceManager(_sys([_gpu("T4", "7.5", 15360, 14820)])) + p = m.detect(device="cpu") + assert p.selected_device == "cpu" + + def test_explicit_bad_device(self): + m = DeviceManager(_sys([])) + with pytest.raises(DeviceUnavailable): + m.detect(device="tpu:0") diff --git a/tests/unit/test_engine_owner.py b/tests/unit/test_engine_owner.py new file mode 100644 index 0000000..c7f2899 --- /dev/null +++ b/tests/unit/test_engine_owner.py @@ -0,0 +1,95 @@ +"""EngineOwner 단위 테스트 — OOM 강등 체인, attempted_profiles 영속화, 우선순위.""" + +from __future__ import annotations + +import pytest + +from luke_scribe.engine.base import TranscriptionOptions +from luke_scribe.engine.owner import MAX_DOWNGRADES, EngineOwner, InferenceRequest +from luke_scribe.errors import OutOfMemory + + +class OOMFakeEngine: + """첫 N번 호출에서 OOM, 이후 성공하는 mock.""" + + def __init__(self, oom_calls: int = 0) -> None: + self.oom_calls = oom_calls + self.calls: list[dict] = [] + self.closed = False + + def transcribe(self, audio_path, options, should_cancel=None, download_progress=None): + self.calls.append({"device": options.device, "compute_type": options.compute_type}) + if len(self.calls) <= self.oom_calls: + raise OutOfMemory("CUDA OOM (mock)") + return type("O", (), {"segments": iter([]), "info": {}})() + + def unload_all(self): + self.closed = True + + +def _opts(**kw) -> TranscriptionOptions: + kw.setdefault("model", "large-v3-turbo") + kw.setdefault("device", "cuda:0") + kw.setdefault("compute_type", "float16") + return TranscriptionOptions(**kw) + + +def test_no_downgrade_on_success(): + owner = EngineOwner.__new__(EngineOwner) + owner.settings = None + owner._engine = OOMFakeEngine(oom_calls=0) + owner._lock = __import__("threading").Lock() + owner._realtime_priority = __import__("threading").Lock() + owner._stats = {"jobs": 0, "downgrades": 0, "oom": 0} + req = InferenceRequest(audio_path="/tmp/x.wav", options=_opts()) + out = owner.transcribe(req) + assert out["compute_type"] == "float16" + assert len(out["attempted_profiles"]) == 1 + + +def test_downgrade_chain_on_oom(): + engine = OOMFakeEngine(oom_calls=1) # 첫 프로파일에서 OOM + owner = EngineOwner.__new__(EngineOwner) + owner.settings = None + owner._engine = engine + owner._lock = __import__("threading").Lock() + owner._realtime_priority = __import__("threading").Lock() + owner._stats = {"jobs": 0, "downgrades": 0, "oom": 0} + req = InferenceRequest(audio_path="/tmp/x.wav", options=_opts()) + out = owner.transcribe(req) + # float16 → int8_float16 (강등 1회) + assert out["compute_type"] == "int8_float16" + assert len(out["attempted_profiles"]) == 2 + + +def test_downgrade_capped_at_max(): + engine = OOMFakeEngine(oom_calls=99) # 항상 OOM + owner = EngineOwner.__new__(EngineOwner) + owner.settings = None + owner._engine = engine + owner._lock = __import__("threading").Lock() + owner._realtime_priority = __import__("threading").Lock() + owner._stats = {"jobs": 0, "downgrades": 0, "oom": 0} + req = InferenceRequest(audio_path="/tmp/x.wav", options=_opts()) + with pytest.raises(OutOfMemory): + owner.transcribe(req) + # 강등 체인은 cpu까지 포함하지만 시도는 MAX_DOWNGRADES+1 이내 + assert len(engine.calls) <= MAX_DOWNGRADES + 1 + + +def test_attempted_profiles_persisted_for_retry(): + """재큐(retry) 시 attempted_profiles가 유지되어 총 시도가 캡된다.""" + engine = OOMFakeEngine(oom_calls=0) + owner = EngineOwner.__new__(EngineOwner) + owner.settings = None + owner._engine = engine + owner._lock = __import__("threading").Lock() + owner._realtime_priority = __import__("threading").Lock() + owner._stats = {"jobs": 0, "downgrades": 0, "oom": 0} + # 이미 2회 시도한 이력이 Job에 영속화된 상황 → 재시도는 즉시 실패해야 함 + req = InferenceRequest( + audio_path="/tmp/x.wav", options=_opts(), attempted_profiles=["float16", "int8_float16"] + ) + with pytest.raises(OutOfMemory): + owner.transcribe(req) + assert len(engine.calls) == 0 # 새 시도 없음 diff --git a/tests/unit/test_formats.py b/tests/unit/test_formats.py new file mode 100644 index 0000000..67401c7 --- /dev/null +++ b/tests/unit/test_formats.py @@ -0,0 +1,85 @@ +"""포맷 렌더링 (json/txt/srt/vtt) + 결과 스키마 계약 테스트.""" + +from __future__ import annotations + +import json + +import pytest + +from luke_scribe.results.formats import render +from luke_scribe.results.models import TranscriptResult + + +@pytest.fixture +def result(transcript_result: dict) -> TranscriptResult: + return TranscriptResult.model_validate(transcript_result) + + +class TestFormats: + def test_json_roundtrip(self, result: TranscriptResult): + out = render(result, "json") + data = json.loads(out) + assert data["schema_version"] == "1.0" + assert data["segments"][0]["text"].startswith("오늘") + assert data["text"] # 계약: text = segment 텍스트 연결 + + def test_txt(self, result: TranscriptResult): + out = render(result, "txt") + assert "오늘 API 서버에서" in out + assert out.endswith("\n") + + def test_srt(self, result: TranscriptResult): + out = render(result, "srt") + assert "1" in out + assert "00:00:00,000 --> 00:00:02,500" in out + # 세그먼트 순번 + 타임스탬프 + 텍스트 + assert "오늘 API 서버에서" in out + + def test_vtt(self, result: TranscriptResult): + out = render(result, "vtt") + assert out.startswith("WEBVTT") + assert "00:00:00.000 --> 00:00:02.500" in out + + def test_unsupported_format(self, result: TranscriptResult): + with pytest.raises(ValueError): + render(result, "xml") # type: ignore[arg-type] + + def test_empty_segments_srt(self): + r = TranscriptResult(status="completed", segments=[], text="") + out = render(r, "srt") + assert out == "" + + +class TestSchemaContract: + def test_status_completed_no_error(self, transcript_result: dict): + result = TranscriptResult.model_validate(transcript_result) + assert result.error is None + + def test_failed_gets_default_error(self): + r = TranscriptResult(status="failed", text="") + assert r.error is not None + assert r.error.code == "transcription_failed" + + def test_cancelled_gets_default_error(self): + r = TranscriptResult(status="cancelled", text="") + assert r.error is not None + assert r.error.code == "cancelled" + + def test_completed_error_rejected(self): + with pytest.raises(ValueError): + TranscriptResult(status="completed", error={"code": "x", "message": "y"}, text="") + + def test_segment_index_must_be_nonneg(self): + with pytest.raises(ValueError): + TranscriptResult( + status="completed", segments=[{"index": -1, "start": 0, "end": 1, "text": "x"}] + ) + + def test_duration_gt_zero(self): + with pytest.raises(ValueError): + TranscriptResult( + status="completed", + normalized_audio={"duration_sec": 0}, + segments=[], + text="", + ) diff --git a/tests/unit/test_ingest.py b/tests/unit/test_ingest.py new file mode 100644 index 0000000..92e3031 --- /dev/null +++ b/tests/unit/test_ingest.py @@ -0,0 +1,206 @@ +"""AudioIngestor — ffprobe 검증, 상한(크기/길이), 취소·임시파일 정리 테스트. + +ffprobe/ffmpeg 바이너리 없이 subprocess를 mock해서 동작을 검증한다. +""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from luke_scribe.audio.ingest import AudioIngestor +from luke_scribe.config import Settings +from luke_scribe.errors import ( + AudioProbeFailed, + CancelledError, + InvalidInput, + UnsupportedInputEnvelope, +) + + +@pytest.fixture +def settings(tmp_path) -> Settings: + return Settings( + _env_file=None, + max_duration_sec=14400, + max_upload_bytes=2 * 1024 * 1024 * 1024, + ) + + +def _fake_probe_json(*, duration: str = "120.0", has_audio: bool = True) -> str: + streams = ( + [{"codec_type": "audio", "codec_name": "mp3"}] if has_audio else [{"codec_type": "video"}] + ) + return json.dumps({"format": {"duration": duration}, "streams": streams}) + + +class FakeSubprocess: + """ffprobe 호출에 대한 확정적 mock.""" + + def __init__( + self, *, probe_json: str | None = None, probe_rc: int = 0, probe_err: str = "" + ) -> None: + self.probe_json = probe_json + self.probe_rc = probe_rc + self.probe_err = probe_err + self.calls: list[list[str]] = [] + + def run(self, cmd, **kwargs): + self.calls.append(cmd) + if "show_entries" in cmd: + return subprocess.CompletedProcess( + cmd, self.probe_rc, stdout=self.probe_json or "", stderr=self.probe_err + ) + return subprocess.CompletedProcess(cmd, 0, stdout=self.probe_json or "", stderr="") + + +class FakeStream: + """Popen stdout/stderr 용 — 즉시 EOF.""" + + def read(self, n: int = -1) -> bytes: + return b"" + + +def test_probe_success(settings: Settings, tmp_path, monkeypatch): + fake = FakeSubprocess(probe_json=_fake_probe_json(duration="120.0")) + monkeypatch.setattr("luke_scribe.audio.ingest.subprocess", fake) + src = tmp_path / "a.mp3" + src.write_bytes(b"x" * 100) + ing = AudioIngestor(settings) + probe = ing.probe(src) + assert probe.duration_sec == 120.0 + assert probe.codec == "mp3" + + +def test_probe_missing_file(settings: Settings, tmp_path): + ing = AudioIngestor(settings) + with pytest.raises(InvalidInput): + ing.probe(tmp_path / "nope.mp3") + + +def test_probe_size_limit(settings: Settings, tmp_path, monkeypatch): + settings.max_upload_bytes = 100 + src = tmp_path / "big.mp3" + src.write_bytes(b"x" * 200) + ing = AudioIngestor(settings) + with pytest.raises(UnsupportedInputEnvelope): + ing.probe(src) + + +def test_probe_duration_limit(settings: Settings, tmp_path, monkeypatch): + fake = FakeSubprocess(probe_json=_fake_probe_json(duration="999999")) + monkeypatch.setattr("luke_scribe.audio.ingest.subprocess", fake) + src = tmp_path / "long.mp3" + src.write_bytes(b"x" * 10) + ing = AudioIngestor(settings) + with pytest.raises(UnsupportedInputEnvelope): + ing.probe(src) + + +def test_probe_no_audio_stream(settings: Settings, tmp_path, monkeypatch): + fake = FakeSubprocess(probe_json=_fake_probe_json(has_audio=False)) + monkeypatch.setattr("luke_scribe.audio.ingest.subprocess", fake) + src = tmp_path / "video.mp4" + src.write_bytes(b"x" * 10) + ing = AudioIngestor(settings) + with pytest.raises(AudioProbeFailed): + ing.probe(src) + + +def test_probe_ffprobe_missing(settings: Settings, tmp_path, monkeypatch): + class NoFfprobe: + def run(self, cmd, **kwargs): + raise FileNotFoundError("ffprobe") + + monkeypatch.setattr("luke_scribe.audio.ingest.subprocess", NoFfprobe()) + src = tmp_path / "a.mp3" + src.write_bytes(b"x") + ing = AudioIngestor(settings) + with pytest.raises(AudioProbeFailed): + ing.probe(src) + + +def test_probe_bad_output(settings: Settings, tmp_path, monkeypatch): + fake = FakeSubprocess(probe_json="not-json{") + monkeypatch.setattr("luke_scribe.audio.ingest.subprocess", fake) + src = tmp_path / "a.mp3" + src.write_bytes(b"x") + ing = AudioIngestor(settings) + with pytest.raises(AudioProbeFailed): + ing.probe(src) + + +class TestIngestCancellation: + def test_cancel_before_ingest(self, settings: Settings, tmp_path): + src = tmp_path / "a.mp3" + src.write_bytes(b"x") + ing = AudioIngestor(settings) + with pytest.raises(CancelledError): + ing.ingest(src, should_cancel=lambda: True) + + def test_temp_dir_cleaned_on_cancel(self, settings: Settings, tmp_path, monkeypatch): + """취소/실패 시 임시 디렉터리가 반드시 정리된다 (Eng P1).""" + src = tmp_path / "a.mp3" + src.write_bytes(b"x") + + class CancellingPopen: + """무한 인코딩 중인 ffmpeg — wait 폴링에서 계속 TimeoutExpired.""" + + def __init__(self, cmd, **kwargs): + self.stdout = FakeStream() + self.stderr = FakeStream() + self.pid = 99999999 # 존재하지 않는 pid → _kill_group의 ProcessLookupError 경로 + + def wait(self, timeout=None): + raise subprocess.TimeoutExpired(cmd=[], timeout=0.5) + + def kill(self): + pass + + # ffprobe는 성공, ffmpeg Popen은 취소 폴링 루프에 진입 + fake = FakeSubprocess(probe_json=_fake_probe_json(duration="60.0")) + monkeypatch.setattr("luke_scribe.audio.ingest.subprocess.run", fake.run) + monkeypatch.setattr("luke_scribe.audio.ingest.subprocess.Popen", CancellingPopen) + monkeypatch.setattr( + "luke_scribe.audio.ingest.tempfile.mkdtemp", + lambda prefix="": str(tmp_path / "ingest-tmp"), + ) + ing = AudioIngestor(settings) + # 초기 검사 2회(프로브 전/후)는 통과, ffmpeg 폴링 루프에서 취소 + calls = {"n": 0} + + def should_cancel(): + calls["n"] += 1 + return calls["n"] > 2 + + with pytest.raises(CancelledError): + ing.ingest(src, should_cancel=should_cancel) + assert not Path(tmp_path / "ingest-tmp").exists() + + def test_temp_dir_cleaned_on_ffmpeg_failure(self, settings: Settings, tmp_path, monkeypatch): + src = tmp_path / "a.mp3" + src.write_bytes(b"x") + + class FailingFfmpeg: + def __init__(self, cmd, **kwargs): + self.stdout = FakeStream() + self.stderr = FakeStream() + self.returncode = 1 + + def wait(self, timeout=None): + return 1 + + fake = FakeSubprocess(probe_json=_fake_probe_json(duration="60.0")) + monkeypatch.setattr("luke_scribe.audio.ingest.subprocess.Popen", FailingFfmpeg) + monkeypatch.setattr("luke_scribe.audio.ingest.subprocess.run", fake.run) + monkeypatch.setattr( + "luke_scribe.audio.ingest.tempfile.mkdtemp", + lambda prefix="": str(tmp_path / "ingest-tmp2"), + ) + ing = AudioIngestor(settings) + with pytest.raises(AudioProbeFailed): + ing.ingest(src) + assert not Path(tmp_path / "ingest-tmp2").exists() diff --git a/tests/unit/test_jobs.py b/tests/unit/test_jobs.py new file mode 100644 index 0000000..371dd46 --- /dev/null +++ b/tests/unit/test_jobs.py @@ -0,0 +1,106 @@ +"""Job 상태 머신 + in-proc 브로커 단위 테스트.""" + +from __future__ import annotations + +import time + +import pytest + +from luke_scribe.errors import JobAlreadyTerminal, QueueFull +from luke_scribe.jobqueue.broker import InProcBroker +from luke_scribe.jobqueue.jobs import Job, JobStatus + + +class TestJobStateMachine: + def test_legal_transitions(self): + job = Job() + assert job.status == JobStatus.QUEUED + job.transition(JobStatus.PROCESSING) + assert job.status == JobStatus.PROCESSING + job.transition(JobStatus.COMPLETED) + assert job.status == JobStatus.COMPLETED + + def test_illegal_transition_rejected(self): + job = Job() + job.transition(JobStatus.PROCESSING) + job.transition(JobStatus.COMPLETED) + with pytest.raises(JobAlreadyTerminal): + job.transition(JobStatus.PROCESSING) # terminal에서 되돌아갈 수 없음 + + def test_queued_cancel(self): + job = Job() + job.transition(JobStatus.CANCELLED) + assert job.status == JobStatus.CANCELLED + assert job.cancelled_at is not None + + def test_lease_expiry(self): + job = Job() + job.refresh_lease(60.0, now=1000.0) + assert not job.lease_expired(now=1050.0) + assert job.lease_expired(now=1100.0) + + def test_attempted_profiles_persisted(self): + job = Job(attempted_profiles=["float16", "int8_float16"]) + assert job.to_meta()["attempted_profiles"] == ["float16", "int8_float16"] + restored = Job.from_meta(job.to_meta()) + assert restored.attempted_profiles == ["float16", "int8_float16"] + + +class TestInProcBroker: + def test_enqueue_claim_complete(self): + broker = InProcBroker() + job = Job() + broker.enqueue(job) + claimed = broker.claim_next("w1", 60.0) + assert claimed is not None + assert claimed.status == JobStatus.PROCESSING + broker.transition(claimed, JobStatus.COMPLETED) + assert broker.get(job.id).status == JobStatus.COMPLETED + + def test_queue_position(self): + broker = InProcBroker() + a, b = Job(), Job() + broker.enqueue(a) + broker.enqueue(b) + assert a.queue_position == 0 + assert b.queue_position == 1 + + def test_queue_full(self): + broker = InProcBroker() + broker.settings.max_queue = 2 + broker.enqueue(Job()) + broker.enqueue(Job()) + with pytest.raises(QueueFull): + broker.enqueue(Job()) + + def test_cancel_queued(self): + broker = InProcBroker() + job = Job() + broker.enqueue(job) + cancelled = broker.cancel(job.id) + assert cancelled.status == JobStatus.CANCELLED + assert broker.queue_depth() == 0 + + def test_cancel_processing_sets_flag(self): + broker = InProcBroker() + job = Job() + broker.enqueue(job) + broker.claim_next("w1", 60.0) + cancelled = broker.cancel(job.id) + assert cancelled.cancel_requested is True + assert cancelled.status == JobStatus.PROCESSING # 워커가 세그먼트 경계에서 종료 + + def test_reconcile_stale_recovers_crash(self): + """Eng 리뷰 P11: 워커 크래시 → stale processing → failed 복구.""" + broker = InProcBroker() + job = Job() + broker.enqueue(job) + broker.claim_next("w1", 60.0) # processing + lease 60s + # lease 만료 시뮬레이션 + job2 = broker.get(job.id) + job2.lease_expires_at = time.time() - 1 + broker.save_meta(job2) + reclaimed = broker.reconcile_stale(60.0) + assert job.id in reclaimed + assert broker.get(job.id).status == JobStatus.FAILED + assert broker.get(job.id).error_code == "worker_crash" diff --git a/tests/unit/test_keystore.py b/tests/unit/test_keystore.py new file mode 100644 index 0000000..f8efc54 --- /dev/null +++ b/tests/unit/test_keystore.py @@ -0,0 +1,86 @@ +"""KeyStore — 다이제스트 인증, 스코프, 키 생성 테스트.""" + +from __future__ import annotations + +import json + +import pytest + +from luke_scribe.api.deps import KeyStore, Principal +from luke_scribe.config import Settings +from luke_scribe.errors import AuthError, ScopeDenied + + +@pytest.fixture +def settings(tmp_path) -> Settings: + return Settings( + _env_file=None, + api_keys="key-one,key-admin:admin,transcribe", + api_key_file=str(tmp_path / "api_keys.json"), + ) + + +class TestKeyStoreAuth: + def test_plain_key_authenticates(self, settings: Settings): + ks = KeyStore(settings) + p = ks.authenticate("key-one") + assert p.key_id.startswith("k-key-one") + assert "transcribe" in p.scopes + + def test_scoped_key(self, settings: Settings): + ks = KeyStore(settings) + p = ks.authenticate("key-admin") + assert {"admin", "transcribe"} <= p.scopes + + def test_wrong_key_rejected(self, settings: Settings): + ks = KeyStore(settings) + with pytest.raises(AuthError): + ks.authenticate("key-wrong") + + def test_empty_key_rejected(self, settings: Settings): + ks = KeyStore(settings) + with pytest.raises(AuthError): + ks.authenticate(None) + + def test_digest_only_on_disk(self, settings: Settings, tmp_path): + """평문 키는 저장하지 않고 다이제스트만 파일에 기록 (Eng P14).""" + created = KeyStore(settings).create_key( + scopes=["transcribe"], save_path=settings.api_key_file + ) + raw = created["key"] + data = json.loads(tmp_path.joinpath("api_keys.json").read_text()) + assert raw not in json.dumps(data) + entry = data["keys"][0] + assert entry["digest"] != raw + assert len(entry["digest"]) == 64 # sha256 hex + + def test_file_keys_loaded(self, settings: Settings, tmp_path): + ks = KeyStore(settings) + created = ks.create_key(scopes=["transcribe"], save_path=settings.api_key_file) + # 새 인스턴스가 파일에서 로드 + ks2 = KeyStore(settings) + p = ks2.authenticate(created["key"]) + assert "transcribe" in p.scopes + + +class TestPrincipalScopes: + def test_require_scope_ok(self): + p = Principal(key_id="k-1", scopes={"transcribe"}) + p.require_scope("transcribe") # no raise + + def test_require_scope_denied(self): + p = Principal(key_id="k-1", scopes={"transcribe"}) + with pytest.raises(ScopeDenied): + p.require_scope("admin") + + def test_require_scope_empty(self): + p = Principal(key_id="k-1", scopes=set()) + with pytest.raises(ScopeDenied): + p.require_scope("transcribe") + + +class TestDigest: + def test_same_key_same_digest(self, settings: Settings): + ks = KeyStore(settings) + assert ks._digest("abc") == ks._digest("abc") + assert ks._digest("abc") != ks._digest("abd") diff --git a/tests/unit/test_metrics.py b/tests/unit/test_metrics.py new file mode 100644 index 0000000..2182d95 --- /dev/null +++ b/tests/unit/test_metrics.py @@ -0,0 +1,84 @@ +"""벤치마크 지표 단위 테스트 — WER/K-CER/entity 보존.""" + +from __future__ import annotations + +from luke_scribe.benchmark.metrics import ( + character_kcer, + clip_metrics, + entity_retention, + normalize_kcer, + word_wer, +) + + +class TestKcer: + def test_normalization_nfkc_punct(self): + # NFKC: 유사 문자 통일, 문장부호/공백 제거 + assert ( + normalize_kcer("오늘 API 서버에서, vLLM을 사용해 보겠습니다.") + == "오늘api서버에서vllm을사용해보겠습니다" + ) + + def test_exact_match_zero(self): + assert character_kcer("오늘 API 사용", "오늘 API 사용") == 0.0 + + def test_punct_diff_zero(self): + # 문장부호 차이는 K-CER에서 0 (정규화됨) + assert character_kcer("오늘 API 사용.", "오늘 API 사용") == 0.0 + + def test_one_char_diff(self): + ref = "오늘 API 사용" + hyp = "오늘 APT 사용" + assert 0 < character_kcer(ref, hyp) < 0.3 + + +class TestWer: + def test_exact(self): + assert word_wer("a b c", "a b c") == 0.0 + + def test_one_sub(self): + assert word_wer("a b c", "a x c") == 1 / 3 + + def test_empty_ref(self): + assert word_wer("", "") == 0.0 + assert word_wer("", "a") == 1.0 + + +class TestEntityRetention: + def test_preserved(self): + ref = "오늘 API 서버에서 vLLM을 사용합니다" + hyp = "오늘 API 서버에서 vLLM을 사용합니다" + entities = [ + {"canonical": "API", "surface": "API", "start_char": 3, "end_char": 6}, + {"canonical": "vLLM", "surface": "vLLM", "start_char": 12, "end_char": 16}, + ] + p, t = entity_retention(ref, hyp, entities) + assert p == 2 and t == 2 + + def test_lost(self): + ref = "오늘 API 서버" + hyp = "오늘 에이피아이 서버" + entities = [{"canonical": "API", "surface": "API", "start_char": 3, "end_char": 6}] + p, t = entity_retention(ref, hyp, entities) + assert p == 0 and t == 1 + + def test_one_occurrence_one_match(self): + # 같은 entity가 두 번 등장 → 2개 annotation, 둘 다 보존 + ref = "API 서버와 API 게이트웨이" + hyp = "API 서버와 API 게이트웨이" + entities = [ + {"canonical": "API", "surface": "API", "start_char": 0, "end_char": 3}, + {"canonical": "API", "surface": "API", "start_char": 8, "end_char": 11}, + ] + p, t = entity_retention(ref, hyp, entities) + assert p == 2 and t == 2 + + +class TestClipMetrics: + def test_perfect(self): + ref = "오늘 API 서버에서 vLLM을 사용합니다" + m = clip_metrics( + ref, ref, [{"canonical": "API", "surface": "API", "start_char": 3, "end_char": 6}] + ) + assert m.wer == 0.0 and m.k_cer == 0.0 + assert m.entities_preserved == 1 and m.entities_total == 1 diff --git a/tests/unit/test_postprocess.py b/tests/unit/test_postprocess.py new file mode 100644 index 0000000..084a6f7 --- /dev/null +++ b/tests/unit/test_postprocess.py @@ -0,0 +1,112 @@ +"""후처리 단위 테스트 — glossary/rules/confidence/span-aware.""" + +from __future__ import annotations + +from luke_scribe.config import Settings +from luke_scribe.engine.base import TranscriptionOptions +from luke_scribe.postprocess.confidence import confidence_flags, segment_confidence +from luke_scribe.postprocess.glossary import apply_glossary +from luke_scribe.postprocess.llm import EgressGuard +from luke_scribe.postprocess.pipeline import run_postprocess +from luke_scribe.postprocess.rules import apply_rules +from luke_scribe.results.models import Segment + + +def _segments(texts: list[str], logprobs: list[float] | None = None) -> list[Segment]: + out = [] + t = 0.0 + for i, text in enumerate(texts): + lp = logprobs[i] if logprobs else -0.2 + out.append( + Segment(index=i, start=t, end=t + 1.0, text=text, avg_logprob=lp, no_speech_prob=0.01) + ) + t += 1.0 + return out + + +class TestGlossary: + def test_restores_romanized(self): + segs = _segments(["브이엘엘엠 서버를 배포했습니다"]) + out = apply_glossary(segs, {"브이엘엘엠": "vLLM"}) + assert out["corrections"][0]["to"] == "vLLM" + assert "vLLM" in out["segments"][0].text + + def test_no_double_match(self): + segs = _segments(["API API"]) + out = apply_glossary(segs, {"API": "API"}) + assert len(out["corrections"]) == 0 # 이미 표준 표기, 교정 없음 + + +class TestRules: + def test_vllm_spaced(self): + segs = _segments(["v l l m 추론 서버"]) + out = apply_rules(segs) + assert "vLLM" in out["segments"][0].text + + def test_whitespace_collapse(self): + segs = _segments(["오늘 API 서버"]) + out = apply_rules(segs) + assert out["segments"][0].text == "오늘 API 서버" + + +class TestConfidence: + def test_low_logprob_flagged(self): + segs = _segments(["불확실한 전사"], logprobs=[-2.0]) + assert segs[0].index in confidence_flags(segs) + + def test_high_confidence_not_flagged(self): + segs = _segments(["확실한 전사"], logprobs=[-0.1]) + assert confidence_flags(segs) == [] + + def test_segment_confidence_range(self): + assert ( + segment_confidence(Segment(index=0, start=0, end=1, text="x", avg_logprob=-0.5)) == 0.5 + ) + + +class TestPipeline: + def test_rules_mode_applies(self): + segs = _segments(["v l l m 서버"]) + settings = Settings(_env_file=None, post_mode="rules", post_enabled=True) + out = run_postprocess(segs, TranscriptionOptions(), settings) + assert any("vLLM" in s.text for s in out["segments"]) + assert out["meta"]["mode"] == "rules" + + def test_none_mode_unchanged(self): + segs = _segments(["v l l m 서버"]) + settings = Settings(_env_file=None, post_mode="none", post_enabled=False) + out = run_postprocess(segs, TranscriptionOptions(), settings) + assert out["segments"][0].text == "v l l m 서버" + + def test_span_aware_keeps_timestamps(self): + # 교정 후에도 세그먼트 타임라인(start/end)은 보존 + segs = _segments(["브이엘엘엠 서버"]) + before = [(s.start, s.end) for s in segs] + out = apply_glossary(segs, {"브이엘엘엠": "vLLM"}) + after = [(s.start, s.end) for s in out["segments"]] + assert before == after + + +class TestEgressGuard: + def test_private_ip_rejected(self): + guard = EgressGuard({"llm": "http://192.168.0.1:8080/v1"}) + assert guard.resolve("llm") is None + + def test_http_scheme_rejected(self): + guard = EgressGuard({"llm": "http://example.com/v1"}) + assert guard.resolve("llm") is None + + def test_https_public_allowed(self, monkeypatch): + # 오프라인 테스트 환경에서 DNS 해석을 공용 IP로 모킹 + monkeypatch.setattr( + "luke_scribe.postprocess.llm.socket.gethostbyname", lambda host: "93.184.216.34" + ) + guard = EgressGuard({"llm": "https://api.example.com/v1"}) + assert guard.resolve("llm") is not None + + def test_private_hostname_resolved_rejected(self, monkeypatch): + monkeypatch.setattr( + "luke_scribe.postprocess.llm.socket.gethostbyname", lambda host: "10.0.0.5" + ) + guard = EgressGuard({"llm": "https://internal.example.com/v1"}) + assert guard.resolve("llm") is None diff --git a/tests/unit/test_realtime.py b/tests/unit/test_realtime.py new file mode 100644 index 0000000..8725e2f --- /dev/null +++ b/tests/unit/test_realtime.py @@ -0,0 +1,77 @@ +"""실시간 LocalAgreement 단위 테스트 — 버퍼 절단 불변식, 단조 타임스탬프.""" + +from __future__ import annotations + +from luke_scribe.pipeline.realtime import LocalAgreement +from luke_scribe.results.models import Segment + + +def _segs(texts: list[str], start: float = 0.0) -> list[Segment]: + out = [] + t = start + for i, text in enumerate(texts): + out.append(Segment(index=i, start=t, end=t + 1.0, text=text)) + t += 1.0 + return out + + +class TestAgreement: + def test_no_confirmation_until_agreement_count(self): + la = LocalAgreement(agreement_count=2) + e1 = la.feed(_segs(["첫 번째 문장"]), 2.0) + assert e1["confirmed"] == [] # 1회만으로는 미확정 + + def test_stable_prefix_confirmed(self): + la = LocalAgreement(agreement_count=2) + la.feed(_segs(["오늘 API 서버"]), 2.0) + e2 = la.feed(_segs(["오늘 API 서버에서 vLLM"]), 2.0) + confirmed = e2["confirmed"] + assert confirmed and confirmed[0].text == "오늘 API 서버" + assert e2["truncated"] is True + + def test_confirmed_immutable(self): + """확정 세그먼트는 이후 가설에 의해 절대 변경되지 않는다 (불변식 1).""" + la = LocalAgreement(agreement_count=2) + la.feed(_segs(["오늘 API"]), 2.0) + e2 = la.feed(_segs(["오늘 API 서버"]), 2.0) + first_confirmed = [s.text for s in e2["confirmed"]] + la.feed(_segs(["오늘 API 서버에서 vLLM 사용"]), 2.0) + # 확정분은 그대로 + assert [s.text for s in la.state.confirmed_segments][ + : len(first_confirmed) + ] == first_confirmed + + def test_no_duplicate_confirmation(self): + """이미 확정된 텍스트는 다음 가설에서 재발행되지 않는다 (Eng P18).""" + la = LocalAgreement(agreement_count=2) + la.feed(_segs(["안정된 발화"]), 2.0) + e2 = la.feed(_segs(["안정된 발화", "다음 문장"]), 2.0) + assert [s.text for s in e2["confirmed"]] == ["안정된 발화"] + # 다음 피드: "안정된 발화"는 재발행 금지, 새 텍스트만 확정 + e3 = la.feed(_segs(["안정된 발화", "다음 문장", "셋째 문장"]), 2.0) + assert [s.text for s in e3["confirmed"]] == ["다음 문장"] + assert [s.text for s in la.state.confirmed_segments] == ["안정된 발화", "다음 문장"] + + def test_monotonic_timestamps(self): + """확정 세그먼트 시간은 단조 증가 (불변식 3).""" + la = LocalAgreement(agreement_count=2) + la.feed(_segs(["첫 문장", "둘째 문장"]), 4.0) + la.feed(_segs(["첫 문장", "둘째 문장", "셋째 문장"]), 4.0) + ends = [s.end for s in la.state.confirmed_segments] + assert all(b >= a for a, b in zip(ends, ends[1:], strict=False)) + + def test_buffer_truncation_invariant(self): + """확정 방출 후 보류 버퍼는 left_context 이하로 절단 (불변식 2).""" + la = LocalAgreement(agreement_count=2, retained_left_context_sec=1.0) + la.feed(_segs(["a", "b", "c"]), 3.0) + e2 = la.feed(_segs(["a", "b", "c", "d", "e", "f"]), 3.0) + assert len(e2["confirmed"]) >= 1 + # audio offset은 마지막 확정 end - left_context + assert la.state.audio_offset_sec >= 0 + + def test_flush_confirms_pending(self): + la = LocalAgreement(agreement_count=2) + la.feed(_segs(["마지막 발화"]), 2.0) + flushed = la.flush() + assert flushed["confirmed"] + assert "마지막 발화" in flushed["new_text"] diff --git a/tests/unit/test_store.py b/tests/unit/test_store.py new file mode 100644 index 0000000..9efc2c1 --- /dev/null +++ b/tests/unit/test_store.py @@ -0,0 +1,184 @@ +"""ResultStore — UUID 키, 경로 트래버설 차단, 원자적 쓰기, 보관 TTL 테스트.""" + +from __future__ import annotations + +import os +import time +import uuid + +import pytest + +from luke_scribe.errors import OutputWriteError +from luke_scribe.results.models import TranscriptResult +from luke_scribe.results.retention import RetentionSweeper +from luke_scribe.results.store import ResultStore + + +@pytest.fixture +def store(tmp_path) -> ResultStore: + return ResultStore(str(tmp_path / "results")) + + +@pytest.fixture +def result(transcript_result: dict) -> TranscriptResult: + return TranscriptResult.model_validate(transcript_result) + + +class TestResultStore: + def test_write_read_roundtrip(self, store: ResultStore, result: TranscriptResult): + job_id = str(uuid.uuid4()) + store.write_result(job_id, result) + read = store.read_result(job_id) + assert read is not None + assert read.text == result.text + assert read.segments[0].text == result.segments[0].text + + def test_missing_result_returns_none(self, store: ResultStore): + assert store.read_result(str(uuid.uuid4())) is None + + def test_path_traversal_rejected(self, store: ResultStore): + with pytest.raises(ValueError): + store._job_dir("../../etc/passwd") + with pytest.raises(ValueError): + store._job_dir("..") + + def test_source_path_sanitized(self, store: ResultStore): + job_id = str(uuid.uuid4()) + p = store.source_path_for(job_id, "../../../evil.mp3") + assert ".." not in str(p) + assert p.name == "evil.mp3" + + def test_symlink_job_dir_rejected(self, store: ResultStore, result: TranscriptResult): + job_id = str(uuid.uuid4()) + real = store.root / f"real-{uuid.uuid4().hex}" + real.mkdir(parents=True) + link = store.root / job_id + os.symlink(real, link) + with pytest.raises(ValueError): + store.write_result(job_id, result) + + def test_atomic_write_no_partial_json(self, store: ResultStore, result: TranscriptResult): + job_id = str(uuid.uuid4()) + target = store.write_result(job_id, result) + # 임시 파일이 남지 않아야 한다 + leftovers = [p for p in target.parent.iterdir() if p.name.startswith(".result-")] + assert leftovers == [] + + def test_delete_job(self, store: ResultStore, result: TranscriptResult): + job_id = str(uuid.uuid4()) + store.write_result(job_id, result) + store.delete_job(job_id) + assert store.read_result(job_id) is None + + def test_iter_job_dirs_includes_all(self, store: ResultStore, result: TranscriptResult): + done_id = str(uuid.uuid4()) + store.write_result(done_id, result) + empty_id = str(uuid.uuid4()) + store._job_dir(empty_id).mkdir(parents=True) + entries = dict(store.iter_job_dirs()) + assert done_id in entries + assert empty_id in entries + + def test_delete_derived_keeps_result_and_meta( + self, store: ResultStore, result: TranscriptResult + ): + job_id = str(uuid.uuid4()) + src = store.source_path_for(job_id, "meeting.mp3") + src.write_bytes(b"audio") + store.write_result(job_id, result) + store.write_source_metadata(job_id, {"status": "completed", "completed_at": 1.0}) + store.delete_derived(job_id) + assert not src.exists() # 원본 오디오 삭제 (privacy-first) + assert store.read_result(job_id) is not None # 결과 보존 + assert (store._job_dir(job_id) / "meta.json").exists() + + +class TestRetentionSweeper: + def _write_terminal( + self, store: ResultStore, result: TranscriptResult, completed_at: float + ) -> str: + job_id = str(uuid.uuid4()) + store.write_result(job_id, result) + store.write_source_metadata(job_id, {"completed_at": completed_at}) + return job_id + + def test_sweeps_stale_terminal(self, store: ResultStore, result: TranscriptResult): + now = time.time() + old = self._write_terminal(store, result, completed_at=now - 8 * 86400) + fresh = self._write_terminal(store, result, completed_at=now - 3600) + sweeper = RetentionSweeper(store, retention_days=7, now=now) + removed = sweeper.sweep() + assert old in removed + assert fresh not in removed + assert store.read_result(old) is None + assert store.read_result(fresh) is not None + + def test_iso_timestamp_parsed(self, store: ResultStore, result: TranscriptResult): + """ISO 8601 타임스탬프도 처리 (워커는 float, 다른 경로는 ISO 가능).""" + from datetime import UTC, datetime + + now = time.time() + iso_old = datetime.fromtimestamp(now - 8 * 86400, tz=UTC).isoformat() + job_id = str(uuid.uuid4()) + store.write_result(job_id, result) + store.write_source_metadata(job_id, {"completed_at": iso_old}) + removed = RetentionSweeper(store, retention_days=7, now=now).sweep() + assert job_id in removed + + def test_mtime_fallback(self, store: ResultStore, result: TranscriptResult): + """메타가 없으면 result.json mtime 기준 폴백.""" + job_id = str(uuid.uuid4()) + p = store.write_result(job_id, result) + old = time.time() - 8 * 86400 + os.utime(p, (old, old)) + removed = RetentionSweeper(store, retention_days=7, now=time.time()).sweep() + assert job_id in removed + + def test_retention_disabled(self, store: ResultStore, result: TranscriptResult): + job_id = self._write_terminal(store, result, completed_at=time.time() - 8 * 86400) + sweeper = RetentionSweeper(store, retention_days=0, now=time.time()) + assert sweeper.sweep() == [] + assert store.read_result(job_id) is not None + + def test_sweeps_failed_job_with_meta_only(self, store: ResultStore, result: TranscriptResult): + """Eng 리뷰: failed/cancelled job(meta.json만)도 보관 정리 대상.""" + now = time.time() + failed_id = str(uuid.uuid4()) + store._job_dir(failed_id).mkdir(parents=True) + store.write_source_metadata( + failed_id, + {"status": "failed", "error_code": "worker_crash", "failed_at": now - 8 * 86400}, + ) + removed = RetentionSweeper(store, retention_days=7, now=now).sweep() + assert failed_id in removed + assert not store._job_dir(failed_id).exists() + + def test_processing_job_not_swept(self, store: ResultStore, result: TranscriptResult): + """queued/processing(터미널 타임스탬프 없음)은 절대 삭제 안 함.""" + now = time.time() + queued_id = str(uuid.uuid4()) + store._job_dir(queued_id).mkdir(parents=True) + store.write_source_metadata(queued_id, {"status": "queued"}) + removed = RetentionSweeper(store, retention_days=7, now=now).sweep() + assert queued_id not in removed + assert store._job_dir(queued_id).exists() + + +class TestAtomicFileWriter: + def test_write_replaces(self, tmp_path): + from luke_scribe.results.store import AtomicFileWriter + + target = tmp_path / "out.txt" + AtomicFileWriter.write(target, "hello") + assert target.read_text() == "hello" + AtomicFileWriter.write(target, "world") + assert target.read_text() == "world" + + def test_write_failure_raises_output_error(self, tmp_path): + from luke_scribe.results.store import AtomicFileWriter + + # 부모 경로가 파일이면 mkdir 실패 → OutputWriteError + blocker = tmp_path / "blocker" + blocker.write_text("not a dir") + with pytest.raises(OutputWriteError): + AtomicFileWriter.write(blocker / "x.txt", "data")