lukehemmin
7327145d7a
Batch+realtime transcription API: faster-whisper engine w/ EngineOwner (single GPU owner, OOM downgrade chain, persisted attempted_profiles), hardware-adaptive device manager (T0-T3 VRAM tiers), Redis/in-proc job queue w/ leases + crash recovery, postprocess (glossary/rules/LLM w/ egress guard), privacy-first result store (UUID keys, source deleted after transcribe), retention sweeper, API-key auth (HMAC digests, scopes, job ownership), WebSocket realtime lane (LocalAgreement), CLI (detect/transcribe/bench/serve/key), Docker, benchmark runner. 127 mock-based tests pass; ruff clean. Includes verification checklist and autoplan review notes.
256 lines
8.5 KiB
Python
256 lines
8.5 KiB
Python
"""AudioIngestor — 입력 검증 + ffmpeg 16kHz mono 정규화 (스트리밍).
|
|
|
|
계약 (plan §3.7b/§6.1 + design doc §3):
|
|
- 확장자가 아니라 **ffprobe**로 형식 검증.
|
|
- 60분/1GB(설계문서 v0.1) 또는 4h/2GB(플랫폼, config) 상한 — 초과 시
|
|
``unsupported_input_envelope``.
|
|
- ffmpeg는 전체 배열을 메모리에 올리지 않고 **파일로 스트리밍**.
|
|
- 모든 종료 경로에서 임시 파일 정리 (``finally``), 원본은 읽기 전용.
|
|
- 취소/오류 시 ffmpeg 프로세스 그룹 kill + reap (Eng 리뷰 P1).
|
|
- ffprobe/ffmpeg timeout: 기본 60s, CLI 옵션으로만 상향.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import tempfile
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
from ..config import Settings
|
|
from ..errors import AudioProbeFailed, InvalidInput, UnsupportedInputEnvelope
|
|
from ..results.models import NormalizedAudio
|
|
|
|
|
|
@dataclass
|
|
class ProbeResult:
|
|
duration_sec: float | None
|
|
codec: str | None
|
|
size_bytes: int
|
|
has_audio: bool = True
|
|
|
|
|
|
@dataclass
|
|
class IngestResult:
|
|
normalized_path: str
|
|
normalized: NormalizedAudio
|
|
probe: ProbeResult
|
|
temp_dir: str
|
|
cleanup: callable = field(repr=False)
|
|
|
|
def close(self) -> None:
|
|
self.cleanup()
|
|
|
|
|
|
class AudioIngestor:
|
|
def __init__(
|
|
self,
|
|
settings: Settings | None = None,
|
|
*,
|
|
ffprobe_bin: str = "ffprobe",
|
|
ffmpeg_bin: str = "ffmpeg",
|
|
probe_timeout: int = 60,
|
|
) -> None:
|
|
self.settings = settings or Settings()
|
|
self._ffprobe = ffprobe_bin
|
|
self._ffmpeg = ffmpeg_bin
|
|
self._probe_timeout = probe_timeout
|
|
|
|
def probe(self, source: Path) -> ProbeResult:
|
|
if not source.exists():
|
|
raise InvalidInput(f"파일이 존재하지 않습니다: {source}")
|
|
if source.stat().st_size > self.settings.max_upload_bytes:
|
|
raise UnsupportedInputEnvelope(
|
|
f"파일 크기 {source.stat().st_size} 바이트가 상한 "
|
|
f"{self.settings.max_upload_bytes} 바이트를 초과합니다"
|
|
)
|
|
cmd = [
|
|
self._ffprobe,
|
|
"-v",
|
|
"error",
|
|
"-show_entries",
|
|
"format=duration:stream=codec_type,codec_name",
|
|
"-of",
|
|
"json",
|
|
str(source),
|
|
]
|
|
try:
|
|
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=self._probe_timeout)
|
|
except FileNotFoundError as exc:
|
|
raise AudioProbeFailed(
|
|
f"ffprobe를 찾을 수 없습니다 ({self._ffprobe}). ffmpeg를 설치하세요: "
|
|
"apt install ffmpeg / brew install ffmpeg"
|
|
) from exc
|
|
except subprocess.TimeoutExpired as exc:
|
|
raise AudioProbeFailed("ffprobe timeout (60s)") from exc
|
|
|
|
if proc.returncode != 0:
|
|
raise AudioProbeFailed(f"입력 오디오를 해석할 수 없습니다. {proc.stderr.strip()[:300]}")
|
|
import json
|
|
|
|
try:
|
|
data = json.loads(proc.stdout or "{}")
|
|
except Exception as exc:
|
|
raise AudioProbeFailed(f"ffprobe 출력 파싱 실패: {exc}") from exc
|
|
|
|
streams = data.get("streams", [])
|
|
audio_streams = [s for s in streams if s.get("codec_type") == "audio"]
|
|
if not audio_streams:
|
|
raise AudioProbeFailed("입력에 오디오 스트림이 없습니다")
|
|
|
|
duration = None
|
|
fmt = data.get("format") or {}
|
|
if fmt.get("duration"):
|
|
try:
|
|
duration = float(fmt["duration"])
|
|
except (TypeError, ValueError):
|
|
duration = None
|
|
if duration is not None and duration > self.settings.max_duration_sec:
|
|
raise UnsupportedInputEnvelope(
|
|
f"오디오 길이 {duration:.0f}초가 상한 {self.settings.max_duration_sec}초를 초과합니다"
|
|
)
|
|
return ProbeResult(
|
|
duration_sec=duration,
|
|
codec=audio_streams[0].get("codec_name"),
|
|
size_bytes=source.stat().st_size,
|
|
)
|
|
|
|
def ingest(self, source: Path, *, should_cancel: callable | None = None) -> IngestResult:
|
|
"""검증 + ffmpeg로 16kHz mono PCM s16 WAV 정규화."""
|
|
if should_cancel and should_cancel():
|
|
from ..errors import CancelledError
|
|
|
|
raise CancelledError("취소됨")
|
|
probe = self.probe(source)
|
|
|
|
temp_dir = tempfile.mkdtemp(prefix="luke-scribe-ingest-")
|
|
out_path = Path(temp_dir) / "normalized.wav"
|
|
|
|
def cleanup() -> None:
|
|
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
|
|
try:
|
|
if should_cancel and should_cancel():
|
|
from ..errors import CancelledError
|
|
|
|
raise CancelledError("취소됨")
|
|
cmd = [
|
|
self._ffmpeg,
|
|
"-v",
|
|
"error",
|
|
"-y",
|
|
"-i",
|
|
str(source),
|
|
"-ac",
|
|
"1",
|
|
"-ar",
|
|
"16000",
|
|
"-c:a",
|
|
"pcm_s16le",
|
|
"-f",
|
|
"wav",
|
|
str(out_path),
|
|
]
|
|
proc = subprocess.Popen(
|
|
cmd,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
start_new_session=True, # 프로세스 그룹 (kill/reap 용이)
|
|
)
|
|
# stdout/stderr 병렬 drain (ffmpeg stderr 블로킹 방지)
|
|
assert proc.stdout is not None and proc.stderr is not None
|
|
out_thread = self._drain(proc.stdout)
|
|
err_data = self._read_all(proc.stderr)
|
|
|
|
while True:
|
|
try:
|
|
rc = proc.wait(timeout=0.5)
|
|
break
|
|
except subprocess.TimeoutExpired:
|
|
if should_cancel and should_cancel():
|
|
self._kill_group(proc)
|
|
from ..errors import CancelledError
|
|
|
|
raise CancelledError("인제스트 중 취소") from None
|
|
out_thread.join(timeout=5)
|
|
if rc != 0:
|
|
raise AudioProbeFailed(
|
|
f"ffmpeg 정규화 실패 (exit {rc}): {err_data.decode(errors='replace')[:300]}"
|
|
)
|
|
if not out_path.exists() or out_path.stat().st_size == 0:
|
|
raise AudioProbeFailed("ffmpeg가 출력 WAV를 생성하지 못했습니다")
|
|
|
|
duration = probe.duration_sec
|
|
if duration is None:
|
|
# ffprobe가 몰랐던 경우: 파생 파일로 재측정
|
|
duration = self._duration_of(out_path) or 0.0
|
|
return IngestResult(
|
|
normalized_path=str(out_path),
|
|
normalized=NormalizedAudio(
|
|
duration_sec=max(duration, 0.001),
|
|
audio_format="pcm_s16le",
|
|
sample_rate=16000,
|
|
channels=1,
|
|
),
|
|
probe=probe,
|
|
temp_dir=temp_dir,
|
|
cleanup=cleanup,
|
|
)
|
|
except BaseException:
|
|
cleanup()
|
|
raise
|
|
|
|
def _duration_of(self, wav: Path) -> float | None:
|
|
"""ffprobe로 파생 WAV 길이 재측정."""
|
|
try:
|
|
out = subprocess.run(
|
|
[
|
|
self._ffprobe,
|
|
"-v",
|
|
"error",
|
|
"-show_entries",
|
|
"format=duration",
|
|
"-of",
|
|
"json",
|
|
str(wav),
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=10,
|
|
)
|
|
import json
|
|
|
|
d = json.loads(out.stdout or "{}").get("format", {}).get("duration")
|
|
return float(d) if d else None
|
|
except Exception:
|
|
return None
|
|
|
|
@staticmethod
|
|
def _drain(stream) -> os._wrap_close:
|
|
import threading
|
|
|
|
def _read() -> None:
|
|
while stream.read(65536):
|
|
pass
|
|
|
|
t = threading.Thread(target=_read, daemon=True)
|
|
t.start()
|
|
return t # type: ignore[return-value]
|
|
|
|
@staticmethod
|
|
def _read_all(stream) -> bytes:
|
|
return stream.read()
|
|
|
|
@staticmethod
|
|
def _kill_group(proc: subprocess.Popen) -> None:
|
|
try:
|
|
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
|
except (ProcessLookupError, PermissionError):
|
|
try:
|
|
proc.kill()
|
|
except Exception:
|
|
pass
|