diff --git a/.gitignore b/.gitignore index e1bc4d5..2a5145b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,3 @@ -.venv/ -__pycache__/ -*.pyc .DS_Store frontend/node_modules/ frontend/dist/ -uv.lock diff --git a/README.md b/README.md index 550047c..d10056b 100644 --- a/README.md +++ b/README.md @@ -6,51 +6,34 @@ Keyboard sounds are sourced from [kbsim](https://github.com/tplai/kbsim) (MIT li ## Requirements -- [uv](https://docs.astral.sh/uv/) (Python package manager) - Node.js 18+ and npm -- **ffmpeg** on your PATH (required by pydub for audio export) - **git** (to fetch kbsim sample assets) -```bash -# macOS -brew install ffmpeg -``` - ## Setup ```bash -# Python dependencies -uv sync - # Frontend dependencies cd frontend && npm install # Download kbsim switch samples (~151 MP3 files) -./scripts/fetch_kbsim_samples.sh +cd .. && ./scripts/fetch_kbsim_samples.sh ``` ## Development -Run both servers in separate terminals: - ```bash -# Terminal 1 — FastAPI backend (audio export API) -uv run uvicorn backend.main:app --reload - -# Terminal 2 — Vite dev server (proxies /api to :8000) cd frontend && npm run dev ``` Open http://localhost:5173 -## Production-like run +## Production ```bash -cd frontend && npm run build -uv run uvicorn backend.main:app --host 127.0.0.1 --port 8000 +cd frontend && npm run build && npm run preview ``` -Open http://127.0.0.1:8000 +Or serve `frontend/dist` with any static file host. ## Usage @@ -60,8 +43,9 @@ Open http://127.0.0.1:8000 - Each physical key maps to a stable press variant (same key always sounds the same) 3. Select markers to override keys, nudge with arrow keys, or multi-select with marquee drag 4. Choose a mechanical switch sound from the dropdown (13 kbsim profiles) -5. **Save Project** / **Open Project** — JSON with version, markers, and switch setting (no video path) -6. **Export Audio** — download a WAV file of keyboard sounds only, full video length +5. Adjust preview speed (25–200%) for fast-paced content — preview only; export is always at 1× +6. **Save Project** / **Open Project** — JSON with version, markers, and switch setting (no video path) +7. **Export Audio** — download a WAV file of keyboard sounds only, full video length (rendered client-side) ### Keyboard shortcuts @@ -107,4 +91,4 @@ Keyboard sound samples from [tplai/kbsim](https://github.com/tplai/kbsim) by Tho - Frame stepping uses `video.currentTime` and may land on nearest keyframes for some MP4 encodings - Project files do not reference the video file path -- Very long videos may take longer to export via pydub in-memory overlay +- Very long videos may take longer to export client-side via OfflineAudioContext diff --git a/backend/audio_export.py b/backend/audio_export.py deleted file mode 100644 index 4b4c643..0000000 --- a/backend/audio_export.py +++ /dev/null @@ -1,51 +0,0 @@ -import io -from pathlib import Path - -from pydub import AudioSegment - -from backend.key_sound_map import ( - MANIFEST, - marker_release_time, - resolve_marker_samples, - sample_path, -) -from backend.models import ExportRequest - -SAMPLE_RATE = 48000 - - -def _load_sample(path: Path) -> AudioSegment: - if not path.exists(): - raise FileNotFoundError(f"Sample not found: {path}") - sample = AudioSegment.from_file(path) - return sample.set_frame_rate(SAMPLE_RATE).set_channels(2) - - -def export_keyboard_audio(request: ExportRequest) -> bytes: - if request.switch not in MANIFEST: - raise FileNotFoundError(f"Unknown switch: {request.switch}") - - duration_ms = int(request.duration * 1000) - base = AudioSegment.silent(duration=duration_ms, frame_rate=SAMPLE_RATE) - - for marker in sorted(request.markers, key=lambda m: m.time): - press_key, release_key = resolve_marker_samples( - request.switch, - marker.key, - marker.code, - ) - press_sample = _load_sample(sample_path(request.switch, "press", press_key)) - release_sample = _load_sample(sample_path(request.switch, "release", release_key)) - - press_ms = int(marker.time * 1000) - if press_ms < duration_ms: - base = base.overlay(press_sample, position=press_ms) - - release_time = marker_release_time(marker.time, marker.release_time) - release_ms = int(release_time * 1000) - if release_ms < duration_ms: - base = base.overlay(release_sample, position=release_ms) - - buffer = io.BytesIO() - base.export(buffer, format="wav") - return buffer.getvalue() diff --git a/backend/key_sound_map.py b/backend/key_sound_map.py deleted file mode 100644 index 5bb8741..0000000 --- a/backend/key_sound_map.py +++ /dev/null @@ -1,83 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from typing import Optional - -SAMPLES_DIR = Path(__file__).resolve().parent.parent / "assets" / "samples" / "kbsim" -MANIFEST_PATH = SAMPLES_DIR / "manifest.json" -DEFAULT_RELEASE_OFFSET = 0.08 - -SPECIAL_KEY_SAMPLES: dict[str, str] = { - "Space": "SPACE", - "Enter": "ENTER", - "Backspace": "BACKSPACE", -} - - -def _load_manifest() -> dict[str, dict[str, list[str]]]: - with MANIFEST_PATH.open(encoding="utf-8") as f: - return json.load(f) - - -MANIFEST = _load_manifest() - - -def hash_code(value: str) -> int: - """djb2 hash — mirrored in frontend/src/lib/audio/keySoundMap.ts""" - hash_val = 5381 - for ch in value: - hash_val = ((hash_val * 33) ^ ord(ch)) & 0xFFFFFFFF - return hash_val - - -def press_variant_index(code: str) -> int: - return hash_code(code) % 5 - - -def press_variant_key(code: str) -> str: - return f"GENERIC_R{press_variant_index(code)}" - - -def resolve_press_sample_key(key_label: str, code: str, available: list[str]) -> str: - special = SPECIAL_KEY_SAMPLES.get(key_label) - if special and special in available: - return special - variant = press_variant_key(code) - if variant in available: - return variant - for sample in available: - if sample.startswith("GENERIC_R"): - return sample - return available[0] - - -def resolve_release_sample_key(key_label: str, available: list[str]) -> str: - special = SPECIAL_KEY_SAMPLES.get(key_label) - if special and special in available: - return special - if "GENERIC" in available: - return "GENERIC" - return available[0] - - -def sample_path(switch_id: str, phase: str, sample_key: str) -> Path: - return SAMPLES_DIR / switch_id / phase / f"{sample_key}.mp3" - - -def marker_release_time(time: float, release_time: Optional[float]) -> float: - return release_time if release_time is not None else time + DEFAULT_RELEASE_OFFSET - - -def resolve_marker_samples( - switch_id: str, - key_label: str, - code: Optional[str], -) -> tuple[str, str]: - switch_manifest = MANIFEST[switch_id] - press_available = switch_manifest["press"] - release_available = switch_manifest["release"] - resolved_code = code or key_label - press_key = resolve_press_sample_key(key_label, resolved_code, press_available) - release_key = resolve_release_sample_key(key_label, release_available) - return press_key, release_key diff --git a/backend/main.py b/backend/main.py deleted file mode 100644 index a578dd0..0000000 --- a/backend/main.py +++ /dev/null @@ -1,56 +0,0 @@ -from pathlib import Path - -import uvicorn -from fastapi import FastAPI, HTTPException -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import Response -from fastapi.staticfiles import StaticFiles - -from backend.audio_export import export_keyboard_audio -from backend.models import ExportRequest - -ROOT = Path(__file__).resolve().parent.parent -DIST = ROOT / "frontend" / "dist" -ASSETS = ROOT / "assets" - -app = FastAPI(title="sfxkeeb") - -app.add_middleware( - CORSMiddleware, - allow_origins=["http://localhost:5173", "http://127.0.0.1:5173"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - -if ASSETS.exists(): - app.mount("/assets", StaticFiles(directory=ASSETS), name="assets") - - -@app.post("/api/export/audio") -def export_audio(request: ExportRequest) -> Response: - try: - wav_bytes = export_keyboard_audio(request) - except FileNotFoundError as exc: - raise HTTPException(status_code=500, detail=str(exc)) from exc - except Exception as exc: - raise HTTPException(status_code=500, detail=f"Export failed: {exc}") from exc - - return Response( - content=wav_bytes, - media_type="audio/wav", - headers={"Content-Disposition": 'attachment; filename="keyboard_track.wav"'}, - ) - - -@app.get("/api/health") -def health() -> dict[str, str]: - return {"status": "ok"} - - -if DIST.exists(): - app.mount("/", StaticFiles(directory=DIST, html=True), name="frontend") - - -def run() -> None: - uvicorn.run("backend.main:app", host="127.0.0.1", port=8000, reload=True) diff --git a/backend/models.py b/backend/models.py deleted file mode 100644 index 8a50c13..0000000 --- a/backend/models.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import annotations - -from typing import Literal, Optional - -from pydantic import BaseModel, Field - -SwitchType = Literal[ - "cream", - "holypanda", - "alpaca", - "turquoise", - "inkblack", - "inkred", - "mxblack", - "mxbrown", - "mxblue", - "boxnavy", - "buckling", - "alpsblue", - "topre", -] - - -class Marker(BaseModel): - id: str - time: float = Field(ge=0) - key: str - code: Optional[str] = None - release_time: Optional[float] = Field(default=None, ge=0, alias="releaseTime") - - model_config = {"populate_by_name": True} - - -class Project(BaseModel): - version: int = 2 - switch: SwitchType = "mxbrown" - markers: list[Marker] = Field(default_factory=list) - - -class ExportRequest(BaseModel): - duration: float = Field(gt=0) - switch: SwitchType = "mxbrown" - markers: list[Marker] = Field(default_factory=list) diff --git a/frontend/package.json b/frontend/package.json index 3430fcb..b186136 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -5,7 +5,7 @@ "type": "module", "scripts": { "dev": "vite", - "build": "vite build", + "build": "vite build && cp -r ../assets dist/assets", "preview": "vite preview", "check": "svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json" }, diff --git a/frontend/src/App.svelte b/frontend/src/App.svelte index 9dacc94..a4c4e04 100644 --- a/frontend/src/App.svelte +++ b/frontend/src/App.svelte @@ -191,7 +191,7 @@ } try { statusMessage = 'Exporting audio...'; - await exportAudio(app.duration); + await exportAudio(app.duration, audioEngine.getSampleCache() ?? undefined); statusMessage = 'Audio export complete'; } catch (error) { statusMessage = error instanceof Error ? error.message : 'Export failed'; diff --git a/frontend/src/lib/audio/engine.ts b/frontend/src/lib/audio/engine.ts index 8337f71..cb0c20f 100644 --- a/frontend/src/lib/audio/engine.ts +++ b/frontend/src/lib/audio/engine.ts @@ -1,42 +1,46 @@ +import { buildMarkerSchedule } from './markerSchedule'; import { - markerReleaseTime, resolvePressSampleKey, resolveReleaseSampleKey, - sampleUrl, type SoundPhase, - type SwitchManifest, } from './keySoundMap'; -import type { Marker, SwitchType } from '../types'; +import { createSampleCache, type SampleCache } from './sampleCache'; +import type { Marker } from '../types'; import { app } from '../store.svelte'; export type AudioEngine = { + /** Wire video element into the Web Audio graph and preload the active switch. */ init: (video: HTMLVideoElement) => Promise; + /** + * Schedule upcoming press/release samples synced to video playback. + * Delays are divided by playbackRate so SFX stays aligned during speed-adjusted preview. + */ reschedule: ( currentTime: number, markers: Marker[], isPlaying: boolean, playbackRate: number, ) => void; + /** Immediate press/release feedback during live annotation (not rate-scaled). */ playKey: (code: string, keyLabel: string, phase: SoundPhase) => void; preloadActiveSwitch: () => Promise; + /** Return the shared sample cache when initialized, otherwise null. */ + getSampleCache: () => SampleCache | null; destroy: () => void; }; -type Manifest = Record; - export function createAudioEngine(): AudioEngine { let ctx: AudioContext | null = null; let videoSource: MediaElementAudioSourceNode | null = null; let videoGain: GainNode | null = null; let sfxGain: GainNode | null = null; - let manifest: Manifest | null = null; - let loadedSwitch: SwitchType | null = null; - const buffers = new Map(); + let cache: SampleCache | null = null; const scheduledSources: AudioBufferSourceNode[] = []; - async function ensureContext() { + async function ensureContext(): Promise { if (!ctx) { ctx = new AudioContext(); + cache = createSampleCache(ctx); sfxGain = ctx.createGain(); sfxGain.connect(ctx.destination); } @@ -44,55 +48,12 @@ export function createAudioEngine(): AudioEngine { return ctx; } - async function ensureManifest() { - if (manifest) return manifest; - const response = await fetch('/assets/samples/kbsim/manifest.json'); - manifest = (await response.json()) as Manifest; - return manifest; + function getCache(): SampleCache { + if (!cache) throw new Error('Audio engine not initialized'); + return cache; } - function bufferKey(switchType: SwitchType, phase: SoundPhase, sampleKey: string): string { - return `${switchType}/${phase}/${sampleKey}`; - } - - async function loadBuffer( - switchType: SwitchType, - phase: SoundPhase, - sampleKey: string, - ): Promise { - const key = bufferKey(switchType, phase, sampleKey); - const cached = buffers.get(key); - if (cached) return cached; - - const audioCtx = await ensureContext(); - const response = await fetch(sampleUrl(switchType, phase, sampleKey)); - if (!response.ok) { - throw new Error(`Sample not found: ${sampleUrl(switchType, phase, sampleKey)}`); - } - const arrayBuffer = await response.arrayBuffer(); - const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer); - buffers.set(key, audioBuffer); - return audioBuffer; - } - - async function preloadSwitch(switchType: SwitchType) { - const data = await ensureManifest(); - const entry = data[switchType]; - if (!entry) throw new Error(`Unknown switch: ${switchType}`); - - const loads: Promise[] = []; - for (const sampleKey of entry.press) { - loads.push(loadBuffer(switchType, 'press', sampleKey)); - } - for (const sampleKey of entry.release) { - if (sampleKey === 'GENERIC_long') continue; - loads.push(loadBuffer(switchType, 'release', sampleKey)); - } - await Promise.all(loads); - loadedSwitch = switchType; - } - - function clearScheduled() { + function clearScheduled(): void { for (const source of scheduledSources) { try { source.stop(); @@ -103,7 +64,7 @@ export function createAudioEngine(): AudioEngine { scheduledSources.length = 0; } - function playBuffer(buffer: AudioBuffer, when = 0) { + function playBuffer(buffer: AudioBuffer, when = 0): AudioBufferSourceNode | undefined { if (!ctx || !sfxGain) return; const source = ctx.createBufferSource(); source.buffer = buffer; @@ -113,14 +74,15 @@ export function createAudioEngine(): AudioEngine { } async function resolveAndPlay( - switchType: SwitchType, code: string, keyLabel: string, phase: SoundPhase, when = 0, - ) { - const data = await ensureManifest(); - const entry = data[switchType]; + ): Promise { + const sampleCache = getCache(); + const switchType = app.activeSwitch; + const manifest = await sampleCache.ensureManifest(); + const entry = manifest[switchType]; if (!entry) return null; const sampleKey = @@ -128,13 +90,11 @@ export function createAudioEngine(): AudioEngine { ? resolvePressSampleKey(keyLabel, code, entry.press) : resolveReleaseSampleKey(keyLabel, entry.release); - if (phase === 'release' && sampleKey === 'GENERIC_long') return null; - - if (loadedSwitch !== switchType) { - await preloadSwitch(switchType); + if (sampleCache.getLoadedSwitch() !== switchType) { + await sampleCache.preloadSwitch(switchType); } - const buffer = await loadBuffer(switchType, phase, sampleKey); + const buffer = await sampleCache.loadBuffer(switchType, phase, sampleKey); return playBuffer(buffer, when); } @@ -147,8 +107,7 @@ export function createAudioEngine(): AudioEngine { videoSource.connect(videoGain); videoGain.connect(audioCtx.destination); } - await ensureManifest(); - await preloadSwitch(app.activeSwitch); + await getCache().preloadSwitch(app.activeSwitch); }, reschedule(time: number, markers: Marker[], playing: boolean, playbackRate: number) { @@ -164,43 +123,41 @@ export function createAudioEngine(): AudioEngine { const switchType = app.activeSwitch; void (async () => { - const data = await ensureManifest(); - const entry = data[switchType]; + const sampleCache = getCache(); + const manifest = await sampleCache.ensureManifest(); + const entry = manifest[switchType]; if (!entry) return; - if (loadedSwitch !== switchType) { - await preloadSwitch(switchType); + + if (sampleCache.getLoadedSwitch() !== switchType) { + await sampleCache.preloadSwitch(switchType); } - for (const marker of markers) { - const code = marker.code ?? marker.key; - const pressKey = resolvePressSampleKey(marker.key, code, entry.press); - const releaseKey = resolveReleaseSampleKey(marker.key, entry.release); - const releaseTime = markerReleaseTime(marker); + const schedule = buildMarkerSchedule(markers, entry); + const windowStart = time - 0.05; - if (marker.time >= time - 0.05) { - const pressDelay = (marker.time - time) / rate; - const pressBuffer = await loadBuffer(switchType, 'press', pressKey); - const pressSource = playBuffer(pressBuffer, startAt + Math.max(0, pressDelay)); - if (pressSource) scheduledSources.push(pressSource); - } + for (const event of schedule) { + if (event.time < windowStart) continue; + if (event.phase === 'release' && event.sampleKey === 'GENERIC_long') continue; - if (releaseTime >= time - 0.05) { - const releaseDelay = (releaseTime - time) / rate; - const releaseBuffer = await loadBuffer(switchType, 'release', releaseKey); - const releaseSource = playBuffer(releaseBuffer, startAt + Math.max(0, releaseDelay)); - if (releaseSource) scheduledSources.push(releaseSource); - } + const delay = (event.time - time) / rate; + const buffer = await sampleCache.loadBuffer(switchType, event.phase, event.sampleKey); + const source = playBuffer(buffer, startAt + Math.max(0, delay)); + if (source) scheduledSources.push(source); } })(); }, playKey(code: string, keyLabel: string, phase: SoundPhase) { - void resolveAndPlay(app.activeSwitch, code, keyLabel, phase); + void resolveAndPlay(code, keyLabel, phase); }, async preloadActiveSwitch() { - await ensureManifest(); - await preloadSwitch(app.activeSwitch); + await ensureContext(); + await getCache().preloadSwitch(app.activeSwitch); + }, + + getSampleCache() { + return cache; }, destroy() { @@ -213,9 +170,7 @@ export function createAudioEngine(): AudioEngine { videoSource = null; videoGain = null; sfxGain = null; - manifest = null; - loadedSwitch = null; - buffers.clear(); + cache = null; }, }; } diff --git a/frontend/src/lib/audio/export.ts b/frontend/src/lib/audio/export.ts new file mode 100644 index 0000000..ea78bd5 --- /dev/null +++ b/frontend/src/lib/audio/export.ts @@ -0,0 +1,57 @@ +import { buildMarkerSchedule } from './markerSchedule'; +import { createSampleCache, type SampleCache } from './sampleCache'; +import { audioBufferToWavBlob, EXPORT_SAMPLE_RATE } from './wav'; +import type { Marker, SwitchType } from '../types'; + +export type RenderKeyboardTrackOptions = { + /** Total track length in video-timeline seconds. */ + duration: number; + switchType: SwitchType; + markers: Marker[]; + /** Reuse decoded buffers from the live preview engine when available. */ + cache?: SampleCache; +}; + +/** + * Render a keyboard-only WAV track at 1× using OfflineAudioContext. + * Marker times are absolute video seconds — preview speed is not applied. + */ +export async function renderKeyboardTrack(options: RenderKeyboardTrackOptions): Promise { + const { duration, switchType, markers } = options; + + let decodeContext: AudioContext | null = null; + const cache = options.cache ?? (() => { + decodeContext = new AudioContext(); + return createSampleCache(decodeContext); + })(); + + try { + const manifest = await cache.ensureManifest(); + const entry = manifest[switchType]; + if (!entry) throw new Error(`Unknown switch: ${switchType}`); + + if (cache.getLoadedSwitch() !== switchType) { + await cache.preloadSwitch(switchType); + } + + const frameCount = Math.ceil(duration * EXPORT_SAMPLE_RATE); + const offline = new OfflineAudioContext(2, frameCount, EXPORT_SAMPLE_RATE); + const schedule = buildMarkerSchedule(markers, entry); + + for (const event of schedule) { + if (event.time >= duration) continue; + if (event.phase === 'release' && event.sampleKey === 'GENERIC_long') continue; + + const buffer = await cache.loadBuffer(switchType, event.phase, event.sampleKey); + const source = offline.createBufferSource(); + source.buffer = buffer; + source.connect(offline.destination); + source.start(event.time); + } + + const rendered = await offline.startRendering(); + return audioBufferToWavBlob(rendered, EXPORT_SAMPLE_RATE); + } finally { + if (decodeContext) await decodeContext.close(); + } +} diff --git a/frontend/src/lib/audio/keySoundMap.ts b/frontend/src/lib/audio/keySoundMap.ts index ca0d5ac..ddae738 100644 --- a/frontend/src/lib/audio/keySoundMap.ts +++ b/frontend/src/lib/audio/keySoundMap.ts @@ -25,7 +25,7 @@ const SPECIAL_KEY_SAMPLES: Record = { Backspace: 'BACKSPACE', }; -/** djb2 hash — mirrored in backend/key_sound_map.py */ +/** djb2 hash for stable per-key press variant selection. */ export function hashCode(value: string): number { let hash = 5381; for (let i = 0; i < value.length; i++) { @@ -42,6 +42,7 @@ export function pressVariantKey(code: string): PressSampleKey { return `GENERIC_R${pressVariantIndex(code)}` as PressSampleKey; } +/** Pick a press sample key from special-key map or GENERIC_R0–R4 variant. */ export function resolvePressSampleKey( keyLabel: string, code: string, @@ -57,12 +58,13 @@ export function resolvePressSampleKey( return (generic ?? available[0]) as PressSampleKey; } +/** Pick a release sample key, preferring special keys then GENERIC. */ export function resolveReleaseSampleKey( keyLabel: string, available: string[], ): ReleaseSampleKey { const special = SPECIAL_KEY_SAMPLES[keyLabel]; - if (special && available.includes(special)) return special; + if (special && available.includes(special)) return special as ReleaseSampleKey; if (available.includes('GENERIC')) return 'GENERIC'; return (available[0] ?? 'GENERIC') as ReleaseSampleKey; } @@ -75,6 +77,7 @@ export function sampleUrl( return `/assets/samples/kbsim/${switchType}/${phase}/${sampleKey}.mp3`; } +/** Keyup timestamp; defaults to press time + 80 ms when not recorded. */ export function markerReleaseTime(marker: { time: number; releaseTime?: number; diff --git a/frontend/src/lib/audio/markerSchedule.ts b/frontend/src/lib/audio/markerSchedule.ts new file mode 100644 index 0000000..fc0f473 --- /dev/null +++ b/frontend/src/lib/audio/markerSchedule.ts @@ -0,0 +1,43 @@ +import { + markerReleaseTime, + resolvePressSampleKey, + resolveReleaseSampleKey, + type SoundPhase, + type SwitchManifest, +} from './keySoundMap'; +import type { Marker } from '../types'; + +/** A single press or release sample placed on the video timeline (seconds, 1×). */ +export type ScheduledEvent = { + phase: SoundPhase; + sampleKey: string; + /** Video-timeline seconds — never adjusted for preview playback speed. */ + time: number; +}; + +/** + * Build a sorted list of press/release events from markers. + * Times are absolute video coordinates used by both preview scheduling and 1× export. + */ +export function buildMarkerSchedule( + markers: Marker[], + switchEntry: SwitchManifest, +): ScheduledEvent[] { + const events: ScheduledEvent[] = []; + + for (const marker of markers) { + const code = marker.code ?? marker.key; + events.push({ + phase: 'press', + sampleKey: resolvePressSampleKey(marker.key, code, switchEntry.press), + time: marker.time, + }); + events.push({ + phase: 'release', + sampleKey: resolveReleaseSampleKey(marker.key, switchEntry.release), + time: markerReleaseTime(marker), + }); + } + + return events.sort((a, b) => a.time - b.time); +} diff --git a/frontend/src/lib/audio/sampleCache.ts b/frontend/src/lib/audio/sampleCache.ts new file mode 100644 index 0000000..63a199f --- /dev/null +++ b/frontend/src/lib/audio/sampleCache.ts @@ -0,0 +1,82 @@ +import { sampleUrl, type SoundPhase, type SwitchManifest } from './keySoundMap'; +import type { SwitchType } from '../types'; + +export type Manifest = Record; + +/** Cache key format: `{switchType}/{phase}/{sampleKey}` */ +function bufferKey(switchType: SwitchType, phase: SoundPhase, sampleKey: string): string { + return `${switchType}/${phase}/${sampleKey}`; +} + +export type SampleCache = { + ensureManifest: () => Promise; + loadBuffer: ( + switchType: SwitchType, + phase: SoundPhase, + sampleKey: string, + ) => Promise; + preloadSwitch: (switchType: SwitchType) => Promise; + getLoadedSwitch: () => SwitchType | null; +}; + +/** + * Shared MP3 fetch + decode cache used by live preview and offline export. + * Requires a BaseAudioContext for `decodeAudioData` (live AudioContext or throwaway decode context). + */ +export function createSampleCache(decodeContext: BaseAudioContext): SampleCache { + let manifest: Manifest | null = null; + let loadedSwitch: SwitchType | null = null; + const buffers = new Map(); + + async function ensureManifest(): Promise { + if (manifest) return manifest; + const response = await fetch('/assets/samples/kbsim/manifest.json'); + if (!response.ok) throw new Error('Failed to load sample manifest'); + manifest = (await response.json()) as Manifest; + return manifest; + } + + async function loadBuffer( + switchType: SwitchType, + phase: SoundPhase, + sampleKey: string, + ): Promise { + const key = bufferKey(switchType, phase, sampleKey); + const cached = buffers.get(key); + if (cached) return cached; + + const response = await fetch(sampleUrl(switchType, phase, sampleKey)); + if (!response.ok) { + throw new Error(`Sample not found: ${sampleUrl(switchType, phase, sampleKey)}`); + } + const arrayBuffer = await response.arrayBuffer(); + const audioBuffer = await decodeContext.decodeAudioData(arrayBuffer); + buffers.set(key, audioBuffer); + return audioBuffer; + } + + /** Preload all press samples and release samples (skips GENERIC_long) for a switch. */ + async function preloadSwitch(switchType: SwitchType): Promise { + const data = await ensureManifest(); + const entry = data[switchType]; + if (!entry) throw new Error(`Unknown switch: ${switchType}`); + + const loads: Promise[] = []; + for (const sampleKey of entry.press) { + loads.push(loadBuffer(switchType, 'press', sampleKey)); + } + for (const sampleKey of entry.release) { + if (sampleKey === 'GENERIC_long') continue; + loads.push(loadBuffer(switchType, 'release', sampleKey)); + } + await Promise.all(loads); + loadedSwitch = switchType; + } + + return { + ensureManifest, + loadBuffer, + preloadSwitch, + getLoadedSwitch: () => loadedSwitch, + }; +} diff --git a/frontend/src/lib/audio/wav.ts b/frontend/src/lib/audio/wav.ts new file mode 100644 index 0000000..1356bc5 --- /dev/null +++ b/frontend/src/lib/audio/wav.ts @@ -0,0 +1,62 @@ +/** Target sample rate for exported WAV files (matches former backend output). */ +export const EXPORT_SAMPLE_RATE = 48000; + +/** + * Encode an AudioBuffer as a 16-bit PCM stereo WAV Blob. + * Float samples are clamped to [-1, 1] before conversion to int16. + */ +export function audioBufferToWavBlob( + buffer: AudioBuffer, + sampleRate = EXPORT_SAMPLE_RATE, +): Blob { + const numChannels = buffer.numberOfChannels; + const length = buffer.length; + const bytesPerSample = 2; + const blockAlign = numChannels * bytesPerSample; + const dataSize = length * blockAlign; + const headerSize = 44; + const arrayBuffer = new ArrayBuffer(headerSize + dataSize); + const view = new DataView(arrayBuffer); + + // RIFF header + writeString(view, 0, 'RIFF'); + view.setUint32(4, 36 + dataSize, true); + writeString(view, 8, 'WAVE'); + + // fmt chunk (PCM, 16-bit) + writeString(view, 12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, numChannels, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * blockAlign, true); + view.setUint16(32, blockAlign, true); + view.setUint16(34, 16, true); + + // data chunk + writeString(view, 36, 'data'); + view.setUint32(40, dataSize, true); + + // Interleave channels and write PCM samples + const channels: Float32Array[] = []; + for (let ch = 0; ch < numChannels; ch++) { + channels.push(buffer.getChannelData(ch)); + } + + let offset = headerSize; + for (let i = 0; i < length; i++) { + for (let ch = 0; ch < numChannels; ch++) { + const sample = Math.max(-1, Math.min(1, channels[ch][i])); + view.setInt16(offset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true); + offset += 2; + } + } + + return new Blob([arrayBuffer], { type: 'audio/wav' }); +} + +function writeString(view: DataView, offset: number, value: string): void { + for (let i = 0; i < value.length; i++) { + view.setUint8(offset + i, value.charCodeAt(i)); + } +} diff --git a/frontend/src/lib/project.ts b/frontend/src/lib/project.ts index 13f358f..0f5bb8d 100644 --- a/frontend/src/lib/project.ts +++ b/frontend/src/lib/project.ts @@ -1,3 +1,5 @@ +import { renderKeyboardTrack } from './audio/export'; +import type { SampleCache } from './audio/sampleCache'; import type { LegacySwitchType, Marker, Project, SwitchType } from './types'; import { app, setActiveSwitch, setMarkers, setSelectedIds } from './store.svelte'; @@ -50,24 +52,19 @@ export function loadProject(data: Project & { version?: number; switch?: string setSelectedIds(new Set()); } -export async function exportAudio(duration: number) { +/** + * Export keyboard SFX only as a WAV file (no video audio). + * Rendered client-side at 1× via OfflineAudioContext. + */ +export async function exportAudio(duration: number, cache?: SampleCache) { const project = buildProject(); - const response = await fetch('/api/export/audio', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - duration, - switch: project.switch, - markers: project.markers, - }), + const blob = await renderKeyboardTrack({ + duration, + switchType: project.switch, + markers: project.markers, + cache, }); - if (!response.ok) { - const detail = await response.text(); - throw new Error(detail || 'Audio export failed'); - } - - const blob = await response.blob(); const url = URL.createObjectURL(blob); const anchor = document.createElement('a'); anchor.href = url; diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 6139ab3..d867977 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -1,12 +1,58 @@ -import { defineConfig } from 'vite' +import { createReadStream, existsSync, statSync } from 'node:fs' +import { join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import type { IncomingMessage, ServerResponse } from 'node:http' +import { defineConfig, type Connect, type Plugin } from 'vite' import { svelte } from '@sveltejs/vite-plugin-svelte' +const repoRoot = resolve(fileURLToPath(import.meta.url), '../..') +const assetsDir = resolve(repoRoot, 'assets') + +function serveStatic(root: string): Connect.NextHandleFunction { + return (req, res, next) => { + const url = (req.url ?? '/').split('?')[0] + const filePath = resolve(join(root, url)) + if (!filePath.startsWith(root) || !existsSync(filePath)) { + next() + return + } + const stat = statSync(filePath) + if (!stat.isFile()) { + next() + return + } + serveFile(filePath, res) + } +} + +function serveFile(filePath: string, res: ServerResponse): void { + const ext = filePath.split('.').pop()?.toLowerCase() + const types: Record = { + json: 'application/json', + mp3: 'audio/mpeg', + } + res.setHeader('Content-Type', types[ext ?? ''] ?? 'application/octet-stream') + createReadStream(filePath).pipe(res) +} + +/** Serve repo-level assets/ at /assets in dev and preview. */ +function serveRepoAssets(): Plugin { + return { + name: 'serve-repo-assets', + configureServer(server) { + server.middlewares.use('/assets', serveStatic(assetsDir)) + }, + configurePreviewServer(server) { + server.middlewares.use('/assets', serveStatic(assetsDir)) + }, + } +} + export default defineConfig({ - plugins: [svelte()], + plugins: [svelte(), serveRepoAssets()], server: { - proxy: { - '/api': 'http://127.0.0.1:8000', - '/assets': 'http://127.0.0.1:8000', + fs: { + allow: [repoRoot], }, }, }) diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index eafedef..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,16 +0,0 @@ -[project] -name = "sfxkeeb" -version = "0.1.0" -description = "Video keyboard SFX annotator with timeline editing and audio export" -readme = "README.md" -requires-python = ">=3.9" -dependencies = [ - "fastapi>=0.128.8", - "pydantic>=2.13.4", - "pydub>=0.25.1", - "python-multipart>=0.0.20", - "uvicorn[standard]>=0.39.0", -] - -[project.scripts] -sfxkeeb = "backend.main:run"