drop python backend layer

This commit is contained in:
2026-06-09 00:42:41 -07:00
parent 2bfa1e9269
commit 21a56e187e
17 changed files with 373 additions and 397 deletions
+1 -1
View File
@@ -5,7 +5,7 @@
"type": "module",
"scripts": {
"dev": "vite",
"build": "vite build",
"build": "vite build && cp -r ../assets dist/assets",
"preview": "vite preview",
"check": "svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json"
},
+1 -1
View File
@@ -191,7 +191,7 @@
}
try {
statusMessage = 'Exporting audio...';
await exportAudio(app.duration);
await exportAudio(app.duration, audioEngine.getSampleCache() ?? undefined);
statusMessage = 'Audio export complete';
} catch (error) {
statusMessage = error instanceof Error ? error.message : 'Export failed';
+51 -96
View File
@@ -1,42 +1,46 @@
import { buildMarkerSchedule } from './markerSchedule';
import {
markerReleaseTime,
resolvePressSampleKey,
resolveReleaseSampleKey,
sampleUrl,
type SoundPhase,
type SwitchManifest,
} from './keySoundMap';
import type { Marker, SwitchType } from '../types';
import { createSampleCache, type SampleCache } from './sampleCache';
import type { Marker } from '../types';
import { app } from '../store.svelte';
export type AudioEngine = {
/** Wire video element into the Web Audio graph and preload the active switch. */
init: (video: HTMLVideoElement) => Promise<void>;
/**
* Schedule upcoming press/release samples synced to video playback.
* Delays are divided by playbackRate so SFX stays aligned during speed-adjusted preview.
*/
reschedule: (
currentTime: number,
markers: Marker[],
isPlaying: boolean,
playbackRate: number,
) => void;
/** Immediate press/release feedback during live annotation (not rate-scaled). */
playKey: (code: string, keyLabel: string, phase: SoundPhase) => void;
preloadActiveSwitch: () => Promise<void>;
/** Return the shared sample cache when initialized, otherwise null. */
getSampleCache: () => SampleCache | null;
destroy: () => void;
};
type Manifest = Record<string, SwitchManifest>;
export function createAudioEngine(): AudioEngine {
let ctx: AudioContext | null = null;
let videoSource: MediaElementAudioSourceNode | null = null;
let videoGain: GainNode | null = null;
let sfxGain: GainNode | null = null;
let manifest: Manifest | null = null;
let loadedSwitch: SwitchType | null = null;
const buffers = new Map<string, AudioBuffer>();
let cache: SampleCache | null = null;
const scheduledSources: AudioBufferSourceNode[] = [];
async function ensureContext() {
async function ensureContext(): Promise<AudioContext> {
if (!ctx) {
ctx = new AudioContext();
cache = createSampleCache(ctx);
sfxGain = ctx.createGain();
sfxGain.connect(ctx.destination);
}
@@ -44,55 +48,12 @@ export function createAudioEngine(): AudioEngine {
return ctx;
}
async function ensureManifest() {
if (manifest) return manifest;
const response = await fetch('/assets/samples/kbsim/manifest.json');
manifest = (await response.json()) as Manifest;
return manifest;
function getCache(): SampleCache {
if (!cache) throw new Error('Audio engine not initialized');
return cache;
}
function bufferKey(switchType: SwitchType, phase: SoundPhase, sampleKey: string): string {
return `${switchType}/${phase}/${sampleKey}`;
}
async function loadBuffer(
switchType: SwitchType,
phase: SoundPhase,
sampleKey: string,
): Promise<AudioBuffer> {
const key = bufferKey(switchType, phase, sampleKey);
const cached = buffers.get(key);
if (cached) return cached;
const audioCtx = await ensureContext();
const response = await fetch(sampleUrl(switchType, phase, sampleKey));
if (!response.ok) {
throw new Error(`Sample not found: ${sampleUrl(switchType, phase, sampleKey)}`);
}
const arrayBuffer = await response.arrayBuffer();
const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer);
buffers.set(key, audioBuffer);
return audioBuffer;
}
async function preloadSwitch(switchType: SwitchType) {
const data = await ensureManifest();
const entry = data[switchType];
if (!entry) throw new Error(`Unknown switch: ${switchType}`);
const loads: Promise<AudioBuffer>[] = [];
for (const sampleKey of entry.press) {
loads.push(loadBuffer(switchType, 'press', sampleKey));
}
for (const sampleKey of entry.release) {
if (sampleKey === 'GENERIC_long') continue;
loads.push(loadBuffer(switchType, 'release', sampleKey));
}
await Promise.all(loads);
loadedSwitch = switchType;
}
function clearScheduled() {
function clearScheduled(): void {
for (const source of scheduledSources) {
try {
source.stop();
@@ -103,7 +64,7 @@ export function createAudioEngine(): AudioEngine {
scheduledSources.length = 0;
}
function playBuffer(buffer: AudioBuffer, when = 0) {
function playBuffer(buffer: AudioBuffer, when = 0): AudioBufferSourceNode | undefined {
if (!ctx || !sfxGain) return;
const source = ctx.createBufferSource();
source.buffer = buffer;
@@ -113,14 +74,15 @@ export function createAudioEngine(): AudioEngine {
}
async function resolveAndPlay(
switchType: SwitchType,
code: string,
keyLabel: string,
phase: SoundPhase,
when = 0,
) {
const data = await ensureManifest();
const entry = data[switchType];
): Promise<AudioBufferSourceNode | null | undefined> {
const sampleCache = getCache();
const switchType = app.activeSwitch;
const manifest = await sampleCache.ensureManifest();
const entry = manifest[switchType];
if (!entry) return null;
const sampleKey =
@@ -128,13 +90,11 @@ export function createAudioEngine(): AudioEngine {
? resolvePressSampleKey(keyLabel, code, entry.press)
: resolveReleaseSampleKey(keyLabel, entry.release);
if (phase === 'release' && sampleKey === 'GENERIC_long') return null;
if (loadedSwitch !== switchType) {
await preloadSwitch(switchType);
if (sampleCache.getLoadedSwitch() !== switchType) {
await sampleCache.preloadSwitch(switchType);
}
const buffer = await loadBuffer(switchType, phase, sampleKey);
const buffer = await sampleCache.loadBuffer(switchType, phase, sampleKey);
return playBuffer(buffer, when);
}
@@ -147,8 +107,7 @@ export function createAudioEngine(): AudioEngine {
videoSource.connect(videoGain);
videoGain.connect(audioCtx.destination);
}
await ensureManifest();
await preloadSwitch(app.activeSwitch);
await getCache().preloadSwitch(app.activeSwitch);
},
reschedule(time: number, markers: Marker[], playing: boolean, playbackRate: number) {
@@ -164,43 +123,41 @@ export function createAudioEngine(): AudioEngine {
const switchType = app.activeSwitch;
void (async () => {
const data = await ensureManifest();
const entry = data[switchType];
const sampleCache = getCache();
const manifest = await sampleCache.ensureManifest();
const entry = manifest[switchType];
if (!entry) return;
if (loadedSwitch !== switchType) {
await preloadSwitch(switchType);
if (sampleCache.getLoadedSwitch() !== switchType) {
await sampleCache.preloadSwitch(switchType);
}
for (const marker of markers) {
const code = marker.code ?? marker.key;
const pressKey = resolvePressSampleKey(marker.key, code, entry.press);
const releaseKey = resolveReleaseSampleKey(marker.key, entry.release);
const releaseTime = markerReleaseTime(marker);
const schedule = buildMarkerSchedule(markers, entry);
const windowStart = time - 0.05;
if (marker.time >= time - 0.05) {
const pressDelay = (marker.time - time) / rate;
const pressBuffer = await loadBuffer(switchType, 'press', pressKey);
const pressSource = playBuffer(pressBuffer, startAt + Math.max(0, pressDelay));
if (pressSource) scheduledSources.push(pressSource);
}
for (const event of schedule) {
if (event.time < windowStart) continue;
if (event.phase === 'release' && event.sampleKey === 'GENERIC_long') continue;
if (releaseTime >= time - 0.05) {
const releaseDelay = (releaseTime - time) / rate;
const releaseBuffer = await loadBuffer(switchType, 'release', releaseKey);
const releaseSource = playBuffer(releaseBuffer, startAt + Math.max(0, releaseDelay));
if (releaseSource) scheduledSources.push(releaseSource);
}
const delay = (event.time - time) / rate;
const buffer = await sampleCache.loadBuffer(switchType, event.phase, event.sampleKey);
const source = playBuffer(buffer, startAt + Math.max(0, delay));
if (source) scheduledSources.push(source);
}
})();
},
playKey(code: string, keyLabel: string, phase: SoundPhase) {
void resolveAndPlay(app.activeSwitch, code, keyLabel, phase);
void resolveAndPlay(code, keyLabel, phase);
},
async preloadActiveSwitch() {
await ensureManifest();
await preloadSwitch(app.activeSwitch);
await ensureContext();
await getCache().preloadSwitch(app.activeSwitch);
},
getSampleCache() {
return cache;
},
destroy() {
@@ -213,9 +170,7 @@ export function createAudioEngine(): AudioEngine {
videoSource = null;
videoGain = null;
sfxGain = null;
manifest = null;
loadedSwitch = null;
buffers.clear();
cache = null;
},
};
}
+57
View File
@@ -0,0 +1,57 @@
import { buildMarkerSchedule } from './markerSchedule';
import { createSampleCache, type SampleCache } from './sampleCache';
import { audioBufferToWavBlob, EXPORT_SAMPLE_RATE } from './wav';
import type { Marker, SwitchType } from '../types';
export type RenderKeyboardTrackOptions = {
/** Total track length in video-timeline seconds. */
duration: number;
switchType: SwitchType;
markers: Marker[];
/** Reuse decoded buffers from the live preview engine when available. */
cache?: SampleCache;
};
/**
* Render a keyboard-only WAV track at 1× using OfflineAudioContext.
* Marker times are absolute video seconds — preview speed is not applied.
*/
export async function renderKeyboardTrack(options: RenderKeyboardTrackOptions): Promise<Blob> {
const { duration, switchType, markers } = options;
let decodeContext: AudioContext | null = null;
const cache = options.cache ?? (() => {
decodeContext = new AudioContext();
return createSampleCache(decodeContext);
})();
try {
const manifest = await cache.ensureManifest();
const entry = manifest[switchType];
if (!entry) throw new Error(`Unknown switch: ${switchType}`);
if (cache.getLoadedSwitch() !== switchType) {
await cache.preloadSwitch(switchType);
}
const frameCount = Math.ceil(duration * EXPORT_SAMPLE_RATE);
const offline = new OfflineAudioContext(2, frameCount, EXPORT_SAMPLE_RATE);
const schedule = buildMarkerSchedule(markers, entry);
for (const event of schedule) {
if (event.time >= duration) continue;
if (event.phase === 'release' && event.sampleKey === 'GENERIC_long') continue;
const buffer = await cache.loadBuffer(switchType, event.phase, event.sampleKey);
const source = offline.createBufferSource();
source.buffer = buffer;
source.connect(offline.destination);
source.start(event.time);
}
const rendered = await offline.startRendering();
return audioBufferToWavBlob(rendered, EXPORT_SAMPLE_RATE);
} finally {
if (decodeContext) await decodeContext.close();
}
}
+5 -2
View File
@@ -25,7 +25,7 @@ const SPECIAL_KEY_SAMPLES: Record<string, PressSampleKey> = {
Backspace: 'BACKSPACE',
};
/** djb2 hash — mirrored in backend/key_sound_map.py */
/** djb2 hash for stable per-key press variant selection. */
export function hashCode(value: string): number {
let hash = 5381;
for (let i = 0; i < value.length; i++) {
@@ -42,6 +42,7 @@ export function pressVariantKey(code: string): PressSampleKey {
return `GENERIC_R${pressVariantIndex(code)}` as PressSampleKey;
}
/** Pick a press sample key from special-key map or GENERIC_R0R4 variant. */
export function resolvePressSampleKey(
keyLabel: string,
code: string,
@@ -57,12 +58,13 @@ export function resolvePressSampleKey(
return (generic ?? available[0]) as PressSampleKey;
}
/** Pick a release sample key, preferring special keys then GENERIC. */
export function resolveReleaseSampleKey(
keyLabel: string,
available: string[],
): ReleaseSampleKey {
const special = SPECIAL_KEY_SAMPLES[keyLabel];
if (special && available.includes(special)) return special;
if (special && available.includes(special)) return special as ReleaseSampleKey;
if (available.includes('GENERIC')) return 'GENERIC';
return (available[0] ?? 'GENERIC') as ReleaseSampleKey;
}
@@ -75,6 +77,7 @@ export function sampleUrl(
return `/assets/samples/kbsim/${switchType}/${phase}/${sampleKey}.mp3`;
}
/** Keyup timestamp; defaults to press time + 80 ms when not recorded. */
export function markerReleaseTime(marker: {
time: number;
releaseTime?: number;
+43
View File
@@ -0,0 +1,43 @@
import {
markerReleaseTime,
resolvePressSampleKey,
resolveReleaseSampleKey,
type SoundPhase,
type SwitchManifest,
} from './keySoundMap';
import type { Marker } from '../types';
/** A single press or release sample placed on the video timeline (seconds, 1×). */
export type ScheduledEvent = {
phase: SoundPhase;
sampleKey: string;
/** Video-timeline seconds — never adjusted for preview playback speed. */
time: number;
};
/**
* Build a sorted list of press/release events from markers.
* Times are absolute video coordinates used by both preview scheduling and 1× export.
*/
export function buildMarkerSchedule(
markers: Marker[],
switchEntry: SwitchManifest,
): ScheduledEvent[] {
const events: ScheduledEvent[] = [];
for (const marker of markers) {
const code = marker.code ?? marker.key;
events.push({
phase: 'press',
sampleKey: resolvePressSampleKey(marker.key, code, switchEntry.press),
time: marker.time,
});
events.push({
phase: 'release',
sampleKey: resolveReleaseSampleKey(marker.key, switchEntry.release),
time: markerReleaseTime(marker),
});
}
return events.sort((a, b) => a.time - b.time);
}
+82
View File
@@ -0,0 +1,82 @@
import { sampleUrl, type SoundPhase, type SwitchManifest } from './keySoundMap';
import type { SwitchType } from '../types';
export type Manifest = Record<string, SwitchManifest>;
/** Cache key format: `{switchType}/{phase}/{sampleKey}` */
function bufferKey(switchType: SwitchType, phase: SoundPhase, sampleKey: string): string {
return `${switchType}/${phase}/${sampleKey}`;
}
export type SampleCache = {
ensureManifest: () => Promise<Manifest>;
loadBuffer: (
switchType: SwitchType,
phase: SoundPhase,
sampleKey: string,
) => Promise<AudioBuffer>;
preloadSwitch: (switchType: SwitchType) => Promise<void>;
getLoadedSwitch: () => SwitchType | null;
};
/**
* Shared MP3 fetch + decode cache used by live preview and offline export.
* Requires a BaseAudioContext for `decodeAudioData` (live AudioContext or throwaway decode context).
*/
export function createSampleCache(decodeContext: BaseAudioContext): SampleCache {
let manifest: Manifest | null = null;
let loadedSwitch: SwitchType | null = null;
const buffers = new Map<string, AudioBuffer>();
async function ensureManifest(): Promise<Manifest> {
if (manifest) return manifest;
const response = await fetch('/assets/samples/kbsim/manifest.json');
if (!response.ok) throw new Error('Failed to load sample manifest');
manifest = (await response.json()) as Manifest;
return manifest;
}
async function loadBuffer(
switchType: SwitchType,
phase: SoundPhase,
sampleKey: string,
): Promise<AudioBuffer> {
const key = bufferKey(switchType, phase, sampleKey);
const cached = buffers.get(key);
if (cached) return cached;
const response = await fetch(sampleUrl(switchType, phase, sampleKey));
if (!response.ok) {
throw new Error(`Sample not found: ${sampleUrl(switchType, phase, sampleKey)}`);
}
const arrayBuffer = await response.arrayBuffer();
const audioBuffer = await decodeContext.decodeAudioData(arrayBuffer);
buffers.set(key, audioBuffer);
return audioBuffer;
}
/** Preload all press samples and release samples (skips GENERIC_long) for a switch. */
async function preloadSwitch(switchType: SwitchType): Promise<void> {
const data = await ensureManifest();
const entry = data[switchType];
if (!entry) throw new Error(`Unknown switch: ${switchType}`);
const loads: Promise<AudioBuffer>[] = [];
for (const sampleKey of entry.press) {
loads.push(loadBuffer(switchType, 'press', sampleKey));
}
for (const sampleKey of entry.release) {
if (sampleKey === 'GENERIC_long') continue;
loads.push(loadBuffer(switchType, 'release', sampleKey));
}
await Promise.all(loads);
loadedSwitch = switchType;
}
return {
ensureManifest,
loadBuffer,
preloadSwitch,
getLoadedSwitch: () => loadedSwitch,
};
}
+62
View File
@@ -0,0 +1,62 @@
/** Target sample rate for exported WAV files (matches former backend output). */
export const EXPORT_SAMPLE_RATE = 48000;
/**
* Encode an AudioBuffer as a 16-bit PCM stereo WAV Blob.
* Float samples are clamped to [-1, 1] before conversion to int16.
*/
export function audioBufferToWavBlob(
buffer: AudioBuffer,
sampleRate = EXPORT_SAMPLE_RATE,
): Blob {
const numChannels = buffer.numberOfChannels;
const length = buffer.length;
const bytesPerSample = 2;
const blockAlign = numChannels * bytesPerSample;
const dataSize = length * blockAlign;
const headerSize = 44;
const arrayBuffer = new ArrayBuffer(headerSize + dataSize);
const view = new DataView(arrayBuffer);
// RIFF header
writeString(view, 0, 'RIFF');
view.setUint32(4, 36 + dataSize, true);
writeString(view, 8, 'WAVE');
// fmt chunk (PCM, 16-bit)
writeString(view, 12, 'fmt ');
view.setUint32(16, 16, true);
view.setUint16(20, 1, true);
view.setUint16(22, numChannels, true);
view.setUint32(24, sampleRate, true);
view.setUint32(28, sampleRate * blockAlign, true);
view.setUint16(32, blockAlign, true);
view.setUint16(34, 16, true);
// data chunk
writeString(view, 36, 'data');
view.setUint32(40, dataSize, true);
// Interleave channels and write PCM samples
const channels: Float32Array[] = [];
for (let ch = 0; ch < numChannels; ch++) {
channels.push(buffer.getChannelData(ch));
}
let offset = headerSize;
for (let i = 0; i < length; i++) {
for (let ch = 0; ch < numChannels; ch++) {
const sample = Math.max(-1, Math.min(1, channels[ch][i]));
view.setInt16(offset, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true);
offset += 2;
}
}
return new Blob([arrayBuffer], { type: 'audio/wav' });
}
function writeString(view: DataView, offset: number, value: string): void {
for (let i = 0; i < value.length; i++) {
view.setUint8(offset + i, value.charCodeAt(i));
}
}
+12 -15
View File
@@ -1,3 +1,5 @@
import { renderKeyboardTrack } from './audio/export';
import type { SampleCache } from './audio/sampleCache';
import type { LegacySwitchType, Marker, Project, SwitchType } from './types';
import { app, setActiveSwitch, setMarkers, setSelectedIds } from './store.svelte';
@@ -50,24 +52,19 @@ export function loadProject(data: Project & { version?: number; switch?: string
setSelectedIds(new Set());
}
export async function exportAudio(duration: number) {
/**
* Export keyboard SFX only as a WAV file (no video audio).
* Rendered client-side at 1× via OfflineAudioContext.
*/
export async function exportAudio(duration: number, cache?: SampleCache) {
const project = buildProject();
const response = await fetch('/api/export/audio', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
duration,
switch: project.switch,
markers: project.markers,
}),
const blob = await renderKeyboardTrack({
duration,
switchType: project.switch,
markers: project.markers,
cache,
});
if (!response.ok) {
const detail = await response.text();
throw new Error(detail || 'Audio export failed');
}
const blob = await response.blob();
const url = URL.createObjectURL(blob);
const anchor = document.createElement('a');
anchor.href = url;
+51 -5
View File
@@ -1,12 +1,58 @@
import { defineConfig } from 'vite'
import { createReadStream, existsSync, statSync } from 'node:fs'
import { join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import type { IncomingMessage, ServerResponse } from 'node:http'
import { defineConfig, type Connect, type Plugin } from 'vite'
import { svelte } from '@sveltejs/vite-plugin-svelte'
const repoRoot = resolve(fileURLToPath(import.meta.url), '../..')
const assetsDir = resolve(repoRoot, 'assets')
function serveStatic(root: string): Connect.NextHandleFunction {
return (req, res, next) => {
const url = (req.url ?? '/').split('?')[0]
const filePath = resolve(join(root, url))
if (!filePath.startsWith(root) || !existsSync(filePath)) {
next()
return
}
const stat = statSync(filePath)
if (!stat.isFile()) {
next()
return
}
serveFile(filePath, res)
}
}
function serveFile(filePath: string, res: ServerResponse<IncomingMessage>): void {
const ext = filePath.split('.').pop()?.toLowerCase()
const types: Record<string, string> = {
json: 'application/json',
mp3: 'audio/mpeg',
}
res.setHeader('Content-Type', types[ext ?? ''] ?? 'application/octet-stream')
createReadStream(filePath).pipe(res)
}
/** Serve repo-level assets/ at /assets in dev and preview. */
function serveRepoAssets(): Plugin {
return {
name: 'serve-repo-assets',
configureServer(server) {
server.middlewares.use('/assets', serveStatic(assetsDir))
},
configurePreviewServer(server) {
server.middlewares.use('/assets', serveStatic(assetsDir))
},
}
}
export default defineConfig({
plugins: [svelte()],
plugins: [svelte(), serveRepoAssets()],
server: {
proxy: {
'/api': 'http://127.0.0.1:8000',
'/assets': 'http://127.0.0.1:8000',
fs: {
allow: [repoRoot],
},
},
})