From 054e9702537c0978c75f2b4e516c16cd1726fadd Mon Sep 17 00:00:00 2001 From: Jonathan Hinkel Date: Wed, 26 Aug 2026 13:12:13 +0200 Subject: [PATCH] Add client resilience, a mock provider for offline dev, and the humanoid avatar-state package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wraps RealtimeClient with reconnect-on-drop (ResilientRealtimeClient), adds a MockRealtimeProvider so client/animation work doesn't need a live OpenAI key, and introduces "optional package" as a third module-layering kind alongside core/project — humanoid is the first instance, providing a renderer-agnostic avatar-state reducer and ARKit blend-shape animation compositor that future eye look-at/blink/lip-sync libraries can build on without forcing that surface onto non-humanoid (e.g. 2D) projects. Co-Authored-By: Claude Sonnet 5 --- .env.example | 6 + AGENTS.md | 1 + docs/ARCHITECTURE.md | 21 +- docs/CONFIGURATION.md | 32 +-- docs/HUMANOID.md | 59 ++++++ docs/REALTIME_SESSION.md | 23 +- server.js | 4 +- src/lib/client/core/client.ts | 18 ++ src/lib/client/core/resilient-client.spec.ts | 176 ++++++++++++++++ src/lib/client/core/resilient-client.ts | 198 ++++++++++++++++++ src/lib/client/core/tool-dispatcher.ts | 24 ++- src/lib/client/humanoid/avatar-state.spec.ts | 103 +++++++++ src/lib/client/humanoid/avatar-state.ts | 98 +++++++++ src/lib/humanoid/animation.spec.ts | 52 +++++ src/lib/humanoid/animation.ts | 114 ++++++++++ src/lib/humanoid/state.spec.ts | 34 +++ src/lib/humanoid/state.ts | 54 +++++ src/lib/server/core/config.spec.ts | 26 +++ src/lib/server/core/config.ts | 32 ++- src/lib/server/core/dev-plugin.ts | 4 +- .../server/core/providers/fixtures/default.ts | 19 ++ src/lib/server/core/providers/index.ts | 17 ++ src/lib/server/core/providers/mock.spec.ts | 56 +++++ src/lib/server/core/providers/mock.ts | 54 +++++ src/routes/demo/humanoid/+page.svelte | 107 ++++++++++ src/routes/demo/realtime/+page.svelte | 45 +++- 26 files changed, 1329 insertions(+), 48 deletions(-) create mode 100644 docs/HUMANOID.md create mode 100644 src/lib/client/core/resilient-client.spec.ts create mode 100644 src/lib/client/core/resilient-client.ts create mode 100644 src/lib/client/humanoid/avatar-state.spec.ts create mode 100644 src/lib/client/humanoid/avatar-state.ts create mode 100644 src/lib/humanoid/animation.spec.ts create mode 100644 src/lib/humanoid/animation.ts create mode 100644 src/lib/humanoid/state.spec.ts create mode 100644 src/lib/humanoid/state.ts create mode 100644 src/lib/server/core/providers/fixtures/default.ts create mode 100644 src/lib/server/core/providers/index.ts create mode 100644 src/lib/server/core/providers/mock.spec.ts create mode 100644 src/lib/server/core/providers/mock.ts create mode 100644 src/routes/demo/humanoid/+page.svelte diff --git a/.env.example b/.env.example index ca0f553..8f7f1ce 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,12 @@ REALTIME_TOKEN_SECRET= # HTTP port the production server (server.js) listens on. # PORT=3000 +# Which RealtimeProvider to use. "openai" (default) talks to the real API and +# requires OPENAI_API_KEY above. "mock" replays a canned local script instead +# — no API key needed — for client/animation work without live usage. See +# docs/REALTIME_SESSION.md's "Development without a live API". +# REALTIME_PROVIDER=openai + # --- Analytics (see docs/ANALYTICS.md and docs/CONFIGURATION.md) --- # Master switch. Set to false to disable analytics recording entirely — no diff --git a/AGENTS.md b/AGENTS.md index 3c4994b..96988bd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,6 +8,7 @@ Base SvelteKit project meant to be branched off for future webavatar client proj - `src/lib/server/core/` — server-only implementation. SvelteKit enforces this boundary at build time; browser code cannot import from here, even transitively. - `src/lib/client/core/` — thin browser-side wrappers around the shared contracts. Transport only, no business logic. - `src/lib/project/`, `src/lib/server/project/`, `src/lib/client/project/` — mirrors the three directories above, but for code specific to whichever client project branches from this template (tool implementations, custom instructions, etc.), instead of reusable template infrastructure. **Core never imports from project, only the reverse** — see [docs/TOOLS.md](docs/TOOLS.md) for the concrete example this convention was introduced for. +- `src/lib/humanoid/`, `src/lib/client/humanoid/` — an **optional package** (not `core`, not `project`; see "Module layering" in [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md)) providing avatar-state derivation and animation-channel composition for 3D humanoid avatars. Nothing in `core`/`project` imports from it — a project only pulls it in if it's building a humanoid avatar. See [docs/HUMANOID.md](docs/HUMANOID.md). - `src/routes/` — thin route glue; delegates into the above rather than holding logic itself. - `docs/` — architecture and subsystem documentation (see "Documentation" below). diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 957c2dd..c477c54 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -12,6 +12,14 @@ Code that's part of this core infrastructure lives under `src/lib/` in three pla This repo is a base template meant to be branched per client project, so there's a second, parallel set of directories for code that's specific to whichever project branches from it — `src/lib/project/`, `src/lib/server/project/`, `src/lib/client/project/`, mirroring the three-way split above. **Core never imports from project, only the reverse** — that one-way dependency is what lets a branched project pull template updates without merge conflicts in core files. Where a project needs something from both (e.g. merging core session defaults with a project's tool registry), that merge happens at a composition root — a repo-root file like `server.js` or `vite.config.ts`, not under `src/lib` and not bound by the core/project layering — never inside a core file. See [TOOLS.md](TOOLS.md) for a concrete example of this split in practice. +### Optional packages + +A third kind of top-level directory, alongside `core` and `project`: a **package** — a named directory mirrored across only the scopes it actually needs (`src/lib//` for isomorphic code, `src/lib/server//`, `src/lib/client//`). Where `core` is infrastructure every branched project needs and `project` is that project's own content, a package is infrastructure only _some_ projects need — one a 2D avatar project, say, has no reason to even look at. + +Same dependency rule as core/project, extended: **a package may import from `core`; `core` and `project` never import from a package; a project opts in by importing directly from the package** — no registration step, no runtime flag, no manifest. If nothing imports it, it's never bundled — that's what makes it free for a project that doesn't use it. + +First (and, as of this writing, only) package: **`humanoid`** (`src/lib/humanoid/`, `src/lib/client/humanoid/`) — a renderer-agnostic avatar-state and animation-channel-composition contract for 3D humanoid avatars rigged with ARKit blend shapes. See [HUMANOID.md](HUMANOID.md). Future shared animation libraries (eye look-at, blink, lip-sync, body/gesture) are expected to build on top of it as further packages or as consumers of it, not to be folded into `core`. + Routes (`src/routes/`) stay thin and delegate into these — a route file is glue, not where logic lives. ## Foundational infrastructure decisions @@ -21,11 +29,12 @@ Routes (`src/routes/`) stay thin and delegate into these — a route file is glu ## Subsystems -| Subsystem | Doc | -| --------------------------------------------------------------------------------------------- | ------------------------------------------ | -| Realtime session management (OpenAI Realtime API today, extensible to other models/pipelines) | [REALTIME_SESSION.md](REALTIME_SESSION.md) | -| Tool calling (server- and client-executed) and the core/project ownership boundary | [TOOLS.md](TOOLS.md) | -| Analytics recording (sessions, messages, tool calls, errors) to a local SQLite database | [ANALYTICS.md](ANALYTICS.md) | -| Configuration (env vars, validation, the core/project split for persona content) | [CONFIGURATION.md](CONFIGURATION.md) | +| Subsystem | Doc | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ | +| Realtime session management (OpenAI Realtime API today, extensible to other models/pipelines; client resilience/reconnect; the mock provider for offline dev) | [REALTIME_SESSION.md](REALTIME_SESSION.md) | +| Tool calling (server- and client-executed) and the core/project ownership boundary | [TOOLS.md](TOOLS.md) | +| Analytics recording (sessions, messages, tool calls, errors) to a local SQLite database | [ANALYTICS.md](ANALYTICS.md) | +| Configuration (env vars, validation, the core/project split for persona content) | [CONFIGURATION.md](CONFIGURATION.md) | +| `humanoid` package — avatar-state derivation and animation-channel composition for humanoid avatars | [HUMANOID.md](HUMANOID.md) | Add a row here (and a corresponding doc) for each new subsystem as it's built — see the "Documentation" section in [AGENTS.md](../AGENTS.md) for when and how. diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 3339135..792501e 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -7,7 +7,8 @@ Every env var this app reads is parsed and validated in one place: [`src/lib/ser ```ts interface AppConfig { port: number; - openaiApiKey: string; + realtimeProvider: 'openai' | 'mock'; // see docs/REALTIME_SESSION.md's "Development without a live API" + openaiApiKey: string; // only required (and only meaningful) when realtimeProvider is 'openai' analytics: AnalyticsConfig; // { enabled, dbPath, messages, toolCalls, events } session: SessionRuntimeConfig; // { modelOverride?, defaultVoice?, clientToolTimeoutMs, tokenTtlMs } } @@ -17,20 +18,21 @@ interface AppConfig { ## Env vars -| Var | Default | Required | -| -------------------------- | ----------------------------------- | -------- | -| `OPENAI_API_KEY` | — | yes | -| `REALTIME_TOKEN_SECRET` | — | yes | -| `PORT` | `3000` | no | -| `ANALYTICS_ENABLED` | `true` | no | -| `ANALYTICS_DB_PATH` | `./data/analytics.db` | no | -| `ANALYTICS_LOG_MESSAGES` | `true` | no | -| `ANALYTICS_LOG_TOOL_CALLS` | `true` | no | -| `ANALYTICS_LOG_EVENTS` | `true` | no | -| `SESSION_MODEL` | `SESSION_PERSONA.model` (see below) | no | -| `SESSION_DEFAULT_VOICE` | unset — provider picks | no | -| `CLIENT_TOOL_TIMEOUT_MS` | `10000` | no | -| `REALTIME_TOKEN_TTL_MS` | `60000` | no | +| Var | Default | Required | +| -------------------------- | ----------------------------------- | ------------------------------------------------------- | +| `OPENAI_API_KEY` | — | only when `REALTIME_PROVIDER` is `openai` (the default) | +| `REALTIME_TOKEN_SECRET` | — | yes | +| `REALTIME_PROVIDER` | `openai` | no — `openai` or `mock` | +| `PORT` | `3000` | no | +| `ANALYTICS_ENABLED` | `true` | no | +| `ANALYTICS_DB_PATH` | `./data/analytics.db` | no | +| `ANALYTICS_LOG_MESSAGES` | `true` | no | +| `ANALYTICS_LOG_TOOL_CALLS` | `true` | no | +| `ANALYTICS_LOG_EVENTS` | `true` | no | +| `SESSION_MODEL` | `SESSION_PERSONA.model` (see below) | no | +| `SESSION_DEFAULT_VOICE` | unset — provider picks | no | +| `CLIENT_TOOL_TIMEOUT_MS` | `10000` | no | +| `REALTIME_TOKEN_TTL_MS` | `60000` | no | Booleans accept exactly `"true"`/`"false"`; anything else (if the var is set at all) is a validation error. Integers are validated for range where one applies (e.g. `PORT` must be `1–65535`). diff --git a/docs/HUMANOID.md b/docs/HUMANOID.md new file mode 100644 index 0000000..b2c4684 --- /dev/null +++ b/docs/HUMANOID.md @@ -0,0 +1,59 @@ +# Humanoid package + +`humanoid` is the first instance of the **optional package** convention (see "Module layering" in [ARCHITECTURE.md](ARCHITECTURE.md)) — a renderer-agnostic contract layer for 3D humanoid avatars rigged with ARKit blend shapes, scoped out of `core` because a 2D or non-humanoid project has no use for it. Nothing in `core` or `project` imports from it; a project opts in by importing directly from `$lib/humanoid` / `$lib/client/humanoid`, and if nothing imports it, it's simply not part of that project's bundle. + +This package ships the **contract** future animation libraries (eye look-at, blink, lip-sync, body/gesture) plug into — not the libraries themselves, and not a renderer. It has no idea what's rendering the avatar (Three.js, Babylon, a game engine WebGL bridge, anything else) — that stays entirely project-owned, per the base project's founding constraint that appearance/UI is never core's job. + +## `$lib/humanoid/state.ts` — avatar state + +```ts +type AvatarState = 'idle' | 'listening' | 'thinking' | 'talking'; +``` + +Deliberately just four states, not the `IdleActive`/`IdlePassive` split a humanoid rig's own animation design (e.g. a future eye look-at library) may eventually want — that distinction needs a user-presence signal (webcam/face-tracking) this package has no opinion on. A project with that signal can layer its own finer idle split on top; `idle` here just means "no signal that the user or the model is currently doing anything." + +`nextAvatarState(current, signal)` is a pure reducer — no WS/browser dependency, directly unit-tested in `state.spec.ts`. The signal set is small and closed (`listening_started`/`listening_stopped`, `response_requested`, `talking_started`/`talking_stopped`, `reset`). + +Note `listening_stopped` moves to `thinking`, not `idle` — ending listening almost always means a turn just completed and a response is expected next, so the avatar shouldn't visibly relax for the split second before the model's response starts arriving. + +## `$lib/client/humanoid/avatar-state.ts` — deriving it live + +`deriveAvatarState(source)` takes a `RealtimeClient` or `ResilientRealtimeClient` (accepted structurally, via an `AvatarStateSource` interface — this package doesn't care which transport variant is in use) and folds its `ServerEvent`s through `nextAvatarState`: + +| `ServerEvent` | Signal | +| ------------------------------------- | ----------------- | +| `session.ready` / `session.closed` | `reset` | +| `output_audio.chunk` | `talking_started` | +| `transcript.delta` (role: assistant) | `talking_started` | +| `output_audio.done` / `response.done` | `talking_stopped` | +| `transcript.done` (role: assistant) | `talking_stopped` | + +Two signals can't come from the event stream alone, so the returned store exposes them imperatively: + +- **`setListening(active)`** — call from whatever mic-capture code a project uses. That pipeline doesn't exist in `core` yet (see "Known limitations" below), and the client is the only thing that currently knows when the user starts/stops talking: the protocol doesn't surface OpenAI's `input_audio_buffer.speech_*` VAD events — see the "not part of our normalized vocabulary" comment in `server/core/providers/openai.ts`'s `fromOpenAiEvent`. +- **`notifyResponseRequested()`** — call right when sending a typed `input_text.send` (the voice path already gets this for free from `setListening(false)`). + +The returned `AvatarStateStore.subscribe()` follows the Svelte store contract (calls `run` immediately with the current value, then on every change), so it's usable directly as `$store` in a `.svelte` file without a hard Svelte dependency in this package itself. See `/demo/humanoid` (`src/routes/demo/humanoid/+page.svelte`) for the reference wiring, including the manual "listening" toggle standing in for real mic-capture UI. + +### Known limitations + +- **`thinking` is a heuristic**, not a precise reflection of server-side activity — the time between a request and the assistant's first output. The browser never sees server-executed tool calls at all (see [TOOLS.md](TOOLS.md)), so "the model is running a tool" isn't observable from here. +- **No presence-based idle split** (see above) — a project with webcam/face-tracking data can extend on top. + +## `$lib/humanoid/animation.ts` — animation-channel composition + +```ts +type ARKitBlendShape = 'jawOpen' | 'eyeBlinkLeft' | /* ...all 52 ARFaceAnchor.BlendShapeLocation names */; +type AnimationFrame = Partial>; // sparse, 0–1 weights +``` + +`compositeAnimationFrames(layers)` folds an ordered list of `AnimationLayer`s (`{ frame, mode: 'add' | 'override', weight? }`) into one `AnimationFrame`, clamping each channel to `[0, 1]`. This generalizes the eye look-at design doc's own additive-layers pattern — Macro, Micro, and Nano gaze offsets summed into one effective offset — to arbitrary channels, plus `'override'` for layers whose values can't be meaningfully summed (a blink closing an eyelid a gaze layer also touches, for instance). + +This is pure, isomorphic, and has no idea how the resulting `AnimationFrame` gets applied to a mesh — that binding (which bone/morph target each `ARKitBlendShape` name maps to in a given rig) is entirely the project's job. + +## What's not built here (on purpose) + +- The eye look-at, blink, lip-sync, and body/gesture libraries themselves — future work, built as consumers of this contract, not part of it. +- Mic-capture/audio-playback in the browser (still open — see `docs/REALTIME_SESSION.md`'s "What's deliberately not built yet"). +- Server-side viseme/phoneme generation — an open design fork (server-side, since the relay already touches raw audio, vs. client-side from the playback buffer) to decide when the lip-sync library actually gets built. +- Any protocol change (`$lib/core/protocol.ts` is untouched by this package) — `AvatarState` derivation works entirely from events the protocol already carries. diff --git a/docs/REALTIME_SESSION.md b/docs/REALTIME_SESSION.md index 100037b..211e167 100644 --- a/docs/REALTIME_SESSION.md +++ b/docs/REALTIME_SESSION.md @@ -62,6 +62,24 @@ The critical constraint: a `ProviderSession` speaks **only** the normalized `Cli **To add a new provider or pipeline:** write a new file under `src/lib/server/core/providers/` implementing `RealtimeProvider`, translating between the normalized protocol and whatever that backend speaks (see `providers/openai.ts` for the reference implementation). Then wire it in wherever `OpenAiRealtimeProvider` is currently instantiated ([`server.js`](../server.js) for production, [`dev-plugin.ts`](../src/lib/server/core/dev-plugin.ts) for dev). Nothing else changes — not `session.ts`, not `ws-handler.ts`, not any client code. +## Resilience + +`src/lib/client/core/resilient-client.ts`'s `ResilientRealtimeClient` wraps `RealtimeClient` (composition, not inheritance — `RealtimeClient` itself stays untouched, still pure transport) with automatic reconnection on an _unexpected_ transport drop. It exposes the same `send()`/`on()`/`close()` shape, plus a separate `onConnectionState(handler)` for transport-level state (`connecting | open | reconnecting | closed`) — kept out of the `ServerEvent`/protocol vocabulary deliberately, since that's the provider-relay contract, not the place for this transport's local retry bookkeeping. + +**Reconnecting the browser↔relay socket does not resurrect a dead relay↔OpenAI session.** If the server proactively sends `session.closed` (`upstream_disconnected`, `client_requested`, etc.), that's a clean end — no auto-reconnect follows. Auto-reconnect only fires when the underlying socket closes with **no** preceding `session.closed` (a network blip, laptop sleep/wake, an idle timeout) — detected via `RealtimeClient.onClose()`, a small addition to `client.ts` alongside its existing `on()` (message events) for exactly this purpose, since without it there was no way to observe a drop that isn't itself a parsed `ServerEvent`. + +Either way, a reconnect always starts a **brand-new** session — fresh token (`getToken` is called again; tokens are short-lived and a stale one can't be reused), fresh `session.start`. The model has no memory of the old session regardless, so there's no conversation state worth trying to resume. Callers see this plainly as a `reconnecting` → `open` transition followed by a fresh `session.ready`, and decide for themselves how to present it (clear the transcript, show a toast, etc.) — the wrapper doesn't hide it. See `/demo/realtime` for the reference wiring, including how it resets its own transcript on `reconnecting`. + +Backoff is exponential (`baseDelayMs * 2^(attempt-1)`), capped at `maxAttempts` consecutive failures before giving up and reporting `closed` — both configurable per instance, no env var (this is browser-local behavior, not server config). + +## Development without a live API + +`src/lib/server/core/providers/mock.ts`'s `MockRealtimeProvider` implements the same `RealtimeProvider` interface as `OpenAiRealtimeProvider` (see "The extensibility seam" below), but replays a canned `MockScript` (`Array<{ delayMs, event: ServerEvent }>`, typed directly against `ServerEvent` so a fixture can't silently drift from the protocol) on a timer instead of talking to a real upstream. `send()` is a no-op — v1 is linear, one-shot replay, not interactive/branching scripting. + +Select it with `REALTIME_PROVIDER=mock` (see [CONFIGURATION.md](CONFIGURATION.md)); `OPENAI_API_KEY` isn't required in that mode. `src/lib/server/core/providers/index.ts`'s `selectProvider(config)` is the one place both composition roots (`server.js`, `dev-plugin.ts`) call to pick between providers, instead of each hardcoding `new OpenAiRealtimeProvider(...)`. + +This exists mainly so client/animation work (transcript rendering, `$lib/humanoid`'s avatar-state derivation — see [HUMANOID.md](HUMANOID.md)) can happen without an API key or live usage. `providers/fixtures/default.ts` ships one short canned exchange for exercising exactly that; see `/demo/humanoid`. + ## The trust boundary The browser's `session.start` message can only set `voice` (see `SessionStartConfig` in `protocol.ts`). `instructions`, `tools`, and `model` are always server-owned — sourced from the project's persona ([`src/lib/project/persona.ts`](../src/lib/project/persona.ts), `model` optionally overridden by `SESSION_MODEL`; see [docs/CONFIGURATION.md](CONFIGURATION.md)) via `resolveSessionDefaults()` (`defaults.ts`), and merged with the client-visible config in `resolveSessionConfig()` (`session.ts`). A server-side `defaultVoice` (`SESSION_DEFAULT_VOICE`) fills in when the browser's `session.start` doesn't specify a voice, but the browser's own choice still wins when it sends one — this doesn't loosen the trust boundary, since voice was already the one client-controllable field. This is deliberate: this relay is shared infrastructure that other people's browsers connect to, so a client project should never let an untrusted browser dictate what the model is told to do or which model/tools it can invoke. If a client project needs more client-controllable config, extend `SessionStartConfig` and `resolveSessionConfig()` explicitly and think about what's safe to expose — don't just pass the browser's config straight through. @@ -99,9 +117,7 @@ Still unverified: the exact event name for a completed **input** audio transcrip ## What's deliberately not built yet -- A second provider or pipeline implementation (the point of the design above is that it's addable later without a rewrite) - Audio capture/playback in the browser client (`client.ts` is protocol-level only) -- Reconnect/retry handling - Rate limiting on the token-mint endpoint - Real user accounts (`tokens.ts` is anonymous-session-only) - Multi-instance/shared session storage (in-memory, single-process only) @@ -110,5 +126,6 @@ Still unverified: the exact event name for a completed **input** audio transcrip ## Verifying changes - `npm run check` / `npm run lint` / `npm run build` for type, lint, and build correctness (`build` also runs the standalone `tsc -p tsconfig.server.json` compile — watch for errors there specifically, since `svelte-check` alone won't catch problems in code that isn't reached through the SvelteKit app). -- `npm run test:unit` — unit tests for `tokens.ts`, `session.ts` (state machine, against a fake in-memory provider), and `providers/openai.ts`'s translation functions. +- `npm run test:unit` — unit tests for `tokens.ts`, `session.ts` (state machine, against a fake in-memory provider), `providers/openai.ts`'s translation functions, `providers/mock.ts`'s script replay, `resilient-client.ts`'s backoff/reconnect logic, and `$lib/humanoid`'s reducer/compositor. - **Live smoke test**: set `OPENAI_API_KEY` and `REALTIME_TOKEN_SECRET` in `.env` (see `.env.example`), run `npm run dev`, open `/demo/realtime`, and actually send a message. Type-checking and unit tests can't catch a wire-protocol mismatch like the one above — only a real round trip against the real API can. Remember the HMR gotcha above if you're iterating on `src/lib/server/core/**`. +- **Mock-provider smoke test**: `REALTIME_PROVIDER=mock npm run dev`, open `/demo/humanoid`, connect, and watch `avatarState` move through the default fixture — zero OpenAI usage, useful for iterating on animation timing without live traffic. diff --git a/server.js b/server.js index da64422..2d49aa4 100644 --- a/server.js +++ b/server.js @@ -14,7 +14,7 @@ import { } from './build-server/server/core/analytics.js'; import { loadConfig } from './build-server/server/core/config.js'; import { resolveSessionDefaults } from './build-server/server/core/defaults.js'; -import { OpenAiRealtimeProvider } from './build-server/server/core/providers/openai.js'; +import { selectProvider } from './build-server/server/core/providers/index.js'; import { handleRealtimeUpgrade } from './build-server/server/core/ws-handler.js'; import { toolRegistry } from './build-server/server/project/tools/index.js'; import { SESSION_PERSONA } from './build-server/project/persona.js'; @@ -23,7 +23,7 @@ import { SESSION_PERSONA } from './build-server/project/persona.js'; // error if anything required is missing or invalid — see docs/CONFIGURATION.md. const config = loadConfig(); -const provider = new OpenAiRealtimeProvider({ apiKey: config.openaiApiKey }); +const provider = selectProvider(config); const recorder = config.analytics.enabled ? new SqliteAnalyticsRecorder(config.analytics.dbPath, config.analytics) : new NoopAnalyticsRecorder(); diff --git a/src/lib/client/core/client.ts b/src/lib/client/core/client.ts index 368724c..a45581d 100644 --- a/src/lib/client/core/client.ts +++ b/src/lib/client/core/client.ts @@ -6,6 +6,7 @@ export interface RealtimeClientOptions { } type ServerEventHandler = (event: ServerEvent) => void; +type CloseHandler = () => void; /** * Thin typed WebSocket wrapper around the `$lib/core/protocol` wire contract @@ -16,6 +17,7 @@ type ServerEventHandler = (event: ServerEvent) => void; export class RealtimeClient { private socket: WebSocket | null = null; private readonly handlers = new Set(); + private readonly closeHandlers = new Set(); connect(token: string, options: RealtimeClientOptions = {}): Promise { const base = options.baseUrl ?? location.origin; @@ -34,6 +36,7 @@ export class RealtimeClient { socket.addEventListener('message', (event) => this.handleMessage(event)); socket.addEventListener('close', () => { this.socket = null; + for (const handler of this.closeHandlers) handler(); }); this.socket = socket; }); @@ -57,6 +60,21 @@ export class RealtimeClient { this.socket = null; } + /** + * Fires whenever the underlying socket closes, for any reason — a clean + * `close()` call, a server-initiated close, or an unexpected drop. Purely + * transport-level (mirrors the WebSocket's own 'close' event); consumers + * that care about *why* it closed should watch the `session.closed` + * `ServerEvent` via `on()` instead — see `ResilientRealtimeClient` + * (`resilient-client.ts`), which uses both together to tell an expected + * end apart from a drop worth reconnecting. Returns an unsubscribe + * function, matching `on()`'s convention. + */ + onClose(handler: CloseHandler): () => void { + this.closeHandlers.add(handler); + return () => this.closeHandlers.delete(handler); + } + private handleMessage(event: MessageEvent): void { let parsed: ServerEvent; try { diff --git a/src/lib/client/core/resilient-client.spec.ts b/src/lib/client/core/resilient-client.spec.ts new file mode 100644 index 0000000..8af2c47 --- /dev/null +++ b/src/lib/client/core/resilient-client.spec.ts @@ -0,0 +1,176 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { ClientEvent, ServerEvent } from '../../core/protocol.js'; +import { ResilientRealtimeClient, type RealtimeTransport } from './resilient-client.js'; + +/** Minimal stand-in for RealtimeClient — only what ResilientRealtimeClient touches. */ +class FakeTransport implements RealtimeTransport { + sent: ClientEvent[] = []; + closed = false; + connectCalls: string[] = []; + connectResult: 'resolve' | 'reject' = 'resolve'; + private eventHandler: ((event: ServerEvent) => void) | null = null; + private closeHandler: (() => void) | null = null; + + async connect(token: string): Promise { + this.connectCalls.push(token); + if (this.connectResult === 'reject') throw new Error('connect failed'); + } + + send(event: ClientEvent): void { + this.sent.push(event); + } + + on(handler: (event: ServerEvent) => void): () => void { + this.eventHandler = handler; + return () => { + this.eventHandler = null; + }; + } + + onClose(handler: () => void): () => void { + this.closeHandler = handler; + return () => { + this.closeHandler = null; + }; + } + + close(): void { + this.closed = true; + } + + /** Simulates a ServerEvent arriving from the (fake) relay. */ + emit(event: ServerEvent): void { + this.eventHandler?.(event); + } + + /** Simulates the underlying socket closing, for any reason. */ + drop(): void { + this.closeHandler?.(); + } +} + +function makeHarness( + opts: { + maxAttempts?: number; + baseDelayMs?: number; + configureTransport?: (t: FakeTransport) => void; + } = {} +) { + const transports: FakeTransport[] = []; + let tokenCount = 0; + const client = new ResilientRealtimeClient({ + getToken: async () => `token-${++tokenCount}`, + maxAttempts: opts.maxAttempts, + baseDelayMs: opts.baseDelayMs ?? 100, + createTransport: () => { + const t = new FakeTransport(); + opts.configureTransport?.(t); + transports.push(t); + return t; + } + }); + return { client, transports }; +} + +describe('ResilientRealtimeClient', () => { + it('connects, sends session.start, and proxies send()/on() to the live transport', async () => { + const { client, transports } = makeHarness(); + const states: string[] = []; + client.onConnectionState((s) => states.push(s)); + + await client.connect(); + + expect(transports).toHaveLength(1); + expect(transports[0].sent).toContainEqual({ type: 'session.start' }); + expect(states).toEqual(['connecting', 'open']); + + const received: ServerEvent[] = []; + client.on((e) => received.push(e)); + transports[0].emit({ type: 'session.ready', sessionId: 's1' }); + expect(received).toEqual([{ type: 'session.ready', sessionId: 's1' }]); + + client.send({ type: 'input_text.send', text: 'hi' }); + expect(transports[0].sent).toContainEqual({ type: 'input_text.send', text: 'hi' }); + }); + + it('does not reconnect after a clean session.closed', async () => { + const { client, transports } = makeHarness(); + const states: string[] = []; + client.onConnectionState((s) => states.push(s)); + await client.connect(); + + transports[0].emit({ type: 'session.closed', reason: 'client_requested' }); + // The underlying socket closing shortly after, as it does in practice — + // must not be mistaken for an unexpected drop. + transports[0].drop(); + + expect(states).toEqual(['connecting', 'open', 'closed']); + expect(transports).toHaveLength(1); + }); + + it('reconnects with backoff after an unexpected drop, fetching a fresh token and re-sending session.start', async () => { + vi.useFakeTimers(); + try { + const { client, transports } = makeHarness({ baseDelayMs: 100 }); + const states: string[] = []; + client.onConnectionState((s) => states.push(s)); + await client.connect(); + + transports[0].drop(); // no prior session.closed — unexpected + + expect(states.at(-1)).toBe('reconnecting'); + expect(transports).toHaveLength(1); // waiting on backoff, hasn't retried yet + + await vi.advanceTimersByTimeAsync(100); + + expect(transports).toHaveLength(2); + expect(transports[1].connectCalls).toEqual(['token-2']); // fresh token, not reused + expect(transports[1].sent).toContainEqual({ type: 'session.start' }); + expect(states).toEqual(['connecting', 'open', 'reconnecting', 'open']); + } finally { + vi.useRealTimers(); + } + }); + + it('gives up after maxAttempts consecutive failures', async () => { + vi.useFakeTimers(); + try { + const { client, transports } = makeHarness({ + maxAttempts: 2, + baseDelayMs: 10, + configureTransport: (t) => { + t.connectResult = 'reject'; + } + }); + const states: string[] = []; + client.onConnectionState((s) => states.push(s)); + + await client.connect(); // initial attempt fails + await vi.advanceTimersByTimeAsync(10); // retry 1 fails (delay = 10 * 2^0) + await vi.advanceTimersByTimeAsync(20); // retry 2 fails (delay = 10 * 2^1) -> exceeds maxAttempts + + expect(states.at(-1)).toBe('closed'); + expect(transports).toHaveLength(3); // initial attempt + 2 retries + } finally { + vi.useRealTimers(); + } + }); + + it('close() tears down the transport and suppresses any pending reconnect', async () => { + vi.useFakeTimers(); + try { + const { client, transports } = makeHarness({ baseDelayMs: 100 }); + await client.connect(); + transports[0].drop(); + expect(transports[0].closed).toBe(true); + + client.close(); + await vi.advanceTimersByTimeAsync(1000); + + expect(transports).toHaveLength(1); // no reconnect happened after close() + expect(() => client.send({ type: 'session.end' })).toThrow(); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/src/lib/client/core/resilient-client.ts b/src/lib/client/core/resilient-client.ts new file mode 100644 index 0000000..53bd7fd --- /dev/null +++ b/src/lib/client/core/resilient-client.ts @@ -0,0 +1,198 @@ +import type { ClientEvent, ServerEvent } from '../../core/protocol.js'; +import { RealtimeClient, type RealtimeClientOptions } from './client.js'; + +export type ConnectionState = 'connecting' | 'open' | 'reconnecting' | 'closed'; + +type ServerEventHandler = (event: ServerEvent) => void; +type ConnectionStateHandler = (state: ConnectionState) => void; + +/** + * The subset of `RealtimeClient`'s surface `ResilientRealtimeClient` needs — + * exists so tests (and, in principle, an alternative transport) can supply a + * fake without touching a real WebSocket. `RealtimeClient` satisfies this + * structurally; no explicit `implements`. + */ +export interface RealtimeTransport { + connect(token: string, options?: RealtimeClientOptions): Promise; + send(event: ClientEvent): void; + on(handler: ServerEventHandler): () => void; + onClose(handler: () => void): () => void; + close(): void; +} + +export interface ResilientRealtimeClientOptions extends RealtimeClientOptions { + /** + * Re-fetches a fresh session token (e.g. `POST /api/realtime/session`). + * Required because tokens are short-lived — a stale one can't be reused + * across a reconnect — and because reconnecting has to happen without the + * caller's involvement for it to be worth wrapping in the first place. + */ + getToken: () => Promise; + /** Max consecutive reconnect attempts before giving up and reporting 'closed'. */ + maxAttempts?: number; + /** Base delay (ms) for exponential backoff between attempts. */ + baseDelayMs?: number; + /** For tests. Defaults to `() => new RealtimeClient()`. */ + createTransport?: () => RealtimeTransport; +} + +const DEFAULT_MAX_ATTEMPTS = 5; +const DEFAULT_BASE_DELAY_MS = 500; + +/** + * Wraps `RealtimeClient` with automatic reconnection on an *unexpected* + * transport drop — see docs/REALTIME_SESSION.md's "Resilience" section for + * the full reasoning. `RealtimeClient` itself is untouched (composition, not + * inheritance) and stays pure transport; `send()`/`on()`/`close()` here have + * the same shape, transparently proxied to whichever underlying transport is + * currently live across reconnects. Connection-state changes are a separate + * observable (`onConnectionState`) from the `ServerEvent`/protocol + * vocabulary — `protocol.ts` is the provider-relay contract, not the place + * for this transport's local retry bookkeeping. + * + * Critical design point: reconnecting the browser↔relay socket does not + * resurrect a dead relay↔OpenAI session. If the server proactively sends + * `session.closed`, that's a clean end — no auto-reconnect. Auto-reconnect + * only fires on an unexpected drop (the underlying socket closing with no + * preceding `session.closed`), and it always starts a **brand-new** session + * (fresh token, fresh `session.start`) — the model has no memory of the old + * one either way, so there's no conversation state worth trying to resume. + * Callers see this plainly as a `reconnecting` → `open` transition and a + * fresh `session.ready`, and decide for themselves how to present it (e.g. + * clear the transcript, show a toast) — this wrapper doesn't hide it. + */ +export class ResilientRealtimeClient { + private readonly createTransport: () => RealtimeTransport; + private readonly maxAttempts: number; + private readonly baseDelayMs: number; + + private transport: RealtimeTransport | null = null; + private state: ConnectionState = 'closed'; + private attempt = 0; + private closedByCaller = false; + private reconnectTimer: ReturnType | null = null; + private stopEvents: (() => void) | null = null; + private stopClose: (() => void) | null = null; + + private readonly eventHandlers = new Set(); + private readonly stateHandlers = new Set(); + + constructor(private readonly options: ResilientRealtimeClientOptions) { + this.createTransport = options.createTransport ?? (() => new RealtimeClient()); + this.maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS; + this.baseDelayMs = options.baseDelayMs ?? DEFAULT_BASE_DELAY_MS; + } + + async connect(): Promise { + this.closedByCaller = false; + this.attempt = 0; + await this.establish(); + } + + send(event: ClientEvent): void { + if (!this.transport) throw new Error('ResilientRealtimeClient is not connected.'); + this.transport.send(event); + } + + /** Returns an unsubscribe function. */ + on(handler: ServerEventHandler): () => void { + this.eventHandlers.add(handler); + return () => this.eventHandlers.delete(handler); + } + + /** Returns an unsubscribe function. */ + onConnectionState(handler: ConnectionStateHandler): () => void { + this.stateHandlers.add(handler); + return () => this.stateHandlers.delete(handler); + } + + close(): void { + this.closedByCaller = true; + if (this.reconnectTimer) { + clearTimeout(this.reconnectTimer); + this.reconnectTimer = null; + } + this.teardownTransport(); + this.setState('closed'); + } + + private async establish(): Promise { + this.setState(this.attempt === 0 ? 'connecting' : 'reconnecting'); + + let token: string; + try { + token = await this.options.getToken(); + } catch { + this.handleUnexpectedDrop(); + return; + } + if (this.closedByCaller) return; // close() was called while awaiting the token + + const nextTransport = this.createTransport(); + this.transport = nextTransport; + this.stopEvents = nextTransport.on((event) => { + this.forwardEvent(event); + if (event.type === 'session.closed' && this.transport === nextTransport) { + this.teardownTransport(); + this.setState('closed'); + } + }); + this.stopClose = nextTransport.onClose(() => { + if (this.transport !== nextTransport || this.state !== 'open') return; + this.handleUnexpectedDrop(); + }); + + try { + await nextTransport.connect(token, this.options); + } catch { + if (this.transport === nextTransport) this.handleUnexpectedDrop(); + return; + } + + // Torn down while connecting — a session.closed raced in, or close() + // was called mid-handshake. Don't resurrect it as 'open'. + if (this.transport !== nextTransport) return; + + this.attempt = 0; + this.setState('open'); + nextTransport.send({ type: 'session.start' }); + } + + private handleUnexpectedDrop(): void { + this.teardownTransport(); + if (this.closedByCaller) { + this.setState('closed'); + return; + } + this.attempt++; + if (this.attempt > this.maxAttempts) { + this.setState('closed'); + return; + } + this.setState('reconnecting'); + const delay = this.baseDelayMs * 2 ** (this.attempt - 1); + this.reconnectTimer = setTimeout(() => { + this.reconnectTimer = null; + void this.establish(); + }, delay); + } + + private teardownTransport(): void { + this.stopEvents?.(); + this.stopEvents = null; + this.stopClose?.(); + this.stopClose = null; + this.transport?.close(); + this.transport = null; + } + + private forwardEvent(event: ServerEvent): void { + for (const handler of this.eventHandlers) handler(event); + } + + private setState(state: ConnectionState): void { + if (this.state === state) return; + this.state = state; + for (const handler of this.stateHandlers) handler(state); + } +} diff --git a/src/lib/client/core/tool-dispatcher.ts b/src/lib/client/core/tool-dispatcher.ts index 4101fff..d22d486 100644 --- a/src/lib/client/core/tool-dispatcher.ts +++ b/src/lib/client/core/tool-dispatcher.ts @@ -1,19 +1,29 @@ +import type { ClientEvent, ServerEvent } from '../../core/protocol.js'; import { formatToolError } from '../../core/tools.js'; -import type { RealtimeClient } from './client.js'; export type ClientToolHandler = (args: Record) => Promise; /** - * Wires a `RealtimeClient` to a map of client-side tool handlers: listens - * for `tool.call`, runs the matching handler, and sends its result back — - * kept out of `RealtimeClient` itself, which stays transport-only. Handler + * The subset of `RealtimeClient`'s (or `ResilientRealtimeClient`'s) surface + * this needs — accepted structurally so either transport works here without + * this module caring which one it's given. + */ +export interface ToolDispatchClient { + send(event: ClientEvent): void; + on(handler: (event: ServerEvent) => void): () => void; +} + +/** + * Wires a client to a map of client-side tool handlers: listens for + * `tool.call`, runs the matching handler, and sends its result back — kept + * out of the transport itself, which stays transport-only. Handler * implementations are project-owned (`$lib/client/project/tools`); this is * just the generic dispatch mechanism. * - * Returns an unsubscribe function, matching `RealtimeClient.on()`'s convention. + * Returns an unsubscribe function, matching `on()`'s convention. */ export function createToolDispatcher( - client: RealtimeClient, + client: ToolDispatchClient, handlers: Record ): () => void { return client.on((event) => { @@ -23,7 +33,7 @@ export function createToolDispatcher( } async function dispatch( - client: RealtimeClient, + client: ToolDispatchClient, handlers: Record, event: { callId: string; name: string; arguments: string } ): Promise { diff --git a/src/lib/client/humanoid/avatar-state.spec.ts b/src/lib/client/humanoid/avatar-state.spec.ts new file mode 100644 index 0000000..5c677f2 --- /dev/null +++ b/src/lib/client/humanoid/avatar-state.spec.ts @@ -0,0 +1,103 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { ServerEvent } from '../../core/protocol.js'; +import { deriveAvatarState, type AvatarStateSource } from './avatar-state.js'; + +/** Minimal stand-in for RealtimeClient/ResilientRealtimeClient — only .on() is used. */ +class FakeSource implements AvatarStateSource { + private handler: ((event: ServerEvent) => void) | null = null; + + on(handler: (event: ServerEvent) => void): () => void { + this.handler = handler; + return () => { + this.handler = null; + }; + } + + emit(event: ServerEvent): void { + this.handler?.(event); + } +} + +describe('deriveAvatarState', () => { + it('subscribe() emits the current state immediately', () => { + const store = deriveAvatarState(new FakeSource()); + const run = vi.fn(); + store.subscribe(run); + expect(run).toHaveBeenCalledTimes(1); + expect(run).toHaveBeenCalledWith('idle'); + }); + + it('walks listening -> thinking -> talking -> idle across a full voice turn', () => { + const source = new FakeSource(); + const store = deriveAvatarState(source); + const seen: string[] = []; + store.subscribe((s) => seen.push(s)); + + store.setListening(true); + store.setListening(false); + source.emit({ type: 'output_audio.chunk', audio: '' }); + source.emit({ type: 'output_audio.done' }); + + expect(seen).toEqual(['idle', 'listening', 'thinking', 'talking', 'idle']); + }); + + it('notifyResponseRequested() drives thinking for the typed-text path', () => { + const source = new FakeSource(); + const store = deriveAvatarState(source); + const seen: string[] = []; + store.subscribe((s) => seen.push(s)); + + store.notifyResponseRequested(); + source.emit({ type: 'transcript.delta', role: 'assistant', text: 'hi' }); + source.emit({ type: 'transcript.done', role: 'assistant' }); + + expect(seen).toEqual(['idle', 'thinking', 'talking', 'idle']); + }); + + it('ignores user transcript events (no signal either way)', () => { + const source = new FakeSource(); + const store = deriveAvatarState(source); + const seen: string[] = []; + store.subscribe((s) => seen.push(s)); + + source.emit({ type: 'transcript.delta', role: 'user', text: 'hello' }); + + expect(seen).toEqual(['idle']); + }); + + it('session.ready and session.closed reset to idle from any state', () => { + const source = new FakeSource(); + const store = deriveAvatarState(source); + const seen: string[] = []; + store.subscribe((s) => seen.push(s)); + + source.emit({ type: 'output_audio.chunk', audio: '' }); // -> talking + source.emit({ type: 'session.closed', reason: 'client_requested' }); + + expect(seen).toEqual(['idle', 'talking', 'idle']); + }); + + it('response.done is a fallback that stops talking even without output_audio.done', () => { + const source = new FakeSource(); + const store = deriveAvatarState(source); + const seen: string[] = []; + store.subscribe((s) => seen.push(s)); + + source.emit({ type: 'output_audio.chunk', audio: '' }); + source.emit({ type: 'response.done' }); + + expect(seen).toEqual(['idle', 'talking', 'idle']); + }); + + it('destroy() stops reacting to further events', () => { + const source = new FakeSource(); + const store = deriveAvatarState(source); + const seen: string[] = []; + store.subscribe((s) => seen.push(s)); + + store.destroy(); + source.emit({ type: 'output_audio.chunk', audio: '' }); + + expect(seen).toEqual(['idle']); + }); +}); diff --git a/src/lib/client/humanoid/avatar-state.ts b/src/lib/client/humanoid/avatar-state.ts new file mode 100644 index 0000000..17480b9 --- /dev/null +++ b/src/lib/client/humanoid/avatar-state.ts @@ -0,0 +1,98 @@ +import type { ServerEvent } from '../../core/protocol.js'; +import { nextAvatarState, type AvatarSignal, type AvatarState } from '../../humanoid/state.js'; + +/** + * The subset of `RealtimeClient`'s (or `ResilientRealtimeClient`'s) surface + * `deriveAvatarState` needs — accepted structurally so this package doesn't + * care which transport variant is in use. + */ +export interface AvatarStateSource { + on(handler: (event: ServerEvent) => void): () => void; +} + +export interface AvatarStateStore { + /** Svelte-store-compatible: calls `run` immediately with the current value, then on every change. */ + subscribe(run: (state: AvatarState) => void): () => void; + /** + * Called by whatever mic-capture code a project uses (that pipeline + * doesn't exist in core yet — see docs/HUMANOID.md's non-goals). The + * client, not the server, is the only thing that currently knows when + * the user starts/stops talking: the protocol doesn't surface OpenAI's + * `input_audio_buffer.speech_*` VAD events — see the "not part of our + * normalized vocabulary" comment in + * `server/core/providers/openai.ts`'s `fromOpenAiEvent`. + */ + setListening(active: boolean): void; + /** + * Call right when sending a request the server-event stream alone won't + * announce — chiefly a typed `input_text.send` (the voice path already + * gets this for free from `setListening(false)`). + */ + notifyResponseRequested(): void; + /** Stops listening to the source and releases all subscribers. */ + destroy(): void; +} + +/** + * Derives a reactive `AvatarState` from a live `RealtimeClient` or + * `ResilientRealtimeClient`. Translates `ServerEvent`s (plus the imperative + * calls above, for signals the event stream alone can't carry) into + * `AvatarSignal`s and folds them through the pure `nextAvatarState` reducer + * from `$lib/humanoid/state.ts`. + * + * `thinking` is an explicit heuristic — the time between a request and the + * assistant's first output — not a precise reflection of server-side + * activity: the browser never sees server-executed tool calls at all (see + * docs/TOOLS.md), so "the model is running a tool" isn't observable here. + */ +export function deriveAvatarState(source: AvatarStateSource): AvatarStateStore { + let state: AvatarState = 'idle'; + const subscribers = new Set<(state: AvatarState) => void>(); + + function apply(signal: AvatarSignal): void { + const next = nextAvatarState(state, signal); + if (next === state) return; + state = next; + for (const run of subscribers) run(state); + } + + const stopEvents = source.on((event) => { + switch (event.type) { + case 'session.ready': + case 'session.closed': + apply({ type: 'reset' }); + break; + case 'output_audio.chunk': + apply({ type: 'talking_started' }); + break; + case 'transcript.delta': + if (event.role === 'assistant') apply({ type: 'talking_started' }); + break; + case 'output_audio.done': + case 'response.done': + apply({ type: 'talking_stopped' }); + break; + case 'transcript.done': + if (event.role === 'assistant') apply({ type: 'talking_stopped' }); + break; + } + }); + + return { + subscribe(run) { + subscribers.add(run); + run(state); + return () => subscribers.delete(run); + }, + setListening(active) { + apply({ type: active ? 'listening_started' : 'listening_stopped' }); + }, + notifyResponseRequested() { + apply({ type: 'response_requested' }); + }, + destroy() { + stopEvents(); + subscribers.clear(); + } + }; +} diff --git a/src/lib/humanoid/animation.spec.ts b/src/lib/humanoid/animation.spec.ts new file mode 100644 index 0000000..72730e6 --- /dev/null +++ b/src/lib/humanoid/animation.spec.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest'; +import { compositeAnimationFrames } from './animation.js'; + +describe('compositeAnimationFrames', () => { + it('sums add-mode layers onto the same channel', () => { + // 0.25 + 0.25, not 0.2 + 0.1, so the sum is exactly representable in + // binary floating point and this isn't a disguised precision test. + expect( + compositeAnimationFrames([ + { frame: { eyeLookInLeft: 0.25 }, mode: 'add' }, + { frame: { eyeLookInLeft: 0.25 }, mode: 'add' } + ]) + ).toEqual({ eyeLookInLeft: 0.5 }); + }); + + it('override replaces rather than sums, later layers winning', () => { + expect( + compositeAnimationFrames([ + { frame: { eyeBlinkLeft: 0.1 }, mode: 'add' }, + { frame: { eyeBlinkLeft: 1 }, mode: 'override' } + ]) + ).toEqual({ eyeBlinkLeft: 1 }); + }); + + it('applies a layer weight multiplier before composing', () => { + expect( + compositeAnimationFrames([{ frame: { jawOpen: 0.5 }, mode: 'add', weight: 0.5 }]) + ).toEqual({ jawOpen: 0.25 }); + }); + + it('clamps summed channels to [0, 1]', () => { + expect( + compositeAnimationFrames([ + { frame: { mouthSmileLeft: 0.8 }, mode: 'add' }, + { frame: { mouthSmileLeft: 0.8 }, mode: 'add' } + ]) + ).toEqual({ mouthSmileLeft: 1 }); + }); + + it('leaves untouched channels from other layers alone', () => { + expect( + compositeAnimationFrames([ + { frame: { eyeBlinkLeft: 1 }, mode: 'override' }, + { frame: { jawOpen: 0.4 }, mode: 'add' } + ]) + ).toEqual({ eyeBlinkLeft: 1, jawOpen: 0.4 }); + }); + + it('returns an empty frame for no layers', () => { + expect(compositeAnimationFrames([])).toEqual({}); + }); +}); diff --git a/src/lib/humanoid/animation.ts b/src/lib/humanoid/animation.ts new file mode 100644 index 0000000..2fad74d --- /dev/null +++ b/src/lib/humanoid/animation.ts @@ -0,0 +1,114 @@ +/** + * Isomorphic, renderer-agnostic animation-channel contract — the other half + * of the optional `humanoid` package alongside `state.ts` (see + * docs/HUMANOID.md). Defines a shared channel vocabulary (the standard + * ARKit blend shapes most 3D humanoid rigs in this project family share) and + * a compositor so independently-built animation sources (a future eye + * look-at library, blink, lip-sync, body/gesture) can layer onto the same + * frame without colliding — mirrors the additive-layers pattern from the + * eye look-at design doc (Macro/Micro/Nano summed into one gaze offset), + * generalized to arbitrary channels and to layers that override rather than + * sum. No rendering, no bone/mesh knowledge — a project's own rig-binding + * code is what actually applies a composited `AnimationFrame` to a mesh. + */ + +/** The 52 standard ARKit face blend shapes (`ARFaceAnchor.BlendShapeLocation`). */ +export type ARKitBlendShape = + | 'browDownLeft' + | 'browDownRight' + | 'browInnerUp' + | 'browOuterUpLeft' + | 'browOuterUpRight' + | 'cheekPuff' + | 'cheekSquintLeft' + | 'cheekSquintRight' + | 'eyeBlinkLeft' + | 'eyeBlinkRight' + | 'eyeLookDownLeft' + | 'eyeLookDownRight' + | 'eyeLookInLeft' + | 'eyeLookInRight' + | 'eyeLookOutLeft' + | 'eyeLookOutRight' + | 'eyeLookUpLeft' + | 'eyeLookUpRight' + | 'eyeSquintLeft' + | 'eyeSquintRight' + | 'eyeWideLeft' + | 'eyeWideRight' + | 'jawForward' + | 'jawLeft' + | 'jawOpen' + | 'jawRight' + | 'mouthClose' + | 'mouthDimpleLeft' + | 'mouthDimpleRight' + | 'mouthFrownLeft' + | 'mouthFrownRight' + | 'mouthFunnel' + | 'mouthLeft' + | 'mouthLowerDownLeft' + | 'mouthLowerDownRight' + | 'mouthPressLeft' + | 'mouthPressRight' + | 'mouthPucker' + | 'mouthRight' + | 'mouthRollLower' + | 'mouthRollUpper' + | 'mouthShrugLower' + | 'mouthShrugUpper' + | 'mouthSmileLeft' + | 'mouthSmileRight' + | 'mouthStretchLeft' + | 'mouthStretchRight' + | 'mouthUpperUpLeft' + | 'mouthUpperUpRight' + | 'noseSneerLeft' + | 'noseSneerRight' + | 'tongueOut'; + +/** Sparse channel → weight (0–1) map for a single frame. */ +export type AnimationFrame = Partial>; + +export interface AnimationLayer { + frame: AnimationFrame; + /** + * `'add'` sums onto whatever the channel already holds (the eye look-at + * doc's Macro/Micro/Nano pattern — several small offsets combining into + * one). `'override'` replaces the channel outright — for layers whose + * values can't be meaningfully summed (e.g. a blink closing an eyelid + * that a gaze layer also touches). + */ + mode: 'add' | 'override'; + /** Multiplies every weight in `frame` before composing. Defaults to 1. */ + weight?: number; +} + +/** + * Folds `layers` in order into one `AnimationFrame`, clamping each channel + * to [0, 1] at the end. Later layers in the array apply after earlier ones — + * order matters for `'override'` layers, not for `'add'` ones. + */ +export function compositeAnimationFrames(layers: AnimationLayer[]): AnimationFrame { + const result: AnimationFrame = {}; + + for (const layer of layers) { + const weight = layer.weight ?? 1; + for (const [channel, value] of Object.entries(layer.frame) as Array< + [ARKitBlendShape, number] + >) { + const scaled = value * weight; + if (layer.mode === 'override') { + result[channel] = scaled; + } else { + result[channel] = (result[channel] ?? 0) + scaled; + } + } + } + + for (const channel of Object.keys(result) as ARKitBlendShape[]) { + result[channel] = Math.min(1, Math.max(0, result[channel]!)); + } + + return result; +} diff --git a/src/lib/humanoid/state.spec.ts b/src/lib/humanoid/state.spec.ts new file mode 100644 index 0000000..e9c2007 --- /dev/null +++ b/src/lib/humanoid/state.spec.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest'; +import { nextAvatarState } from './state.js'; + +describe('nextAvatarState', () => { + it('listening_started always moves to listening', () => { + expect(nextAvatarState('idle', { type: 'listening_started' })).toBe('listening'); + expect(nextAvatarState('talking', { type: 'listening_started' })).toBe('listening'); + }); + + it('listening_stopped moves to thinking (a response is expected next), only from listening', () => { + expect(nextAvatarState('listening', { type: 'listening_stopped' })).toBe('thinking'); + expect(nextAvatarState('idle', { type: 'listening_stopped' })).toBe('idle'); + }); + + it('response_requested moves to thinking unless already talking', () => { + expect(nextAvatarState('idle', { type: 'response_requested' })).toBe('thinking'); + expect(nextAvatarState('listening', { type: 'response_requested' })).toBe('thinking'); + expect(nextAvatarState('talking', { type: 'response_requested' })).toBe('talking'); + }); + + it('talking_started always moves to talking', () => { + expect(nextAvatarState('thinking', { type: 'talking_started' })).toBe('talking'); + }); + + it('talking_stopped only returns to idle from talking', () => { + expect(nextAvatarState('talking', { type: 'talking_stopped' })).toBe('idle'); + expect(nextAvatarState('idle', { type: 'talking_stopped' })).toBe('idle'); + }); + + it('reset always returns to idle', () => { + expect(nextAvatarState('talking', { type: 'reset' })).toBe('idle'); + expect(nextAvatarState('listening', { type: 'reset' })).toBe('idle'); + }); +}); diff --git a/src/lib/humanoid/state.ts b/src/lib/humanoid/state.ts new file mode 100644 index 0000000..10740b7 --- /dev/null +++ b/src/lib/humanoid/state.ts @@ -0,0 +1,54 @@ +/** + * Isomorphic, renderer-agnostic avatar state — part of the optional + * `humanoid` package (see docs/HUMANOID.md and the "Module layering" section + * of docs/ARCHITECTURE.md for what that means). Pure types and a pure + * reducer, no WS/browser dependency, so future animation libraries (eye + * look-at, blink, lip-sync, body/gesture) can all key off the same signal + * without each re-deriving turn-taking themselves. + * + * Deliberately just four states, not the `IdleActive`/`IdlePassive` split a + * humanoid rig's own animation design may eventually want — that + * distinction needs a user-presence signal (webcam/face-tracking) this + * package has no opinion on. A project with that signal can layer its own + * finer-grained idle split on top; `idle` here just means "no signal that + * the user or the model is currently doing anything." + */ +export type AvatarState = 'idle' | 'listening' | 'thinking' | 'talking'; + +export type AvatarSignal = + | { type: 'listening_started' } + | { type: 'listening_stopped' } + | { type: 'response_requested' } + | { type: 'talking_started' } + | { type: 'talking_stopped' } + | { type: 'reset' }; + +/** + * Pure state transition — no side effects, directly unit-testable. The + * browser-facing translation from real `ServerEvent`s (and the imperative + * `setListening` a project's mic-capture code calls) into `AvatarSignal`s + * lives in `$lib/client/humanoid/avatar-state.ts`, which folds them through + * this function; this module doesn't know a `RealtimeClient` exists. + */ +export function nextAvatarState(current: AvatarState, signal: AvatarSignal): AvatarState { + switch (signal.type) { + case 'listening_started': + return 'listening'; + // Ending listening almost always means a turn just completed and a + // response is expected next — go straight to 'thinking' rather than + // 'idle', so the avatar doesn't visibly relax for the split second + // before the model's response starts arriving. + case 'listening_stopped': + return current === 'listening' ? 'thinking' : current; + case 'response_requested': + return current === 'talking' ? current : 'thinking'; + case 'talking_started': + return 'talking'; + case 'talking_stopped': + return current === 'talking' ? 'idle' : current; + case 'reset': + return 'idle'; + default: + return current; + } +} diff --git a/src/lib/server/core/config.spec.ts b/src/lib/server/core/config.spec.ts index c01a0ea..790a146 100644 --- a/src/lib/server/core/config.spec.ts +++ b/src/lib/server/core/config.spec.ts @@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const ENV_KEYS = [ 'OPENAI_API_KEY', 'REALTIME_TOKEN_SECRET', + 'REALTIME_PROVIDER', 'PORT', 'ANALYTICS_ENABLED', 'ANALYTICS_DB_PATH', @@ -57,6 +58,7 @@ describe('loadConfig', () => { expect(loadConfig()).toEqual({ port: 3000, + realtimeProvider: 'openai', openaiApiKey: 'test-key', analytics: { enabled: true, @@ -98,6 +100,7 @@ describe('loadConfig', () => { expect(loadConfig()).toEqual({ port: 8080, + realtimeProvider: 'openai', openaiApiKey: 'test-key', analytics: { enabled: false, @@ -124,6 +127,29 @@ describe('loadConfig', () => { expect(loadConfig().port).toBe(3000); }); + it('does not require OPENAI_API_KEY when REALTIME_PROVIDER=mock', async () => { + process.env.REALTIME_TOKEN_SECRET = 'test-secret'; + process.env.REALTIME_PROVIDER = 'mock'; + const loadConfig = await freshLoadConfig(); + + const config = loadConfig(); + expect(config.realtimeProvider).toBe('mock'); + expect(config.openaiApiKey).toBe(''); + }); + + it('still requires OPENAI_API_KEY when REALTIME_PROVIDER is left unset (defaults to openai)', async () => { + process.env.REALTIME_TOKEN_SECRET = 'test-secret'; + const loadConfig = await freshLoadConfig(); + + expect(() => loadConfig()).toThrow(/OPENAI_API_KEY/); + }); + + it('throws a descriptive error for an invalid REALTIME_PROVIDER value', async () => { + Object.assign(process.env, REQUIRED_ENV, { REALTIME_PROVIDER: 'anthropic' }); + const loadConfig = await freshLoadConfig(); + expect(() => loadConfig()).toThrow(/REALTIME_PROVIDER must be one of openai, mock/); + }); + it('throws mentioning every missing required var when none are set', async () => { const loadConfig = await freshLoadConfig(); expect(() => loadConfig()).toThrow(/OPENAI_API_KEY/); diff --git a/src/lib/server/core/config.ts b/src/lib/server/core/config.ts index d5a85eb..dabb35d 100644 --- a/src/lib/server/core/config.ts +++ b/src/lib/server/core/config.ts @@ -24,6 +24,9 @@ export interface SessionRuntimeConfig { export interface AppConfig { port: number; + /** Which `RealtimeProvider` to construct — see `providers/index.ts`'s `selectProvider`. */ + realtimeProvider: 'openai' | 'mock'; + /** Only required (and only meaningful) when `realtimeProvider` is `'openai'`. */ openaiApiKey: string; analytics: AnalyticsConfig; session: SessionRuntimeConfig; @@ -54,7 +57,19 @@ export function loadConfig(): AppConfig { const errors: string[] = []; const port = parseIntEnv('PORT', 3000, { min: 1, max: 65535 }, errors); - const openaiApiKey = requireEnv('OPENAI_API_KEY', errors); + const realtimeProvider = parseEnumEnv( + 'REALTIME_PROVIDER', + 'openai', + ['openai', 'mock'] as const, + errors + ); + // Only required when actually talking to OpenAI — the mock provider (see + // providers/mock.ts) exists specifically so animation/UI work doesn't need + // a real key at all. + const openaiApiKey = + realtimeProvider === 'openai' + ? requireEnv('OPENAI_API_KEY', errors) + : (optionalEnv('OPENAI_API_KEY') ?? ''); requireEnv('REALTIME_TOKEN_SECRET', errors); // presence only — see doc comment above const analytics: AnalyticsConfig = { @@ -76,7 +91,7 @@ export function loadConfig(): AppConfig { throw new Error(`Invalid configuration:\n${errors.map((e) => ` - ${e}`).join('\n')}`); } - cached = { port, openaiApiKey, analytics, session }; + cached = { port, realtimeProvider, openaiApiKey, analytics, session }; return cached; } @@ -89,6 +104,19 @@ function parseBoolEnv(name: string, def: boolean, errors: string[]): boolean { return def; } +function parseEnumEnv( + name: string, + def: T, + allowed: readonly T[], + errors: string[] +): T { + const raw = process.env[name]; + if (raw === undefined || raw === '') return def; + if ((allowed as readonly string[]).includes(raw)) return raw as T; + errors.push(`${name} must be one of ${allowed.join(', ')} if set (got "${raw}").`); + return def; +} + function parseIntEnv( name: string, def: number, diff --git a/src/lib/server/core/dev-plugin.ts b/src/lib/server/core/dev-plugin.ts index a82fa2b..6bc8e4e 100644 --- a/src/lib/server/core/dev-plugin.ts +++ b/src/lib/server/core/dev-plugin.ts @@ -3,7 +3,7 @@ import type { SessionPersona } from '../../project/persona.js'; import { NoopAnalyticsRecorder, SqliteAnalyticsRecorder } from './analytics.js'; import { loadConfig } from './config.js'; import { resolveSessionDefaults } from './defaults.js'; -import { OpenAiRealtimeProvider } from './providers/openai.js'; +import { selectProvider } from './providers/index.js'; import type { ToolRegistry } from './tools.js'; import { handleRealtimeUpgrade } from './ws-handler.js'; @@ -51,7 +51,7 @@ export function realtimeDevWsPlugin(options: RealtimeDevWsPluginOptions): Plugin } const config = loadConfig(); - const provider = new OpenAiRealtimeProvider({ apiKey: config.openaiApiKey }); + const provider = selectProvider(config); const recorder = config.analytics.enabled ? new SqliteAnalyticsRecorder(config.analytics.dbPath, config.analytics) : new NoopAnalyticsRecorder(); diff --git a/src/lib/server/core/providers/fixtures/default.ts b/src/lib/server/core/providers/fixtures/default.ts new file mode 100644 index 0000000..5f36112 --- /dev/null +++ b/src/lib/server/core/providers/fixtures/default.ts @@ -0,0 +1,19 @@ +import type { MockScript } from '../mock.js'; + +/** + * Short canned exchange for exercising transcript rendering and (via + * output_audio.chunk/done timing) humanoid avatar-state derivation without a + * real API key. `audio` is an empty string — nothing decodes/plays it yet (no + * client-side audio capture/playback pipeline exists in core today), only its + * start/stop timing matters here. + */ +export const DEFAULT_MOCK_SCRIPT: MockScript = [ + { delayMs: 300, event: { type: 'output_audio.chunk', audio: '' } }, + { + delayMs: 200, + event: { type: 'transcript.delta', role: 'assistant', text: "Hey, I'm a mock avatar reply." } + }, + { delayMs: 1200, event: { type: 'transcript.done', role: 'assistant' } }, + { delayMs: 0, event: { type: 'output_audio.done' } }, + { delayMs: 0, event: { type: 'response.done' } } +]; diff --git a/src/lib/server/core/providers/index.ts b/src/lib/server/core/providers/index.ts new file mode 100644 index 0000000..4b064fd --- /dev/null +++ b/src/lib/server/core/providers/index.ts @@ -0,0 +1,17 @@ +import type { AppConfig } from '../config.js'; +import type { RealtimeProvider } from '../provider.js'; +import { DEFAULT_MOCK_SCRIPT } from './fixtures/default.js'; +import { MockRealtimeProvider } from './mock.js'; +import { OpenAiRealtimeProvider } from './openai.js'; + +/** + * Picks the `RealtimeProvider` implementation for `config.realtimeProvider` + * — the one place both composition roots (`server.js`, `dev-plugin.ts`) call, + * instead of each hardcoding `new OpenAiRealtimeProvider(...)` directly. + */ +export function selectProvider(config: AppConfig): RealtimeProvider { + if (config.realtimeProvider === 'mock') { + return new MockRealtimeProvider(DEFAULT_MOCK_SCRIPT); + } + return new OpenAiRealtimeProvider({ apiKey: config.openaiApiKey }); +} diff --git a/src/lib/server/core/providers/mock.spec.ts b/src/lib/server/core/providers/mock.spec.ts new file mode 100644 index 0000000..b33c285 --- /dev/null +++ b/src/lib/server/core/providers/mock.spec.ts @@ -0,0 +1,56 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { ServerEvent } from '../../../core/protocol.js'; +import { MockRealtimeProvider, type MockScript } from './mock.js'; + +const SCRIPT: MockScript = [ + { delayMs: 100, event: { type: 'transcript.delta', role: 'assistant', text: 'hi' } }, + { delayMs: 50, event: { type: 'transcript.done', role: 'assistant' } } +]; + +beforeEach(() => { + vi.useFakeTimers(); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('MockRealtimeProvider', () => { + it('replays the script in order, at the cumulative scheduled delay', async () => { + const provider = new MockRealtimeProvider(SCRIPT); + const session = await provider.createSession(); + const received: ServerEvent[] = []; + session.onEvent((event) => received.push(event)); + + await vi.advanceTimersByTimeAsync(99); + expect(received).toEqual([]); + + await vi.advanceTimersByTimeAsync(1); // total 100ms — first step fires + expect(received).toEqual([{ type: 'transcript.delta', role: 'assistant', text: 'hi' }]); + + await vi.advanceTimersByTimeAsync(50); // total 150ms — second step fires + expect(received).toEqual([ + { type: 'transcript.delta', role: 'assistant', text: 'hi' }, + { type: 'transcript.done', role: 'assistant' } + ]); + }); + + it('close() cancels any not-yet-fired steps', async () => { + const provider = new MockRealtimeProvider(SCRIPT); + const session = await provider.createSession(); + const received: ServerEvent[] = []; + session.onEvent((event) => received.push(event)); + + session.close(); + await vi.advanceTimersByTimeAsync(1000); + + expect(received).toEqual([]); + }); + + it('send() is a no-op — the script does not react to browser input', async () => { + const provider = new MockRealtimeProvider(SCRIPT); + const session = await provider.createSession(); + + expect(() => session.send({ type: 'input_text.send', text: 'hello' })).not.toThrow(); + }); +}); diff --git a/src/lib/server/core/providers/mock.ts b/src/lib/server/core/providers/mock.ts new file mode 100644 index 0000000..adf58db --- /dev/null +++ b/src/lib/server/core/providers/mock.ts @@ -0,0 +1,54 @@ +import type { ServerEvent } from '../../../core/protocol.js'; +import type { ProviderSession, RealtimeProvider } from '../provider.js'; + +/** One scripted event, replayed `delayMs` after the previous step (or after `createSession` for the first step). */ +export interface MockScriptStep { + delayMs: number; + event: ServerEvent; +} + +export type MockScript = MockScriptStep[]; + +/** + * `RealtimeProvider` that replays a canned `MockScript` on a timer instead of + * talking to a real upstream — lets client work (transcript rendering, avatar + * state, animation timing) happen without an API key or live usage. See + * docs/REALTIME_SESSION.md's "Development without a live API" section. + * + * Deliberately simple: linear, one-shot replay. `send()` is a no-op — the + * script doesn't react to what the browser sends. Interactive/branching + * fixtures are a plausible future enhancement, not needed yet. + */ +export class MockRealtimeProvider implements RealtimeProvider { + constructor(private readonly script: MockScript) {} + + // Ignores `config` — the script is canned, not generated from session config. + async createSession(): Promise { + return new MockProviderSession(this.script); + } +} + +class MockProviderSession implements ProviderSession { + private handler: ((event: ServerEvent) => void) | null = null; + private readonly timers: NodeJS.Timeout[] = []; + + constructor(script: MockScript) { + let elapsed = 0; + for (const step of script) { + elapsed += step.delayMs; + this.timers.push(setTimeout(() => this.handler?.(step.event), elapsed)); + } + } + + // No-op — see the class doc comment on MockRealtimeProvider. + send(): void {} + + onEvent(handler: (event: ServerEvent) => void): void { + this.handler = handler; + } + + close(): void { + for (const timer of this.timers) clearTimeout(timer); + this.timers.length = 0; + } +} diff --git a/src/routes/demo/humanoid/+page.svelte b/src/routes/demo/humanoid/+page.svelte new file mode 100644 index 0000000..82bd066 --- /dev/null +++ b/src/routes/demo/humanoid/+page.svelte @@ -0,0 +1,107 @@ + + +

Humanoid avatar-state demo

+

+ Exercises $lib/humanoid's state derivation without a live API key: run + REALTIME_PROVIDER=mock npm run dev (dev-server restart required — see AGENTS.md's HMR + gotcha) and connect below to watch avatarState move through the mock provider's default + fixture. +

+

Status: {status}{statusMessage ? ` — ${statusMessage}` : ''}

+

Avatar state: {avatarState}

+ +{#if status === 'idle' || status === 'error' || status === 'closed'} + +{:else if status === 'ready'} + + +{/if} diff --git a/src/routes/demo/realtime/+page.svelte b/src/routes/demo/realtime/+page.svelte index 56bb13c..345416e 100644 --- a/src/routes/demo/realtime/+page.svelte +++ b/src/routes/demo/realtime/+page.svelte @@ -1,10 +1,13 @@