From c81b9ac9532ab256bfbd8b998fc4e76311d937f0 Mon Sep 17 00:00:00 2001 From: Jonathan Hinkel Date: Wed, 26 Aug 2026 13:27:48 +0200 Subject: [PATCH] Split humanoid AvatarState into idleActive/idlePassive by default Ported animation systems (from an Unreal Engine avatar app) key idle behavior directly off this distinction, so it moves from an unbuilt extension point to a core state. The pure state.ts reducer stays timer-free (idleActive -> idlePassive rides in on an explicit idle_timeout signal); deriveAvatarState in the client wrapper owns the actual clock via a configurable idlePassiveTimeoutMs (default 30s). Co-Authored-By: Claude Sonnet 5 --- docs/HUMANOID.md | 14 ++-- src/lib/client/humanoid/avatar-state.spec.ts | 68 +++++++++++++++++--- src/lib/client/humanoid/avatar-state.ts | 49 +++++++++++++- src/lib/humanoid/state.spec.ts | 30 ++++++--- src/lib/humanoid/state.ts | 28 +++++--- src/routes/demo/humanoid/+page.svelte | 2 +- 6 files changed, 153 insertions(+), 38 deletions(-) diff --git a/docs/HUMANOID.md b/docs/HUMANOID.md index b2c4684..ca3b3e5 100644 --- a/docs/HUMANOID.md +++ b/docs/HUMANOID.md @@ -7,18 +7,18 @@ This package ships the **contract** future animation libraries (eye look-at, bli ## `$lib/humanoid/state.ts` — avatar state ```ts -type AvatarState = 'idle' | 'listening' | 'thinking' | 'talking'; +type AvatarState = 'idleActive' | 'idlePassive' | 'listening' | 'thinking' | 'talking'; ``` -Deliberately just four states, not the `IdleActive`/`IdlePassive` split a humanoid rig's own animation design (e.g. a future eye look-at library) may eventually want — that distinction needs a user-presence signal (webcam/face-tracking) this package has no opinion on. A project with that signal can layer its own finer idle split on top; `idle` here just means "no signal that the user or the model is currently doing anything." +`idle` is split into `idleActive`/`idlePassive` from the start, since ported animation systems (this one's lineage traces to an Unreal Engine avatar app) commonly key their idle behavior off exactly this distinction: `idleActive` right after a turn ends (still "engaged"), settling into `idlePassive` once nothing has happened for a while (bored/ambient idle animation). The reducer itself has no timers — pure and isomorphic — so the active→passive transition rides in on an explicit `idle_timeout` signal; the client wrapper (`deriveAvatarState`, below) owns the actual clock, same as it owns `setListening`. -`nextAvatarState(current, signal)` is a pure reducer — no WS/browser dependency, directly unit-tested in `state.spec.ts`. The signal set is small and closed (`listening_started`/`listening_stopped`, `response_requested`, `talking_started`/`talking_stopped`, `reset`). +`nextAvatarState(current, signal)` is a pure reducer — no WS/browser dependency, directly unit-tested in `state.spec.ts`. The signal set is small and closed (`listening_started`/`listening_stopped`, `response_requested`, `talking_started`/`talking_stopped`, `idle_timeout`, `reset`). -Note `listening_stopped` moves to `thinking`, not `idle` — ending listening almost always means a turn just completed and a response is expected next, so the avatar shouldn't visibly relax for the split second before the model's response starts arriving. +Note `listening_stopped` moves to `thinking`, not back to idle — ending listening almost always means a turn just completed and a response is expected next, so the avatar shouldn't visibly relax for the split second before the model's response starts arriving. `talking_stopped` and `reset` both land on `idleActive`, never straight to `idlePassive` — passive idle is only ever reached via `idle_timeout`. ## `$lib/client/humanoid/avatar-state.ts` — deriving it live -`deriveAvatarState(source)` takes a `RealtimeClient` or `ResilientRealtimeClient` (accepted structurally, via an `AvatarStateSource` interface — this package doesn't care which transport variant is in use) and folds its `ServerEvent`s through `nextAvatarState`: +`deriveAvatarState(source, options?)` takes a `RealtimeClient` or `ResilientRealtimeClient` (accepted structurally, via an `AvatarStateSource` interface — this package doesn't care which transport variant is in use) and folds its `ServerEvent`s through `nextAvatarState`: | `ServerEvent` | Signal | | ------------------------------------- | ----------------- | @@ -33,12 +33,14 @@ Two signals can't come from the event stream alone, so the returned store expose - **`setListening(active)`** — call from whatever mic-capture code a project uses. That pipeline doesn't exist in `core` yet (see "Known limitations" below), and the client is the only thing that currently knows when the user starts/stops talking: the protocol doesn't surface OpenAI's `input_audio_buffer.speech_*` VAD events — see the "not part of our normalized vocabulary" comment in `server/core/providers/openai.ts`'s `fromOpenAiEvent`. - **`notifyResponseRequested()`** — call right when sending a typed `input_text.send` (the voice path already gets this for free from `setListening(false)`). +This wrapper also owns the `idleActive` → `idlePassive` clock: the store starts (and every `reset`/`talking_stopped` returns to) `idleActive`, and a timer — `options.idlePassiveTimeoutMs`, default `DEFAULT_IDLE_PASSIVE_TIMEOUT_MS` (30s) — fires `idle_timeout` if nothing else happens first. Any other signal re-arms it on the next `idleActive` entry; `destroy()` cancels it. + The returned `AvatarStateStore.subscribe()` follows the Svelte store contract (calls `run` immediately with the current value, then on every change), so it's usable directly as `$store` in a `.svelte` file without a hard Svelte dependency in this package itself. See `/demo/humanoid` (`src/routes/demo/humanoid/+page.svelte`) for the reference wiring, including the manual "listening" toggle standing in for real mic-capture UI. ### Known limitations - **`thinking` is a heuristic**, not a precise reflection of server-side activity — the time between a request and the assistant's first output. The browser never sees server-executed tool calls at all (see [TOOLS.md](TOOLS.md)), so "the model is running a tool" isn't observable from here. -- **No presence-based idle split** (see above) — a project with webcam/face-tracking data can extend on top. +- **The idle split is timer-based, not presence-based** — `idlePassive` is reached purely by elapsed time with no other signal, not by knowing whether the user actually left. A project with webcam/face-tracking data can layer a real presence signal on top (e.g. calling back into `idleActive` early, or suppressing the timeout) — that stays project-owned. ## `$lib/humanoid/animation.ts` — animation-channel composition diff --git a/src/lib/client/humanoid/avatar-state.spec.ts b/src/lib/client/humanoid/avatar-state.spec.ts index 5c677f2..dd2e780 100644 --- a/src/lib/client/humanoid/avatar-state.spec.ts +++ b/src/lib/client/humanoid/avatar-state.spec.ts @@ -1,4 +1,4 @@ -import { describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { ServerEvent } from '../../core/protocol.js'; import { deriveAvatarState, type AvatarStateSource } from './avatar-state.js'; @@ -19,15 +19,23 @@ class FakeSource implements AvatarStateSource { } describe('deriveAvatarState', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + it('subscribe() emits the current state immediately', () => { const store = deriveAvatarState(new FakeSource()); const run = vi.fn(); store.subscribe(run); expect(run).toHaveBeenCalledTimes(1); - expect(run).toHaveBeenCalledWith('idle'); + expect(run).toHaveBeenCalledWith('idleActive'); }); - it('walks listening -> thinking -> talking -> idle across a full voice turn', () => { + it('walks listening -> thinking -> talking -> idleActive across a full voice turn', () => { const source = new FakeSource(); const store = deriveAvatarState(source); const seen: string[] = []; @@ -38,7 +46,7 @@ describe('deriveAvatarState', () => { source.emit({ type: 'output_audio.chunk', audio: '' }); source.emit({ type: 'output_audio.done' }); - expect(seen).toEqual(['idle', 'listening', 'thinking', 'talking', 'idle']); + expect(seen).toEqual(['idleActive', 'listening', 'thinking', 'talking', 'idleActive']); }); it('notifyResponseRequested() drives thinking for the typed-text path', () => { @@ -51,7 +59,7 @@ describe('deriveAvatarState', () => { source.emit({ type: 'transcript.delta', role: 'assistant', text: 'hi' }); source.emit({ type: 'transcript.done', role: 'assistant' }); - expect(seen).toEqual(['idle', 'thinking', 'talking', 'idle']); + expect(seen).toEqual(['idleActive', 'thinking', 'talking', 'idleActive']); }); it('ignores user transcript events (no signal either way)', () => { @@ -62,10 +70,10 @@ describe('deriveAvatarState', () => { source.emit({ type: 'transcript.delta', role: 'user', text: 'hello' }); - expect(seen).toEqual(['idle']); + expect(seen).toEqual(['idleActive']); }); - it('session.ready and session.closed reset to idle from any state', () => { + it('session.ready and session.closed reset to idleActive from any state', () => { const source = new FakeSource(); const store = deriveAvatarState(source); const seen: string[] = []; @@ -74,7 +82,7 @@ describe('deriveAvatarState', () => { source.emit({ type: 'output_audio.chunk', audio: '' }); // -> talking source.emit({ type: 'session.closed', reason: 'client_requested' }); - expect(seen).toEqual(['idle', 'talking', 'idle']); + expect(seen).toEqual(['idleActive', 'talking', 'idleActive']); }); it('response.done is a fallback that stops talking even without output_audio.done', () => { @@ -86,7 +94,7 @@ describe('deriveAvatarState', () => { source.emit({ type: 'output_audio.chunk', audio: '' }); source.emit({ type: 'response.done' }); - expect(seen).toEqual(['idle', 'talking', 'idle']); + expect(seen).toEqual(['idleActive', 'talking', 'idleActive']); }); it('destroy() stops reacting to further events', () => { @@ -98,6 +106,46 @@ describe('deriveAvatarState', () => { store.destroy(); source.emit({ type: 'output_audio.chunk', audio: '' }); - expect(seen).toEqual(['idle']); + expect(seen).toEqual(['idleActive']); + }); + + it('settles idleActive into idlePassive after the configured timeout with no other signal', () => { + const source = new FakeSource(); + const store = deriveAvatarState(source, { idlePassiveTimeoutMs: 5_000 }); + const seen: string[] = []; + store.subscribe((s) => seen.push(s)); + + vi.advanceTimersByTime(5_000); + + expect(seen).toEqual(['idleActive', 'idlePassive']); + }); + + it('does not fire idle_timeout while something else is happening', () => { + const source = new FakeSource(); + const store = deriveAvatarState(source, { idlePassiveTimeoutMs: 5_000 }); + const seen: string[] = []; + store.subscribe((s) => seen.push(s)); + + vi.advanceTimersByTime(4_000); + store.setListening(true); // -> listening, cancels/re-arms the idle timer + vi.advanceTimersByTime(4_000); // < 5s since re-arm — still listening + store.setListening(false); // -> thinking + + expect(seen).toEqual(['idleActive', 'listening', 'thinking']); + }); + + it('re-arms the idle timer once back in idleActive, and destroy() cancels it', () => { + const source = new FakeSource(); + const store = deriveAvatarState(source, { idlePassiveTimeoutMs: 5_000 }); + const seen: string[] = []; + store.subscribe((s) => seen.push(s)); + + source.emit({ type: 'output_audio.chunk', audio: '' }); // -> talking + source.emit({ type: 'output_audio.done' }); // -> idleActive, re-arms timer + + store.destroy(); + vi.advanceTimersByTime(5_000); + + expect(seen).toEqual(['idleActive', 'talking', 'idleActive']); }); }); diff --git a/src/lib/client/humanoid/avatar-state.ts b/src/lib/client/humanoid/avatar-state.ts index 17480b9..0e6b65a 100644 --- a/src/lib/client/humanoid/avatar-state.ts +++ b/src/lib/client/humanoid/avatar-state.ts @@ -29,10 +29,24 @@ export interface AvatarStateStore { * gets this for free from `setListening(false)`). */ notifyResponseRequested(): void; - /** Stops listening to the source and releases all subscribers. */ + /** Stops listening to the source, cancels the idle timer, and releases all subscribers. */ destroy(): void; } +/** Default `idlePassiveTimeoutMs` — see `DeriveAvatarStateOptions`. */ +export const DEFAULT_IDLE_PASSIVE_TIMEOUT_MS = 30_000; + +export interface DeriveAvatarStateOptions { + /** + * How long the avatar sits in `idleActive` with no other signal before + * this wrapper fires the `idle_timeout` signal that settles it into + * `idlePassive`. The pure reducer in `$lib/humanoid/state.ts` has no + * timers of its own, so this wrapper is what actually owns the clock — + * tune to whatever cadence a project's idle animation set expects. + */ + idlePassiveTimeoutMs?: number; +} + /** * Derives a reactive `AvatarState` from a live `RealtimeClient` or * `ResilientRealtimeClient`. Translates `ServerEvent`s (plus the imperative @@ -44,18 +58,46 @@ export interface AvatarStateStore { * assistant's first output — not a precise reflection of server-side * activity: the browser never sees server-executed tool calls at all (see * docs/TOOLS.md), so "the model is running a tool" isn't observable here. + * + * Also owns the `idleActive` -> `idlePassive` clock: the store starts (and + * every `reset`/`talking_stopped` returns to) `idleActive`, and this + * function arms a timer — `options.idlePassiveTimeoutMs`, default + * `DEFAULT_IDLE_PASSIVE_TIMEOUT_MS` — that fires `idle_timeout` if nothing + * else happens first. */ -export function deriveAvatarState(source: AvatarStateSource): AvatarStateStore { - let state: AvatarState = 'idle'; +export function deriveAvatarState( + source: AvatarStateSource, + options: DeriveAvatarStateOptions = {} +): AvatarStateStore { + const idlePassiveTimeoutMs = options.idlePassiveTimeoutMs ?? DEFAULT_IDLE_PASSIVE_TIMEOUT_MS; + let state: AvatarState = 'idleActive'; const subscribers = new Set<(state: AvatarState) => void>(); + let idleTimer: ReturnType | null = null; + + // Re-armed after every transition: scheduled only while `idleActive` (the + // one state `idle_timeout` acts on), cleared everywhere else so a stale + // timer from a prior idleActive spell can't fire `idle_timeout` while + // e.g. the avatar is now talking. + function syncIdleTimer(): void { + if (idleTimer !== null) { + clearTimeout(idleTimer); + idleTimer = null; + } + if (state === 'idleActive') { + idleTimer = setTimeout(() => apply({ type: 'idle_timeout' }), idlePassiveTimeoutMs); + } + } function apply(signal: AvatarSignal): void { const next = nextAvatarState(state, signal); if (next === state) return; state = next; + syncIdleTimer(); for (const run of subscribers) run(state); } + syncIdleTimer(); + const stopEvents = source.on((event) => { switch (event.type) { case 'session.ready': @@ -93,6 +135,7 @@ export function deriveAvatarState(source: AvatarStateSource): AvatarStateStore { destroy() { stopEvents(); subscribers.clear(); + if (idleTimer !== null) clearTimeout(idleTimer); } }; } diff --git a/src/lib/humanoid/state.spec.ts b/src/lib/humanoid/state.spec.ts index e9c2007..c452601 100644 --- a/src/lib/humanoid/state.spec.ts +++ b/src/lib/humanoid/state.spec.ts @@ -3,17 +3,19 @@ import { nextAvatarState } from './state.js'; describe('nextAvatarState', () => { it('listening_started always moves to listening', () => { - expect(nextAvatarState('idle', { type: 'listening_started' })).toBe('listening'); + expect(nextAvatarState('idleActive', { type: 'listening_started' })).toBe('listening'); + expect(nextAvatarState('idlePassive', { type: 'listening_started' })).toBe('listening'); expect(nextAvatarState('talking', { type: 'listening_started' })).toBe('listening'); }); it('listening_stopped moves to thinking (a response is expected next), only from listening', () => { expect(nextAvatarState('listening', { type: 'listening_stopped' })).toBe('thinking'); - expect(nextAvatarState('idle', { type: 'listening_stopped' })).toBe('idle'); + expect(nextAvatarState('idleActive', { type: 'listening_stopped' })).toBe('idleActive'); }); it('response_requested moves to thinking unless already talking', () => { - expect(nextAvatarState('idle', { type: 'response_requested' })).toBe('thinking'); + expect(nextAvatarState('idleActive', { type: 'response_requested' })).toBe('thinking'); + expect(nextAvatarState('idlePassive', { type: 'response_requested' })).toBe('thinking'); expect(nextAvatarState('listening', { type: 'response_requested' })).toBe('thinking'); expect(nextAvatarState('talking', { type: 'response_requested' })).toBe('talking'); }); @@ -22,13 +24,23 @@ describe('nextAvatarState', () => { expect(nextAvatarState('thinking', { type: 'talking_started' })).toBe('talking'); }); - it('talking_stopped only returns to idle from talking', () => { - expect(nextAvatarState('talking', { type: 'talking_stopped' })).toBe('idle'); - expect(nextAvatarState('idle', { type: 'talking_stopped' })).toBe('idle'); + it('talking_stopped only returns to idleActive from talking', () => { + expect(nextAvatarState('talking', { type: 'talking_stopped' })).toBe('idleActive'); + expect(nextAvatarState('idleActive', { type: 'talking_stopped' })).toBe('idleActive'); + expect(nextAvatarState('idlePassive', { type: 'talking_stopped' })).toBe('idlePassive'); }); - it('reset always returns to idle', () => { - expect(nextAvatarState('talking', { type: 'reset' })).toBe('idle'); - expect(nextAvatarState('listening', { type: 'reset' })).toBe('idle'); + it('idle_timeout moves idleActive to idlePassive, and is a no-op everywhere else', () => { + expect(nextAvatarState('idleActive', { type: 'idle_timeout' })).toBe('idlePassive'); + expect(nextAvatarState('idlePassive', { type: 'idle_timeout' })).toBe('idlePassive'); + expect(nextAvatarState('listening', { type: 'idle_timeout' })).toBe('listening'); + expect(nextAvatarState('thinking', { type: 'idle_timeout' })).toBe('thinking'); + expect(nextAvatarState('talking', { type: 'idle_timeout' })).toBe('talking'); + }); + + it('reset always returns to idleActive', () => { + expect(nextAvatarState('talking', { type: 'reset' })).toBe('idleActive'); + expect(nextAvatarState('listening', { type: 'reset' })).toBe('idleActive'); + expect(nextAvatarState('idlePassive', { type: 'reset' })).toBe('idleActive'); }); }); diff --git a/src/lib/humanoid/state.ts b/src/lib/humanoid/state.ts index 10740b7..3f685f6 100644 --- a/src/lib/humanoid/state.ts +++ b/src/lib/humanoid/state.ts @@ -6,14 +6,18 @@ * look-at, blink, lip-sync, body/gesture) can all key off the same signal * without each re-deriving turn-taking themselves. * - * Deliberately just four states, not the `IdleActive`/`IdlePassive` split a - * humanoid rig's own animation design may eventually want — that - * distinction needs a user-presence signal (webcam/face-tracking) this - * package has no opinion on. A project with that signal can layer its own - * finer-grained idle split on top; `idle` here just means "no signal that - * the user or the model is currently doing anything." + * `idle` is split into `idleActive`/`idlePassive` from the start — ported + * animation systems (this one's lineage traces to an Unreal Engine avatar + * app) commonly key their idle behavior off exactly this distinction: + * `idleActive` right after a turn ends (still "engaged"), `idlePassive` + * once nothing has happened for a while (settle into bored/ambient idle + * animation). This reducer has no timers of its own — pure and isomorphic — + * so the active→passive transition rides in on an explicit `idle_timeout` + * signal rather than an internal side effect; the client wrapper + * (`$lib/client/humanoid/avatar-state.ts`) owns the actual timer, same as + * it owns `setListening`. */ -export type AvatarState = 'idle' | 'listening' | 'thinking' | 'talking'; +export type AvatarState = 'idleActive' | 'idlePassive' | 'listening' | 'thinking' | 'talking'; export type AvatarSignal = | { type: 'listening_started' } @@ -21,6 +25,7 @@ export type AvatarSignal = | { type: 'response_requested' } | { type: 'talking_started' } | { type: 'talking_stopped' } + | { type: 'idle_timeout' } | { type: 'reset' }; /** @@ -45,9 +50,14 @@ export function nextAvatarState(current: AvatarState, signal: AvatarSignal): Ava case 'talking_started': return 'talking'; case 'talking_stopped': - return current === 'talking' ? 'idle' : current; + return current === 'talking' ? 'idleActive' : current; + // Only fires the active->passive transition; a client that emits this + // while something else is happening (listening/thinking/talking) is + // racing its own timer against real activity, so it's a no-op there. + case 'idle_timeout': + return current === 'idleActive' ? 'idlePassive' : current; case 'reset': - return 'idle'; + return 'idleActive'; default: return current; } diff --git a/src/routes/demo/humanoid/+page.svelte b/src/routes/demo/humanoid/+page.svelte index 82bd066..a69516b 100644 --- a/src/routes/demo/humanoid/+page.svelte +++ b/src/routes/demo/humanoid/+page.svelte @@ -11,7 +11,7 @@ let status = $state('idle'); let statusMessage = $state(''); - let avatarState = $state('idle'); + let avatarState = $state('idleActive'); let listening = $state(false); let client: ResilientRealtimeClient | null = null;