Browse Source
Provider-agnostic session layer for the base project: a shared client/server wire protocol, a RealtimeProvider adapter interface, and an OpenAI Realtime implementation of it, wired through a WebSocket relay (browser <-> our SvelteKit server <-> OpenAI) so the API key never reaches the browser and future providers/pipelines can be added without touching client code. Switches the adapter to adapter-node and adds a custom server.js + dev-mode Vite plugin, since SvelteKit has no native WebSocket support yet. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>main
22 changed files with 1939 additions and 30 deletions
@ -0,0 +1,11 @@ |
|||||
|
{ |
||||
|
"version": "0.0.1", |
||||
|
"configurations": [ |
||||
|
{ |
||||
|
"name": "webavatar-dev", |
||||
|
"runtimeExecutable": "npm", |
||||
|
"runtimeArgs": ["run", "dev"], |
||||
|
"port": 5173 |
||||
|
} |
||||
|
] |
||||
|
} |
||||
@ -0,0 +1,6 @@ |
|||||
|
# OpenAI API key used server-side only (never sent to the browser). |
||||
|
OPENAI_API_KEY= |
||||
|
|
||||
|
# Secret used to sign short-lived browser session tokens for the realtime WS handshake. |
||||
|
# Generate one with: node -e "console.log(require('crypto').randomBytes(32).toString('hex'))" |
||||
|
REALTIME_TOKEN_SECRET= |
||||
@ -0,0 +1,30 @@ |
|||||
|
// Custom production server: wraps adapter-node's generated request handler in
|
||||
|
// a plain http.Server so we can also intercept WebSocket upgrade requests,
|
||||
|
// which SvelteKit itself has no hook for (see src/lib/server/core/ws-handler.ts).
|
||||
|
//
|
||||
|
// `.env` is not auto-loaded here (SvelteKit's own adapter-node doesn't in
|
||||
|
// production either) — run with `node --env-file=.env server.js` locally, or
|
||||
|
// provide OPENAI_API_KEY / REALTIME_TOKEN_SECRET via your deployment's real
|
||||
|
// secret mechanism.
|
||||
|
import { createServer } from 'node:http'; |
||||
|
import { handler } from './build/handler.js'; |
||||
|
import { DEFAULT_SESSION_DEFAULTS } from './build-server/server/core/defaults.js'; |
||||
|
import { OpenAiRealtimeProvider } from './build-server/server/core/providers/openai.js'; |
||||
|
import { handleRealtimeUpgrade } from './build-server/server/core/ws-handler.js'; |
||||
|
|
||||
|
const apiKey = process.env.OPENAI_API_KEY; |
||||
|
if (!apiKey) { |
||||
|
throw new Error('OPENAI_API_KEY is not set. See .env.example.'); |
||||
|
} |
||||
|
|
||||
|
const provider = new OpenAiRealtimeProvider({ apiKey }); |
||||
|
const server = createServer(handler); |
||||
|
|
||||
|
server.on('upgrade', (req, socket, head) => { |
||||
|
handleRealtimeUpgrade(req, socket, head, { provider, defaults: DEFAULT_SESSION_DEFAULTS }); |
||||
|
}); |
||||
|
|
||||
|
const port = process.env.PORT ? Number(process.env.PORT) : 3000; |
||||
|
server.listen(port, () => { |
||||
|
console.log(`listening on port ${port}`); |
||||
|
}); |
||||
@ -0,0 +1,69 @@ |
|||||
|
import { REALTIME_WS_PATH, type ClientEvent, type ServerEvent } from '../../core/protocol.js'; |
||||
|
|
||||
|
export interface RealtimeClientOptions { |
||||
|
/** Origin to connect to. Defaults to the current page's origin. */ |
||||
|
baseUrl?: string; |
||||
|
} |
||||
|
|
||||
|
type ServerEventHandler = (event: ServerEvent) => void; |
||||
|
|
||||
|
/** |
||||
|
* Thin typed WebSocket wrapper around the `$lib/core/protocol` wire contract |
||||
|
* — transport only, no audio capture/playback and no session-start policy. |
||||
|
* Callers fetch their own token (see `POST /api/realtime/session`) and |
||||
|
* decide when/what to send. |
||||
|
*/ |
||||
|
export class RealtimeClient { |
||||
|
private socket: WebSocket | null = null; |
||||
|
private readonly handlers = new Set<ServerEventHandler>(); |
||||
|
|
||||
|
connect(token: string, options: RealtimeClientOptions = {}): Promise<void> { |
||||
|
const base = options.baseUrl ?? location.origin; |
||||
|
const wsUrl = new URL(REALTIME_WS_PATH, base); |
||||
|
wsUrl.protocol = wsUrl.protocol === 'https:' ? 'wss:' : 'ws:'; |
||||
|
wsUrl.searchParams.set('token', token); |
||||
|
|
||||
|
return new Promise<void>((resolve, reject) => { |
||||
|
const socket = new WebSocket(wsUrl); |
||||
|
socket.addEventListener('open', () => resolve(), { once: true }); |
||||
|
socket.addEventListener( |
||||
|
'error', |
||||
|
() => reject(new Error('Failed to connect to the realtime WS endpoint.')), |
||||
|
{ once: true } |
||||
|
); |
||||
|
socket.addEventListener('message', (event) => this.handleMessage(event)); |
||||
|
socket.addEventListener('close', () => { |
||||
|
this.socket = null; |
||||
|
}); |
||||
|
this.socket = socket; |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
send(event: ClientEvent): void { |
||||
|
if (!this.socket || this.socket.readyState !== WebSocket.OPEN) { |
||||
|
throw new Error('RealtimeClient is not connected.'); |
||||
|
} |
||||
|
this.socket.send(JSON.stringify(event)); |
||||
|
} |
||||
|
|
||||
|
/** Returns an unsubscribe function. */ |
||||
|
on(handler: ServerEventHandler): () => void { |
||||
|
this.handlers.add(handler); |
||||
|
return () => this.handlers.delete(handler); |
||||
|
} |
||||
|
|
||||
|
close(): void { |
||||
|
this.socket?.close(); |
||||
|
this.socket = null; |
||||
|
} |
||||
|
|
||||
|
private handleMessage(event: MessageEvent): void { |
||||
|
let parsed: ServerEvent; |
||||
|
try { |
||||
|
parsed = JSON.parse(event.data); |
||||
|
} catch { |
||||
|
return; |
||||
|
} |
||||
|
for (const handler of this.handlers) handler(parsed); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,46 @@ |
|||||
|
/** |
||||
|
* Wire contract between a browser client and our realtime WS relay. |
||||
|
* |
||||
|
* This vocabulary is intentionally our own, not a passthrough of any specific |
||||
|
* provider's event schema (e.g. OpenAI Realtime's `input_audio_buffer.append`, |
||||
|
* `response.create`, etc). Provider adapters under `$lib/server/core/providers` |
||||
|
* translate to/from this shape, which is what keeps the client and the |
||||
|
* session orchestrator provider-agnostic. |
||||
|
*/ |
||||
|
|
||||
|
export const REALTIME_WS_PATH = '/api/realtime/ws'; |
||||
|
|
||||
|
/** |
||||
|
* Client-controllable subset of session config. Deliberately small: things |
||||
|
* like `instructions`, `tools`, and `model` are server-owned (see |
||||
|
* `$lib/server/core/session`) since this relay is shared infrastructure that |
||||
|
* other people's browsers connect to. |
||||
|
*/ |
||||
|
export interface SessionStartConfig { |
||||
|
voice?: string; |
||||
|
} |
||||
|
|
||||
|
export type ClientEvent = |
||||
|
| { type: 'session.start'; config?: SessionStartConfig } |
||||
|
| { type: 'input_audio.chunk'; audio: string } // base64-encoded PCM16
|
||||
|
| { type: 'input_audio.commit' } |
||||
|
| { type: 'input_text.send'; text: string } |
||||
|
| { type: 'response.cancel' } |
||||
|
| { type: 'session.end' }; |
||||
|
|
||||
|
export type RealtimeErrorCode = |
||||
|
| 'auth_expired' |
||||
|
| 'invalid_client_event' |
||||
|
| 'upstream_unavailable' |
||||
|
| 'upstream_error' |
||||
|
| 'internal_error'; |
||||
|
|
||||
|
export type ServerEvent = |
||||
|
| { type: 'session.ready'; sessionId: string } |
||||
|
| { type: 'output_audio.chunk'; audio: string } // base64-encoded PCM16
|
||||
|
| { type: 'output_audio.done' } |
||||
|
| { type: 'transcript.delta'; role: 'user' | 'assistant'; text: string } |
||||
|
| { type: 'transcript.done'; role: 'user' | 'assistant' } |
||||
|
| { type: 'response.done' } |
||||
|
| { type: 'error'; code: RealtimeErrorCode; message: string } |
||||
|
| { type: 'session.closed'; reason: string }; |
||||
@ -0,0 +1,12 @@ |
|||||
|
import type { SessionDefaults } from './session.js'; |
||||
|
|
||||
|
/** |
||||
|
* Server-owned session policy shared by the dev-mode Vite plugin and the |
||||
|
* production custom server, so both environments start sessions identically. |
||||
|
* A real client project will likely replace this with its own |
||||
|
* persona/instructions, possibly per-route rather than a single constant. |
||||
|
*/ |
||||
|
export const DEFAULT_SESSION_DEFAULTS: SessionDefaults = { |
||||
|
model: 'gpt-realtime', |
||||
|
instructions: 'You are a helpful voice assistant for a webavatar demo. Keep responses concise.' |
||||
|
}; |
||||
@ -0,0 +1,42 @@ |
|||||
|
import { loadEnv, type Plugin } from 'vite'; |
||||
|
import { DEFAULT_SESSION_DEFAULTS } from './defaults.js'; |
||||
|
import { OpenAiRealtimeProvider } from './providers/openai.js'; |
||||
|
import { handleRealtimeUpgrade } from './ws-handler.js'; |
||||
|
|
||||
|
/** |
||||
|
* Wires the same upgrade handling used in production (see repo-root |
||||
|
* server.js) into Vite's dev server, so `npm run dev` supports the realtime |
||||
|
* WS endpoint without a production build. |
||||
|
* |
||||
|
* Loads .env itself: this hook runs at dev-server startup, outside |
||||
|
* SvelteKit's own request/module graph, so `$env/...` isn't available here — |
||||
|
* see the note in ./tokens.ts for why this whole module tree reads |
||||
|
* `process.env` directly instead. |
||||
|
*/ |
||||
|
export function realtimeDevWsPlugin(): Plugin { |
||||
|
return { |
||||
|
name: 'webavatar-realtime-ws-dev', |
||||
|
configureServer(server) { |
||||
|
// vitest's browser-mode project boots its own Vite dev server for
|
||||
|
// component tests; this plugin has nothing to do there.
|
||||
|
if (process.env.VITEST) return; |
||||
|
|
||||
|
const loaded = loadEnv('development', process.cwd(), ''); |
||||
|
for (const [key, value] of Object.entries(loaded)) { |
||||
|
if (process.env[key] === undefined) process.env[key] = value; |
||||
|
} |
||||
|
|
||||
|
const apiKey = process.env.OPENAI_API_KEY; |
||||
|
if (!apiKey) { |
||||
|
console.warn( |
||||
|
'[realtime] OPENAI_API_KEY is not set (see .env.example) — realtime sessions will fail to start.' |
||||
|
); |
||||
|
} |
||||
|
const provider = new OpenAiRealtimeProvider({ apiKey: apiKey ?? '' }); |
||||
|
|
||||
|
server.httpServer?.on('upgrade', (req, socket, head) => { |
||||
|
handleRealtimeUpgrade(req, socket, head, { provider, defaults: DEFAULT_SESSION_DEFAULTS }); |
||||
|
}); |
||||
|
} |
||||
|
}; |
||||
|
} |
||||
@ -0,0 +1,36 @@ |
|||||
|
import type { ClientEvent, ServerEvent } from '../../core/protocol.js'; |
||||
|
|
||||
|
/** |
||||
|
* Fully-resolved, server-trusted session config. Never expose this shape to |
||||
|
* the browser directly — `instructions` and `tools` in particular are |
||||
|
* server-owned policy, not client input (see `resolveSessionConfig` in |
||||
|
* `./session`). |
||||
|
*/ |
||||
|
export interface ResolvedSessionConfig { |
||||
|
model: string; |
||||
|
instructions: string; |
||||
|
voice?: string; |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* A single live conversation with a backend model/pipeline. Speaks only the |
||||
|
* normalized `ClientEvent`/`ServerEvent` vocabulary from `$lib/core/protocol` |
||||
|
* — never a provider's own wire format. That constraint is what lets |
||||
|
* `RealtimeSession` (./session.ts) stay identical across providers. |
||||
|
*/ |
||||
|
export interface ProviderSession { |
||||
|
send(event: ClientEvent): void; |
||||
|
/** Only one handler is supported; a session has exactly one consumer. */ |
||||
|
onEvent(handler: (event: ServerEvent) => void): void; |
||||
|
close(reason?: string): void; |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* A backend capable of running realtime sessions — today, a single upstream |
||||
|
* WebSocket to OpenAI Realtime. A future pipeline (STT -> LLM -> TTS) or a |
||||
|
* different model provider implements this same interface; nothing else in |
||||
|
* `$lib/server/core` needs to change. |
||||
|
*/ |
||||
|
export interface RealtimeProvider { |
||||
|
createSession(config: ResolvedSessionConfig): Promise<ProviderSession>; |
||||
|
} |
||||
@ -0,0 +1,91 @@ |
|||||
|
import { describe, expect, it } from 'vitest'; |
||||
|
import { fromOpenAiEvent, toOpenAiEvents } from './openai.js'; |
||||
|
|
||||
|
describe('toOpenAiEvents', () => { |
||||
|
it('translates an audio chunk to input_audio_buffer.append', () => { |
||||
|
expect(toOpenAiEvents({ type: 'input_audio.chunk', audio: 'abc123' })).toEqual([ |
||||
|
{ type: 'input_audio_buffer.append', audio: 'abc123' } |
||||
|
]); |
||||
|
}); |
||||
|
|
||||
|
it('translates an audio commit to input_audio_buffer.commit', () => { |
||||
|
expect(toOpenAiEvents({ type: 'input_audio.commit' })).toEqual([ |
||||
|
{ type: 'input_audio_buffer.commit' } |
||||
|
]); |
||||
|
}); |
||||
|
|
||||
|
it('translates text input into a conversation item plus a response trigger', () => { |
||||
|
expect(toOpenAiEvents({ type: 'input_text.send', text: 'hello' })).toEqual([ |
||||
|
{ |
||||
|
type: 'conversation.item.create', |
||||
|
item: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'hello' }] } |
||||
|
}, |
||||
|
{ type: 'response.create' } |
||||
|
]); |
||||
|
}); |
||||
|
|
||||
|
it('translates response.cancel', () => { |
||||
|
expect(toOpenAiEvents({ type: 'response.cancel' })).toEqual([{ type: 'response.cancel' }]); |
||||
|
}); |
||||
|
|
||||
|
it('produces no wire events for session.start / session.end (handled by the orchestrator)', () => { |
||||
|
expect(toOpenAiEvents({ type: 'session.start' })).toEqual([]); |
||||
|
expect(toOpenAiEvents({ type: 'session.end' })).toEqual([]); |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
describe('fromOpenAiEvent', () => { |
||||
|
it('translates response.output_audio.delta', () => { |
||||
|
expect(fromOpenAiEvent({ type: 'response.output_audio.delta', delta: 'xyz' })).toEqual({ |
||||
|
type: 'output_audio.chunk', |
||||
|
audio: 'xyz' |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
it('translates response.output_audio.done', () => { |
||||
|
expect(fromOpenAiEvent({ type: 'response.output_audio.done' })).toEqual({ |
||||
|
type: 'output_audio.done' |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
it('translates response.output_audio_transcript.delta to an assistant transcript delta', () => { |
||||
|
expect( |
||||
|
fromOpenAiEvent({ type: 'response.output_audio_transcript.delta', delta: 'hi there' }) |
||||
|
).toEqual({ |
||||
|
type: 'transcript.delta', |
||||
|
role: 'assistant', |
||||
|
text: 'hi there' |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
it('translates a completed user transcription to a user transcript delta', () => { |
||||
|
expect( |
||||
|
fromOpenAiEvent({ |
||||
|
type: 'conversation.item.input_audio_transcription.completed', |
||||
|
transcript: 'what is the weather' |
||||
|
}) |
||||
|
).toEqual({ type: 'transcript.delta', role: 'user', text: 'what is the weather' }); |
||||
|
}); |
||||
|
|
||||
|
it('translates response.done', () => { |
||||
|
expect(fromOpenAiEvent({ type: 'response.done' })).toEqual({ type: 'response.done' }); |
||||
|
}); |
||||
|
|
||||
|
it('translates an upstream error, falling back to a default message', () => { |
||||
|
expect(fromOpenAiEvent({ type: 'error', error: { message: 'boom' } })).toEqual({ |
||||
|
type: 'error', |
||||
|
code: 'upstream_error', |
||||
|
message: 'boom' |
||||
|
}); |
||||
|
expect(fromOpenAiEvent({ type: 'error' })).toEqual({ |
||||
|
type: 'error', |
||||
|
code: 'upstream_error', |
||||
|
message: 'Unknown upstream error' |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
it('returns null for events outside our normalized vocabulary', () => { |
||||
|
expect(fromOpenAiEvent({ type: 'session.created' })).toBeNull(); |
||||
|
expect(fromOpenAiEvent({ type: 'rate_limits.updated' })).toBeNull(); |
||||
|
}); |
||||
|
}); |
||||
@ -0,0 +1,174 @@ |
|||||
|
import WebSocket from 'ws'; |
||||
|
import type { ClientEvent, ServerEvent } from '../../../core/protocol.js'; |
||||
|
import type { ProviderSession, RealtimeProvider, ResolvedSessionConfig } from '../provider.js'; |
||||
|
|
||||
|
const REALTIME_URL = 'wss://api.openai.com/v1/realtime'; |
||||
|
|
||||
|
export interface OpenAiRealtimeProviderOptions { |
||||
|
apiKey: string; |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* `RealtimeProvider` backed by a single upstream WebSocket connection to |
||||
|
* OpenAI's Realtime API (server-to-server: the real API key lives only here, |
||||
|
* never handed to a browser). Translates between our normalized |
||||
|
* `ClientEvent`/`ServerEvent` vocabulary and OpenAI's wire event schema. |
||||
|
* |
||||
|
* Always requests `output_modalities: ['audio']` — OpenAI streams an audio |
||||
|
* transcript (`response.audio_transcript.delta`) alongside the audio itself, |
||||
|
* which we surface as `transcript.delta`, so text-only consumers can just |
||||
|
* ignore the `output_audio.*` events rather than needing a separate |
||||
|
* text-only session mode. |
||||
|
*/ |
||||
|
export class OpenAiRealtimeProvider implements RealtimeProvider { |
||||
|
constructor(private readonly options: OpenAiRealtimeProviderOptions) {} |
||||
|
|
||||
|
createSession(config: ResolvedSessionConfig): Promise<ProviderSession> { |
||||
|
return new Promise((resolve, reject) => { |
||||
|
const url = `${REALTIME_URL}?model=${encodeURIComponent(config.model)}`; |
||||
|
const ws = new WebSocket(url, { |
||||
|
headers: { Authorization: `Bearer ${this.options.apiKey}` } |
||||
|
}); |
||||
|
|
||||
|
let settled = false; |
||||
|
|
||||
|
ws.once('open', () => { |
||||
|
ws.send( |
||||
|
JSON.stringify({ |
||||
|
type: 'session.update', |
||||
|
session: { |
||||
|
type: 'realtime', |
||||
|
model: config.model, |
||||
|
instructions: config.instructions, |
||||
|
voice: config.voice, |
||||
|
output_modalities: ['audio'] |
||||
|
} |
||||
|
}) |
||||
|
); |
||||
|
settled = true; |
||||
|
resolve(new OpenAiProviderSession(ws)); |
||||
|
}); |
||||
|
|
||||
|
ws.once('error', (err) => { |
||||
|
if (!settled) { |
||||
|
settled = true; |
||||
|
reject(err instanceof Error ? err : new Error(String(err))); |
||||
|
} |
||||
|
}); |
||||
|
}); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
class OpenAiProviderSession implements ProviderSession { |
||||
|
private handler: ((event: ServerEvent) => void) | null = null; |
||||
|
private closingLocally = false; |
||||
|
|
||||
|
constructor(private readonly ws: WebSocket) { |
||||
|
this.ws.on('message', (data, isBinary) => this.handleUpstreamMessage(data, isBinary)); |
||||
|
this.ws.on('close', () => this.handleUpstreamClose()); |
||||
|
this.ws.on('error', (err) => |
||||
|
this.emit({ type: 'error', code: 'upstream_error', message: err.message }) |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
send(event: ClientEvent): void { |
||||
|
const wireEvents = toOpenAiEvents(event); |
||||
|
for (const wireEvent of wireEvents) { |
||||
|
this.ws.send(JSON.stringify(wireEvent)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
onEvent(handler: (event: ServerEvent) => void): void { |
||||
|
this.handler = handler; |
||||
|
} |
||||
|
|
||||
|
close(): void { |
||||
|
this.closingLocally = true; |
||||
|
if (this.ws.readyState === WebSocket.OPEN || this.ws.readyState === WebSocket.CONNECTING) { |
||||
|
this.ws.close(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private handleUpstreamMessage(data: WebSocket.RawData, isBinary: boolean): void { |
||||
|
if (isBinary) return; // OpenAI Realtime sends JSON text frames only
|
||||
|
let parsed: Record<string, unknown>; |
||||
|
try { |
||||
|
parsed = JSON.parse(data.toString()); |
||||
|
} catch { |
||||
|
return; |
||||
|
} |
||||
|
const event = fromOpenAiEvent(parsed); |
||||
|
if (event) this.emit(event); |
||||
|
} |
||||
|
|
||||
|
private handleUpstreamClose(): void { |
||||
|
if (this.closingLocally) return; |
||||
|
this.emit({ type: 'session.closed', reason: 'upstream_disconnected' }); |
||||
|
} |
||||
|
|
||||
|
private emit(event: ServerEvent): void { |
||||
|
this.handler?.(event); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Exported (alongside fromOpenAiEvent below) for direct unit testing of the
|
||||
|
// translation logic, without spinning up a real upstream WebSocket.
|
||||
|
export function toOpenAiEvents(event: ClientEvent): Array<Record<string, unknown>> { |
||||
|
switch (event.type) { |
||||
|
case 'input_audio.chunk': |
||||
|
return [{ type: 'input_audio_buffer.append', audio: event.audio }]; |
||||
|
case 'input_audio.commit': |
||||
|
return [{ type: 'input_audio_buffer.commit' }]; |
||||
|
case 'input_text.send': |
||||
|
return [ |
||||
|
{ |
||||
|
type: 'conversation.item.create', |
||||
|
item: { |
||||
|
type: 'message', |
||||
|
role: 'user', |
||||
|
content: [{ type: 'input_text', text: event.text }] |
||||
|
} |
||||
|
}, |
||||
|
{ type: 'response.create' } |
||||
|
]; |
||||
|
case 'response.cancel': |
||||
|
return [{ type: 'response.cancel' }]; |
||||
|
// 'session.start' and 'session.end' are handled by the orchestrator
|
||||
|
// (RealtimeSession) before reaching a provider — see ../session.ts.
|
||||
|
default: |
||||
|
return []; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
export function fromOpenAiEvent(raw: Record<string, unknown>): ServerEvent | null { |
||||
|
const type = raw.type; |
||||
|
switch (type) { |
||||
|
// Verified against a live session (Aug 2026): output content events carry
|
||||
|
// an `output_` prefix not documented consistently anywhere we could find
|
||||
|
// — e.g. `response.output_audio.delta`, not `response.audio.delta`.
|
||||
|
case 'response.output_audio.delta': |
||||
|
return { type: 'output_audio.chunk', audio: raw.delta as string }; |
||||
|
case 'response.output_audio.done': |
||||
|
return { type: 'output_audio.done' }; |
||||
|
case 'response.output_audio_transcript.delta': |
||||
|
return { type: 'transcript.delta', role: 'assistant', text: raw.delta as string }; |
||||
|
case 'response.output_audio_transcript.done': |
||||
|
return { type: 'transcript.done', role: 'assistant' }; |
||||
|
case 'conversation.item.input_audio_transcription.completed': |
||||
|
return { type: 'transcript.delta', role: 'user', text: raw.transcript as string }; |
||||
|
case 'response.done': |
||||
|
return { type: 'response.done' }; |
||||
|
case 'error': { |
||||
|
const err = raw.error as { message?: string } | undefined; |
||||
|
return { |
||||
|
type: 'error', |
||||
|
code: 'upstream_error', |
||||
|
message: err?.message ?? 'Unknown upstream error' |
||||
|
}; |
||||
|
} |
||||
|
default: |
||||
|
// session.created/updated, conversation.*, input_audio_buffer.speech_*,
|
||||
|
// rate_limits.updated, etc. — not part of our normalized vocabulary.
|
||||
|
return null; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,205 @@ |
|||||
|
import { EventEmitter } from 'node:events'; |
||||
|
import { describe, expect, it, vi } from 'vitest'; |
||||
|
import type WebSocket from 'ws'; |
||||
|
import type { ClientEvent, ServerEvent } from '../../core/protocol.js'; |
||||
|
import type { ProviderSession, RealtimeProvider } from './provider.js'; |
||||
|
import { createRealtimeSession, resolveSessionConfig } from './session.js'; |
||||
|
|
||||
|
const DEFAULTS = { model: 'test-model', instructions: 'be helpful' }; |
||||
|
|
||||
|
/** Minimal stand-in for a `ws` WebSocket — only what RealtimeSession touches. */ |
||||
|
class FakeBrowserSocket extends EventEmitter { |
||||
|
readyState = 1; // WebSocket.OPEN
|
||||
|
sentRaw: string[] = []; |
||||
|
closeCalls = 0; |
||||
|
|
||||
|
send(data: string) { |
||||
|
this.sentRaw.push(data); |
||||
|
} |
||||
|
|
||||
|
close() { |
||||
|
this.closeCalls++; |
||||
|
this.readyState = 3; // WebSocket.CLOSED
|
||||
|
} |
||||
|
|
||||
|
get sent(): ServerEvent[] { |
||||
|
return this.sentRaw.map((raw) => JSON.parse(raw)); |
||||
|
} |
||||
|
|
||||
|
receive(event: ClientEvent) { |
||||
|
this.emit('message', Buffer.from(JSON.stringify(event)), false); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
class FakeProviderSession implements ProviderSession { |
||||
|
sent: ClientEvent[] = []; |
||||
|
closed = false; |
||||
|
closeReason: string | undefined; |
||||
|
private handler: ((event: ServerEvent) => void) | null = null; |
||||
|
|
||||
|
send(event: ClientEvent): void { |
||||
|
this.sent.push(event); |
||||
|
} |
||||
|
|
||||
|
onEvent(handler: (event: ServerEvent) => void): void { |
||||
|
this.handler = handler; |
||||
|
} |
||||
|
|
||||
|
close(reason?: string): void { |
||||
|
this.closed = true; |
||||
|
this.closeReason = reason; |
||||
|
} |
||||
|
|
||||
|
emitToOrchestrator(event: ServerEvent): void { |
||||
|
this.handler?.(event); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
class FakeProvider implements RealtimeProvider { |
||||
|
sessions: FakeProviderSession[] = []; |
||||
|
private readonly gate: Promise<void>; |
||||
|
private release!: () => void; |
||||
|
|
||||
|
constructor(opts: { deferred?: boolean } = {}) { |
||||
|
this.gate = opts.deferred |
||||
|
? new Promise((resolve) => (this.release = resolve)) |
||||
|
: Promise.resolve(); |
||||
|
} |
||||
|
|
||||
|
async createSession(): Promise<ProviderSession> { |
||||
|
await this.gate; |
||||
|
const session = new FakeProviderSession(); |
||||
|
this.sessions.push(session); |
||||
|
return session; |
||||
|
} |
||||
|
|
||||
|
releaseDeferred(): void { |
||||
|
this.release(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
function makeSocket() { |
||||
|
const socket = new FakeBrowserSocket(); |
||||
|
return { socket, asWs: socket as unknown as WebSocket }; |
||||
|
} |
||||
|
|
||||
|
describe('resolveSessionConfig', () => { |
||||
|
it('always uses server-owned model/instructions, merging only the client-visible voice', () => { |
||||
|
expect(resolveSessionConfig(DEFAULTS, { voice: 'coral' })).toEqual({ |
||||
|
model: 'test-model', |
||||
|
instructions: 'be helpful', |
||||
|
voice: 'coral' |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
it('is safe to call with no client config at all', () => { |
||||
|
expect(resolveSessionConfig(DEFAULTS)).toEqual({ |
||||
|
model: 'test-model', |
||||
|
instructions: 'be helpful', |
||||
|
voice: undefined |
||||
|
}); |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
describe('RealtimeSession', () => { |
||||
|
it('starts a provider session on session.start and replies with session.ready', async () => { |
||||
|
const provider = new FakeProvider(); |
||||
|
const { socket, asWs } = makeSocket(); |
||||
|
createRealtimeSession(asWs, provider, DEFAULTS); |
||||
|
|
||||
|
socket.receive({ type: 'session.start' }); |
||||
|
await vi.waitFor(() => expect(provider.sessions).toHaveLength(1)); |
||||
|
await vi.waitFor(() => |
||||
|
expect(socket.sent).toContainEqual(expect.objectContaining({ type: 'session.ready' })) |
||||
|
); |
||||
|
}); |
||||
|
|
||||
|
it('rejects content events sent before session.start', async () => { |
||||
|
const provider = new FakeProvider(); |
||||
|
const { socket, asWs } = makeSocket(); |
||||
|
createRealtimeSession(asWs, provider, DEFAULTS); |
||||
|
|
||||
|
socket.receive({ type: 'input_text.send', text: 'too early' }); |
||||
|
await vi.waitFor(() => |
||||
|
expect(socket.sent).toContainEqual( |
||||
|
expect.objectContaining({ type: 'error', code: 'invalid_client_event' }) |
||||
|
) |
||||
|
); |
||||
|
expect(provider.sessions).toHaveLength(0); |
||||
|
}); |
||||
|
|
||||
|
it('forwards events to the provider session once active', async () => { |
||||
|
const provider = new FakeProvider(); |
||||
|
const { socket, asWs } = makeSocket(); |
||||
|
createRealtimeSession(asWs, provider, DEFAULTS); |
||||
|
|
||||
|
socket.receive({ type: 'session.start' }); |
||||
|
await vi.waitFor(() => expect(provider.sessions).toHaveLength(1)); |
||||
|
|
||||
|
socket.receive({ type: 'input_text.send', text: 'hello' }); |
||||
|
await vi.waitFor(() => |
||||
|
expect(provider.sessions[0].sent).toContainEqual({ type: 'input_text.send', text: 'hello' }) |
||||
|
); |
||||
|
}); |
||||
|
|
||||
|
it('relays provider server events back to the browser socket', async () => { |
||||
|
const provider = new FakeProvider(); |
||||
|
const { socket, asWs } = makeSocket(); |
||||
|
createRealtimeSession(asWs, provider, DEFAULTS); |
||||
|
|
||||
|
socket.receive({ type: 'session.start' }); |
||||
|
await vi.waitFor(() => expect(provider.sessions).toHaveLength(1)); |
||||
|
|
||||
|
provider.sessions[0].emitToOrchestrator({ |
||||
|
type: 'transcript.delta', |
||||
|
role: 'assistant', |
||||
|
text: 'hi' |
||||
|
}); |
||||
|
await vi.waitFor(() => |
||||
|
expect(socket.sent).toContainEqual({ |
||||
|
type: 'transcript.delta', |
||||
|
role: 'assistant', |
||||
|
text: 'hi' |
||||
|
}) |
||||
|
); |
||||
|
}); |
||||
|
|
||||
|
it('closes the provider session when the browser disconnects', async () => { |
||||
|
const provider = new FakeProvider(); |
||||
|
const { socket, asWs } = makeSocket(); |
||||
|
createRealtimeSession(asWs, provider, DEFAULTS); |
||||
|
|
||||
|
socket.receive({ type: 'session.start' }); |
||||
|
await vi.waitFor(() => expect(provider.sessions).toHaveLength(1)); |
||||
|
|
||||
|
socket.emit('close'); |
||||
|
await vi.waitFor(() => expect(provider.sessions[0].closed).toBe(true)); |
||||
|
expect(provider.sessions[0].closeReason).toBe('client_disconnected'); |
||||
|
}); |
||||
|
|
||||
|
it('closes the provider session on session.end and closes the browser socket', async () => { |
||||
|
const provider = new FakeProvider(); |
||||
|
const { socket, asWs } = makeSocket(); |
||||
|
createRealtimeSession(asWs, provider, DEFAULTS); |
||||
|
|
||||
|
socket.receive({ type: 'session.start' }); |
||||
|
await vi.waitFor(() => expect(provider.sessions).toHaveLength(1)); |
||||
|
|
||||
|
socket.receive({ type: 'session.end' }); |
||||
|
await vi.waitFor(() => expect(provider.sessions[0].closed).toBe(true)); |
||||
|
expect(socket.closeCalls).toBe(1); |
||||
|
}); |
||||
|
|
||||
|
it('does not leak an upstream session if the browser disconnects mid-handshake', async () => { |
||||
|
const provider = new FakeProvider({ deferred: true }); |
||||
|
const { socket, asWs } = makeSocket(); |
||||
|
createRealtimeSession(asWs, provider, DEFAULTS); |
||||
|
|
||||
|
socket.receive({ type: 'session.start' }); |
||||
|
socket.emit('close'); // disconnect while the provider handshake is still pending
|
||||
|
provider.releaseDeferred(); |
||||
|
|
||||
|
await vi.waitFor(() => expect(provider.sessions).toHaveLength(1)); |
||||
|
await vi.waitFor(() => expect(provider.sessions[0].closed).toBe(true)); |
||||
|
}); |
||||
|
}); |
||||
@ -0,0 +1,161 @@ |
|||||
|
import { randomUUID } from 'node:crypto'; |
||||
|
import WebSocket from 'ws'; |
||||
|
import type { ClientEvent, ServerEvent, SessionStartConfig } from '../../core/protocol.js'; |
||||
|
import type { ProviderSession, RealtimeProvider, ResolvedSessionConfig } from './provider.js'; |
||||
|
|
||||
|
export type RealtimeSessionState = 'idle' | 'connecting' | 'active' | 'closing' | 'closed'; |
||||
|
|
||||
|
/** Server-owned policy for a realtime session. Never client-controlled. */ |
||||
|
export interface SessionDefaults { |
||||
|
model: string; |
||||
|
instructions: string; |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Merges the browser's (deliberately tiny) client-visible config into the |
||||
|
* server-owned defaults. `instructions`/`model` always come from `defaults` |
||||
|
* — this is the trust boundary that keeps a shared relay safe to expose to |
||||
|
* untrusted browsers. |
||||
|
*/ |
||||
|
export function resolveSessionConfig( |
||||
|
defaults: SessionDefaults, |
||||
|
clientConfig?: SessionStartConfig |
||||
|
): ResolvedSessionConfig { |
||||
|
return { |
||||
|
model: defaults.model, |
||||
|
instructions: defaults.instructions, |
||||
|
voice: clientConfig?.voice |
||||
|
}; |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* Owns the lifecycle of one browser WS connection: turns its first |
||||
|
* `session.start` into a `ProviderSession` via the injected `RealtimeProvider`, |
||||
|
* pipes events both directions afterward, and guarantees that either side |
||||
|
* closing/erroring tears down the other. |
||||
|
*/ |
||||
|
export class RealtimeSession { |
||||
|
private state: RealtimeSessionState = 'idle'; |
||||
|
private providerSession: ProviderSession | null = null; |
||||
|
|
||||
|
constructor( |
||||
|
private readonly browserSocket: WebSocket, |
||||
|
private readonly provider: RealtimeProvider, |
||||
|
private readonly defaults: SessionDefaults |
||||
|
) { |
||||
|
this.browserSocket.on('message', (data, isBinary) => { |
||||
|
void this.handleBrowserMessage(data, isBinary); |
||||
|
}); |
||||
|
this.browserSocket.on('close', () => this.close('client_disconnected')); |
||||
|
this.browserSocket.on('error', () => this.close('client_disconnected')); |
||||
|
} |
||||
|
|
||||
|
getState(): RealtimeSessionState { |
||||
|
return this.state; |
||||
|
} |
||||
|
|
||||
|
private async handleBrowserMessage(data: WebSocket.RawData, isBinary: boolean): Promise<void> { |
||||
|
if (isBinary) { |
||||
|
this.sendToBrowser({ |
||||
|
type: 'error', |
||||
|
code: 'invalid_client_event', |
||||
|
message: 'Binary frames are not supported yet.' |
||||
|
}); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
let event: ClientEvent; |
||||
|
try { |
||||
|
event = JSON.parse(data.toString()); |
||||
|
} catch { |
||||
|
this.sendToBrowser({ |
||||
|
type: 'error', |
||||
|
code: 'invalid_client_event', |
||||
|
message: 'Malformed JSON.' |
||||
|
}); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
if (event.type === 'session.start') { |
||||
|
await this.start(event.config); |
||||
|
return; |
||||
|
} |
||||
|
if (event.type === 'session.end') { |
||||
|
this.close('client_requested'); |
||||
|
return; |
||||
|
} |
||||
|
if (this.state !== 'active' || !this.providerSession) { |
||||
|
this.sendToBrowser({ |
||||
|
type: 'error', |
||||
|
code: 'invalid_client_event', |
||||
|
message: `Cannot handle '${event.type}' before session.start.` |
||||
|
}); |
||||
|
return; |
||||
|
} |
||||
|
this.providerSession.send(event); |
||||
|
} |
||||
|
|
||||
|
private async start(clientConfig?: SessionStartConfig): Promise<void> { |
||||
|
if (this.state !== 'idle') { |
||||
|
this.sendToBrowser({ |
||||
|
type: 'error', |
||||
|
code: 'invalid_client_event', |
||||
|
message: 'Session already started.' |
||||
|
}); |
||||
|
return; |
||||
|
} |
||||
|
this.state = 'connecting'; |
||||
|
const config = resolveSessionConfig(this.defaults, clientConfig); |
||||
|
|
||||
|
let providerSession: ProviderSession; |
||||
|
try { |
||||
|
providerSession = await this.provider.createSession(config); |
||||
|
} catch (err) { |
||||
|
this.state = 'closed'; |
||||
|
this.sendToBrowser({ |
||||
|
type: 'error', |
||||
|
code: 'upstream_unavailable', |
||||
|
message: err instanceof Error ? err.message : 'Failed to start session.' |
||||
|
}); |
||||
|
this.browserSocket.close(); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
// The browser may have disconnected (session.end / close) while we were
|
||||
|
// awaiting the provider handshake — don't leak the upstream connection.
|
||||
|
const stateAfterConnect: RealtimeSessionState = this.state; |
||||
|
if (stateAfterConnect !== 'connecting') { |
||||
|
providerSession.close('client_requested'); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
this.providerSession = providerSession; |
||||
|
this.providerSession.onEvent((event) => this.sendToBrowser(event)); |
||||
|
this.state = 'active'; |
||||
|
this.sendToBrowser({ type: 'session.ready', sessionId: randomUUID() }); |
||||
|
} |
||||
|
|
||||
|
private close(reason: string): void { |
||||
|
if (this.state === 'closing' || this.state === 'closed') return; |
||||
|
this.state = 'closing'; |
||||
|
this.providerSession?.close(reason); |
||||
|
this.providerSession = null; |
||||
|
if (this.browserSocket.readyState === WebSocket.OPEN) { |
||||
|
this.browserSocket.close(); |
||||
|
} |
||||
|
this.state = 'closed'; |
||||
|
} |
||||
|
|
||||
|
private sendToBrowser(event: ServerEvent): void { |
||||
|
if (this.browserSocket.readyState !== WebSocket.OPEN) return; |
||||
|
this.browserSocket.send(JSON.stringify(event)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
export function createRealtimeSession( |
||||
|
browserSocket: WebSocket, |
||||
|
provider: RealtimeProvider, |
||||
|
defaults: SessionDefaults |
||||
|
): RealtimeSession { |
||||
|
return new RealtimeSession(browserSocket, provider, defaults); |
||||
|
} |
||||
@ -0,0 +1,48 @@ |
|||||
|
import { afterEach, beforeEach, describe, expect, it } from 'vitest'; |
||||
|
import { mintSessionToken, verifySessionToken } from './tokens.js'; |
||||
|
|
||||
|
describe('tokens', () => { |
||||
|
const originalSecret = process.env.REALTIME_TOKEN_SECRET; |
||||
|
|
||||
|
beforeEach(() => { |
||||
|
process.env.REALTIME_TOKEN_SECRET = 'test-secret'; |
||||
|
}); |
||||
|
|
||||
|
afterEach(() => { |
||||
|
process.env.REALTIME_TOKEN_SECRET = originalSecret; |
||||
|
}); |
||||
|
|
||||
|
it('mints a token that verifies as valid', () => { |
||||
|
const { token } = mintSessionToken(); |
||||
|
expect(verifySessionToken(token)).toEqual({ valid: true }); |
||||
|
}); |
||||
|
|
||||
|
it('rejects a malformed token', () => { |
||||
|
expect(verifySessionToken('not-a-token')).toEqual({ valid: false, reason: 'malformed' }); |
||||
|
}); |
||||
|
|
||||
|
it('rejects a tampered signature', () => { |
||||
|
const { token } = mintSessionToken(); |
||||
|
const [payload] = token.split('.'); |
||||
|
expect(verifySessionToken(`${payload}.deadbeef`)).toEqual({ |
||||
|
valid: false, |
||||
|
reason: 'invalid_signature' |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
it('rejects a token signed with a different secret', () => { |
||||
|
const { token } = mintSessionToken(); |
||||
|
process.env.REALTIME_TOKEN_SECRET = 'a-different-secret'; |
||||
|
expect(verifySessionToken(token)).toEqual({ valid: false, reason: 'invalid_signature' }); |
||||
|
}); |
||||
|
|
||||
|
it('rejects an expired token', () => { |
||||
|
const { token } = mintSessionToken(-1000); |
||||
|
expect(verifySessionToken(token)).toEqual({ valid: false, reason: 'expired' }); |
||||
|
}); |
||||
|
|
||||
|
it('throws when the secret is not configured', () => { |
||||
|
delete process.env.REALTIME_TOKEN_SECRET; |
||||
|
expect(() => mintSessionToken()).toThrow(); |
||||
|
}); |
||||
|
}); |
||||
@ -0,0 +1,66 @@ |
|||||
|
import { createHmac, timingSafeEqual } from 'node:crypto'; |
||||
|
|
||||
|
// Short-lived: this token only needs to survive the moment between minting
|
||||
|
// and completing the WS upgrade handshake, not the whole conversation.
|
||||
|
const DEFAULT_TTL_MS = 60_000; |
||||
|
|
||||
|
// Read from process.env (not SvelteKit's $env/static/private) so this module
|
||||
|
// stays plain, portable TypeScript: it's compiled standalone for the custom
|
||||
|
// Node server (see ../../../../server.js and tsconfig.server.json), not just
|
||||
|
// bundled through Vite. Runtime env access is also simply more correct for a
|
||||
|
// secret that shouldn't be inlined at build time.
|
||||
|
function getSecret(): string { |
||||
|
const secret = process.env.REALTIME_TOKEN_SECRET; |
||||
|
if (!secret) { |
||||
|
throw new Error('REALTIME_TOKEN_SECRET is not set. See .env.example.'); |
||||
|
} |
||||
|
return secret; |
||||
|
} |
||||
|
|
||||
|
export interface MintedToken { |
||||
|
token: string; |
||||
|
expiresAt: number; // unix ms
|
||||
|
} |
||||
|
|
||||
|
export type TokenVerification = |
||||
|
{ valid: true } | { valid: false; reason: 'expired' | 'invalid_signature' | 'malformed' }; |
||||
|
|
||||
|
/** |
||||
|
* Generic, HMAC-signed, short-lived bearer token — not tied to any specific |
||||
|
* provider. Intended as a reusable "prove you got this from our own server a |
||||
|
* moment ago" primitive for gating the realtime WS upgrade (see |
||||
|
* `../ws-handler`), reusable by future server-only features too. |
||||
|
*/ |
||||
|
export function mintSessionToken(ttlMs = DEFAULT_TTL_MS): MintedToken { |
||||
|
const expiresAt = Date.now() + ttlMs; |
||||
|
const signature = sign(String(expiresAt)); |
||||
|
return { token: `${expiresAt}.${signature}`, expiresAt }; |
||||
|
} |
||||
|
|
||||
|
export function verifySessionToken(token: string): TokenVerification { |
||||
|
const parts = token.split('.'); |
||||
|
if (parts.length !== 2) return { valid: false, reason: 'malformed' }; |
||||
|
|
||||
|
const [payload, signature] = parts; |
||||
|
if (!timingSafeCompare(signature, sign(payload))) { |
||||
|
return { valid: false, reason: 'invalid_signature' }; |
||||
|
} |
||||
|
|
||||
|
const expiresAt = Number(payload); |
||||
|
if (!Number.isFinite(expiresAt) || Date.now() > expiresAt) { |
||||
|
return { valid: false, reason: 'expired' }; |
||||
|
} |
||||
|
|
||||
|
return { valid: true }; |
||||
|
} |
||||
|
|
||||
|
function sign(payload: string): string { |
||||
|
return createHmac('sha256', getSecret()).update(payload).digest('hex'); |
||||
|
} |
||||
|
|
||||
|
function timingSafeCompare(a: string, b: string): boolean { |
||||
|
const bufA = Buffer.from(a); |
||||
|
const bufB = Buffer.from(b); |
||||
|
if (bufA.length !== bufB.length) return false; |
||||
|
return timingSafeEqual(bufA, bufB); |
||||
|
} |
||||
@ -0,0 +1,47 @@ |
|||||
|
import type { IncomingMessage } from 'node:http'; |
||||
|
import type { Duplex } from 'node:stream'; |
||||
|
import { WebSocketServer } from 'ws'; |
||||
|
import { REALTIME_WS_PATH } from '../../core/protocol.js'; |
||||
|
import type { RealtimeProvider } from './provider.js'; |
||||
|
import { createRealtimeSession, type SessionDefaults } from './session.js'; |
||||
|
import { verifySessionToken } from './tokens.js'; |
||||
|
|
||||
|
export interface RealtimeUpgradeDeps { |
||||
|
provider: RealtimeProvider; |
||||
|
defaults: SessionDefaults; |
||||
|
} |
||||
|
|
||||
|
// `noServer: true` + `handleUpgrade` below is what lets this share an
|
||||
|
// `http.Server` with SvelteKit's own request handling (and, in dev, Vite's
|
||||
|
// HMR websocket) instead of needing its own listening port.
|
||||
|
const wss = new WebSocketServer({ noServer: true }); |
||||
|
|
||||
|
/** |
||||
|
* Raw `http.Server` `'upgrade'` handler, shared verbatim between the |
||||
|
* production custom server (../../../../server.js) and the dev-mode Vite |
||||
|
* plugin (./dev-plugin.ts) so the two environments behave identically. |
||||
|
* |
||||
|
* Must be a no-op (never touch `socket`) for any request whose path isn't |
||||
|
* ours: the same underlying HTTP server also carries Vite's own HMR upgrade |
||||
|
* in dev, and destroying/responding to that would silently break HMR. |
||||
|
*/ |
||||
|
export function handleRealtimeUpgrade( |
||||
|
req: IncomingMessage, |
||||
|
socket: Duplex, |
||||
|
head: Buffer, |
||||
|
deps: RealtimeUpgradeDeps |
||||
|
): void { |
||||
|
const url = new URL(req.url ?? '', 'http://internal'); |
||||
|
if (url.pathname !== REALTIME_WS_PATH) return; |
||||
|
|
||||
|
const verification = verifySessionToken(url.searchParams.get('token') ?? ''); |
||||
|
if (!verification.valid) { |
||||
|
socket.write('HTTP/1.1 401 Unauthorized\r\nConnection: close\r\n\r\n'); |
||||
|
socket.destroy(); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
wss.handleUpgrade(req, socket, head, (browserSocket) => { |
||||
|
createRealtimeSession(browserSocket, deps.provider, deps.defaults); |
||||
|
}); |
||||
|
} |
||||
@ -0,0 +1,8 @@ |
|||||
|
import { json } from '@sveltejs/kit'; |
||||
|
import { mintSessionToken } from '$lib/server/core/tokens.js'; |
||||
|
import type { RequestHandler } from './$types'; |
||||
|
|
||||
|
export const POST: RequestHandler = () => { |
||||
|
const { token, expiresAt } = mintSessionToken(); |
||||
|
return json({ token, expiresAt }); |
||||
|
}; |
||||
@ -0,0 +1,117 @@ |
|||||
|
<script lang="ts"> |
||||
|
import { RealtimeClient } from '$lib/client/core/client.js'; |
||||
|
import type { ServerEvent } from '$lib/core/protocol.js'; |
||||
|
|
||||
|
type Status = 'idle' | 'connecting' | 'ready' | 'error' | 'closed'; |
||||
|
type TranscriptEntry = { role: 'user' | 'assistant'; text: string }; |
||||
|
|
||||
|
let status = $state<Status>('idle'); |
||||
|
let statusMessage = $state(''); |
||||
|
let transcript = $state<TranscriptEntry[]>([]); |
||||
|
let draft = $state(''); |
||||
|
|
||||
|
let client: RealtimeClient | null = null; |
||||
|
|
||||
|
async function connect() { |
||||
|
status = 'connecting'; |
||||
|
statusMessage = ''; |
||||
|
try { |
||||
|
const res = await fetch('/api/realtime/session', { method: 'POST' }); |
||||
|
if (!res.ok) throw new Error(`Failed to mint session token (${res.status})`); |
||||
|
const { token } = await res.json(); |
||||
|
|
||||
|
const nextClient = new RealtimeClient(); |
||||
|
nextClient.on(handleServerEvent); |
||||
|
await nextClient.connect(token); |
||||
|
client = nextClient; |
||||
|
client.send({ type: 'session.start' }); |
||||
|
} catch (err) { |
||||
|
status = 'error'; |
||||
|
statusMessage = err instanceof Error ? err.message : 'Failed to connect.'; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
function handleServerEvent(event: ServerEvent) { |
||||
|
switch (event.type) { |
||||
|
case 'session.ready': |
||||
|
status = 'ready'; |
||||
|
break; |
||||
|
case 'transcript.delta': |
||||
|
appendDelta(event.role, event.text); |
||||
|
break; |
||||
|
case 'error': |
||||
|
status = 'error'; |
||||
|
statusMessage = event.message; |
||||
|
break; |
||||
|
case 'session.closed': |
||||
|
status = 'closed'; |
||||
|
statusMessage = event.reason; |
||||
|
break; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
function appendDelta(role: 'user' | 'assistant', text: string) { |
||||
|
const last = transcript.at(-1); |
||||
|
if (last && last.role === role) { |
||||
|
last.text += text; |
||||
|
} else { |
||||
|
transcript.push({ role, text }); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
function sendMessage() { |
||||
|
const text = draft.trim(); |
||||
|
if (!text || !client) return; |
||||
|
transcript.push({ role: 'user', text }); |
||||
|
client.send({ type: 'input_text.send', text }); |
||||
|
draft = ''; |
||||
|
} |
||||
|
|
||||
|
function disconnect() { |
||||
|
client?.send({ type: 'session.end' }); |
||||
|
client?.close(); |
||||
|
client = null; |
||||
|
status = 'closed'; |
||||
|
} |
||||
|
</script> |
||||
|
|
||||
|
<h1>Realtime session demo</h1> |
||||
|
<p>Status: {status}{statusMessage ? ` — ${statusMessage}` : ''}</p> |
||||
|
|
||||
|
{#if status === 'idle' || status === 'error' || status === 'closed'} |
||||
|
<button onclick={connect}>Connect</button> |
||||
|
{:else if status === 'ready'} |
||||
|
<button onclick={disconnect}>Disconnect</button> |
||||
|
{/if} |
||||
|
|
||||
|
<div class="transcript"> |
||||
|
{#each transcript as entry, i (i)} |
||||
|
<p class={entry.role}><strong>{entry.role}:</strong> {entry.text}</p> |
||||
|
{/each} |
||||
|
</div> |
||||
|
|
||||
|
<form |
||||
|
onsubmit={(e) => { |
||||
|
e.preventDefault(); |
||||
|
sendMessage(); |
||||
|
}} |
||||
|
> |
||||
|
<input bind:value={draft} placeholder="Type a message…" disabled={status !== 'ready'} /> |
||||
|
<button type="submit" disabled={status !== 'ready'}>Send</button> |
||||
|
</form> |
||||
|
|
||||
|
<style> |
||||
|
.transcript { |
||||
|
display: flex; |
||||
|
flex-direction: column; |
||||
|
gap: 0.5rem; |
||||
|
margin: 1rem 0; |
||||
|
min-height: 8rem; |
||||
|
} |
||||
|
.user { |
||||
|
color: #666; |
||||
|
} |
||||
|
.assistant { |
||||
|
font-weight: 500; |
||||
|
} |
||||
|
</style> |
||||
@ -0,0 +1,17 @@ |
|||||
|
{ |
||||
|
"compilerOptions": { |
||||
|
"target": "es2023", |
||||
|
"module": "nodenext", |
||||
|
"moduleResolution": "nodenext", |
||||
|
"outDir": "build-server", |
||||
|
"rootDir": "src/lib", |
||||
|
"strict": true, |
||||
|
"skipLibCheck": true, |
||||
|
"esModuleInterop": true, |
||||
|
"forceConsistentCasingInFileNames": true, |
||||
|
"declaration": false, |
||||
|
"types": ["node"] |
||||
|
}, |
||||
|
"include": ["src/lib/core/**/*.ts", "src/lib/server/core/**/*.ts"], |
||||
|
"exclude": ["**/*.spec.ts"] |
||||
|
} |
||||
Loading…
Reference in new issue