jacquardSnapshot

← snapshot

5625 bytes
"use client";

/**
 * Playing Jackie's voice as it arrives.
 *
 * The server hands back raw 24 kHz mono PCM. Rather than waiting for the whole
 * utterance, each chunk is converted to an AudioBuffer and scheduled onto a
 * running clock the moment it lands, so she starts talking about as fast as
 * the model can produce sound.
 *
 * A side benefit worth more than it sounds: because we hold the samples, the
 * orb can breathe on Jackie's *actual* output amplitude instead of the random
 * flicker the browser's synthesis path had to fake.
 */

const SAMPLE_RATE = 24000;
/** Small enough that the first sound is prompt, big enough to stay ahead. */
const LEAD_SECONDS = 0.12;

export interface CedarHandle {
  /** Resolves when the last scheduled chunk has finished playing. */
  done: Promise<void>;
  /** Cuts the utterance short. */
  cancel: () => void;
}

export class CedarUnavailable extends Error {}

/** Cached across calls: whether the key is configured is not going to change. */
let configured: Promise<boolean> | null = null;

/** One quiet probe, rather than a failing POST per utterance. */
export function cedarConfigured(): Promise<boolean> {
  configured ??= fetch("/speech")
    .then((r) => (r.ok ? r.json() : { configured: false }))
    .then((j: { configured?: boolean }) => Boolean(j.configured))
    .catch(() => false);
  return configured;
}

/**
 * Speaks `text`, reporting output level 0..1 as it plays.
 *
 * Throws [`CedarUnavailable`] when the route has no key configured or the
 * upstream refuses — callers should fall back to `speechSynthesis` rather than
 * failing the turn.
 */
export async function speakCedar(
  text: string,
  onLevel: (level: number) => void,
  signal?: AbortSignal,
): Promise<CedarHandle> {
  if (!(await cedarConfigured())) throw new CedarUnavailable("not configured");

  const Ctx =
    window.AudioContext ??
    (window as unknown as { webkitAudioContext?: typeof AudioContext })
      .webkitAudioContext;
  if (!Ctx) throw new CedarUnavailable("no Web Audio");

  const controller = new AbortController();
  signal?.addEventListener("abort", () => controller.abort());

  const response = await fetch("/speech", {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ text }),
    signal: controller.signal,
  });

  if (response.status === 503) throw new CedarUnavailable("not configured");
  if (!response.ok || !response.body) {
    const detail = await response.text().catch(() => "");
    throw new CedarUnavailable(`tts ${response.status} ${detail.slice(0, 160)}`);
  }

  const ctx = new Ctx({ sampleRate: SAMPLE_RATE });
  // Browsers start suspended until a gesture; Jackie speaks in response to
  // one, so this normally resolves immediately.
  await ctx.resume().catch(() => {});

  const gain = ctx.createGain();
  const analyser = ctx.createAnalyser();
  analyser.fftSize = 512;
  gain.connect(analyser);
  analyser.connect(ctx.destination);

  let cursor = ctx.currentTime + LEAD_SECONDS;
  let cancelled = false;
  const sources: AudioBufferSourceNode[] = [];

  // Level metering, from what is actually coming out.
  const meterBuf = new Float32Array(analyser.fftSize);
  let raf = 0;
  const meter = () => {
    analyser.getFloatTimeDomainData(meterBuf);
    let sum = 0;
    for (const v of meterBuf) sum += v * v;
    onLevel(Math.min(1, Math.sqrt(sum / meterBuf.length) * 4.5));
    raf = requestAnimationFrame(meter);
  };
  raf = requestAnimationFrame(meter);

  const cancel = () => {
    cancelled = true;
    controller.abort();
    for (const s of sources) {
      try {
        s.stop();
      } catch {
        /* already finished */
      }
    }
  };

  const pump = (async () => {
    const reader = response.body!.getReader();
    // PCM frames are two bytes; a chunk can split one, so carry the odd byte.
    let carry: Uint8Array | null = null;
    try {
      for (;;) {
        const { done, value } = await reader.read();
        if (done || cancelled) break;
        let bytes = value;
        if (carry) {
          const joined = new Uint8Array(carry.length + bytes.length);
          joined.set(carry);
          joined.set(bytes, carry.length);
          bytes = joined;
          carry = null;
        }
        if (bytes.length % 2 === 1) {
          carry = bytes.subarray(bytes.length - 1).slice();
          bytes = bytes.subarray(0, bytes.length - 1);
        }
        if (bytes.length === 0) continue;

        const samples = new Int16Array(
          bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.length),
        );
        const buffer = ctx.createBuffer(1, samples.length, SAMPLE_RATE);
        const channel = buffer.getChannelData(0);
        for (let i = 0; i < samples.length; i++) {
          channel[i] = (samples[i] as number) / 32768;
        }
        const source = ctx.createBufferSource();
        source.buffer = buffer;
        source.connect(gain);
        // Never schedule in the past: if the network stalled, resume from now.
        cursor = Math.max(cursor, ctx.currentTime + 0.02);
        source.start(cursor);
        cursor += buffer.duration;
        sources.push(source);
      }
    } catch {
      /* aborted or stream error — whatever was scheduled still plays */
    }

    // Wait out the audio already queued, then tear down.
    const remaining = Math.max(0, cursor - ctx.currentTime);
    await new Promise((r) => setTimeout(r, cancelled ? 0 : remaining * 1000 + 60));
    cancelAnimationFrame(raf);
    onLevel(0);
    await ctx.close().catch(() => {});
  })();

  return { done: pump, cancel };
}