"use client";
import { useCallback, useEffect, useRef, useState } from "react";
import { CedarUnavailable, cedarConfigured, speakCedar } from "./cedar";
/**
* Voice in and out, in one hook.
*
* Both halves are browser-native: `SpeechRecognition` for listening (Chrome
* and Edge only) and `speechSynthesis` for speaking (everywhere). Nothing is
* uploaded, no key is held, and no audio is retained — Jackie hears a
* transcript, not a recording.
*/
/** Which engine actually produced the last thing Jackie said. */
export type SpeechEngine =
| "unknown"
/** Streaming from OpenAI. The only voice this app speaks in. */
| "cedar"
/** No key configured, so nothing was said. */
| "silent";
export interface SpeechApi {
/** Recognition available — speaking usually is even when this is false. */
canListen: boolean;
canSpeak: boolean;
speaking: boolean;
listening: boolean;
/** Rough input level 0..1, for the orb. */
level: number;
/** Resolves when the utterance finishes (or immediately if muted). */
speak: (text: string) => Promise<void>;
/**
* Resolves with what was heard and, when nothing was, why. A denied or
* missing microphone is a different situation from silence, and callers
* need to tell them apart — one should fall back to typing, the other
* should just move on.
*/
listen: () => Promise<{ text: string; error: string | null }>;
/** Live partial transcript while listening. */
interim: string;
stop: () => void;
muted: boolean;
setMuted: (m: boolean) => void;
/**
* What spoke last. Surfaced so a silent fall back to a dated formant synth
* is visible rather than just sounding bad for no stated reason.
*/
engine: SpeechEngine;
}
/** Where the opt-in is remembered. */
const VOICE_KEY = "jac-jackie-voice";
export function useSpeech(): SpeechApi {
const [speaking, setSpeaking] = useState(false);
const [listening, setListening] = useState(false);
const [interim, setInterim] = useState("");
const [level, setLevel] = useState(0);
// Silent until asked. Speech that starts on its own is startling at best,
// and on a machine with only formant voices it is worse than nothing — so
// the voice is something you turn on, never something you turn off.
const [muted, setMutedState] = useState(true);
const [canListen, setCanListen] = useState(false);
const [canSpeak, setCanSpeak] = useState(false);
const [engine, setEngine] = useState<SpeechEngine>("unknown");
const recognitionRef = useRef<SpeechRecognitionLike | null>(null);
// Mute lives in a ref as well as state so `speak` can stay referentially
// stable — consumers put it in effect deps, and an identity that changes
// every render turns those effects into a loop.
const mutedRef = useRef(true);
/** Settles the in-flight listen(), so stop() returns what was heard. */
const finishRef = useRef<(() => void) | null>(null);
/** The in-flight cedar utterance, so stop() can cut it off. */
const cedarRef = useRef<{ cancel: () => void } | null>(null);
/** Set once cedar proves unavailable, so we stop asking. */
const cedarOffRef = useRef(false);
const setMuted = useCallback((m: boolean) => {
mutedRef.current = m;
setMutedState(m);
try {
// Remembered, so opting in is a one-time decision rather than a
// per-visit one. Absent or unreadable storage means silence.
localStorage.setItem(VOICE_KEY, m ? "off" : "on");
} catch {
/* non-fatal */
}
if (m) {
cedarRef.current?.cancel();
cedarRef.current = null;
}
}, []);
// Restore a previous opt-in. Deliberately in an effect: the server has no
// localStorage, and rendering "on" before we know would both mismatch and
// risk speaking uninvited.
useEffect(() => {
try {
if (localStorage.getItem(VOICE_KEY) === "on") {
mutedRef.current = false;
setMutedState(false);
}
} catch {
/* stay silent */
}
}, []);
useEffect(() => {
const w = window as WindowWithSpeech;
setCanListen(Boolean(w.SpeechRecognition ?? w.webkitSpeechRecognition));
// Speaking now means cedar, which is a server capability rather than a
// browser one — the probe in `lib/cedar.ts` answers for it.
void cedarConfigured().then(setCanSpeak);
}, []);
const stop = useCallback(() => {
// Resolve whatever listen() is waiting on, so an explicit stop hands back
// the words already heard rather than abandoning the promise.
finishRef.current?.();
finishRef.current = null;
cedarRef.current?.cancel();
cedarRef.current = null;
try {
recognitionRef.current?.stop();
} catch {
/* already stopped */
}
setSpeaking(false);
setListening(false);
setInterim("");
setLevel(0);
}, []);
useEffect(() => stop, [stop]);
const speak = useCallback(
(text: string) =>
new Promise<void>((resolve) => {
if (mutedRef.current) {
resolve();
return;
}
/* ---- no browser synthesis ----------------------------------------
* There used to be a `speechSynthesis` fallback here. It is gone on
* purpose. The platform voices available on a typical machine are
* dated formant synths — on this one the list holds 47 English voices
* of which 19 are novelties like Zarvox and Bad News — and Jackie
* reading technical prose in one of those is worse than Jackie not
* reading it at all.
*
* So the voice is cedar or it is nothing. Silence is a legible state:
* the transcript is on screen either way, and the UI says plainly that
* the voice wants a key rather than quietly sounding bad.
*/
const silent = () => {
setSpeaking(false);
setLevel(0);
setEngine("silent");
resolve();
};
/* ---- cedar, when it is configured -------------------------------- */
if (cedarOffRef.current) {
silent();
return;
}
setSpeaking(true);
void speakCedar(text, setLevel)
.then(async (handle) => {
setEngine("cedar");
cedarRef.current = handle;
await handle.done;
cedarRef.current = null;
setSpeaking(false);
setLevel(0);
resolve();
})
.catch((e) => {
cedarRef.current = null;
// Remember, so every later line skips the round trip. A missing
// key is a permanent condition, not a transient failure.
if (e instanceof CedarUnavailable) cedarOffRef.current = true;
silent();
});
}),
[],
);
const listen = useCallback(
() =>
new Promise<{ text: string; error: string | null }>((resolve) => {
const w = window as WindowWithSpeech;
const Ctor = w.SpeechRecognition ?? w.webkitSpeechRecognition;
if (!Ctor) {
resolve({ text: "", error: "unsupported" });
return;
}
const recognition = new Ctor();
// Continuous, because a person thinking mid-sentence is not finished
// speaking. With this false the engine ends at the first pause and the
// second half of the thought is simply lost.
recognition.continuous = true;
recognition.interimResults = true;
recognition.lang = "en-US";
let finalText = "";
let settled = false;
let failure: string | null = null;
let started = false;
let stopping = false;
let restarts = 0;
/** Last moment the microphone actually heard speech. */
let lastVoiceAt = performance.now();
const timers: number[] = [];
let meter: (() => void) | null = null;
const finish = () => {
if (settled) return;
settled = true;
stopping = true;
for (const t of timers) window.clearTimeout(t);
meter?.();
try {
recognition.stop();
} catch {
/* already stopped */
}
recognitionRef.current = null;
setListening(false);
setInterim("");
setLevel(0);
resolve({ text: finalText.trim(), error: failure });
};
finishRef.current = finish;
/* ---- level -------------------------------------------------------
* Amplitude from the microphone itself, not from how much text has
* arrived. It is what tells you the app can actually hear you, and it
* is what decides when you have stopped talking. If capture is
* refused we fall back to the transcript-length approximation rather
* than failing the whole attempt — recognition may still work.
*/
let haveMeter = false;
navigator.mediaDevices
?.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
})
.then((stream) => {
if (settled) {
for (const t of stream.getTracks()) t.stop();
return;
}
const Ctx =
window.AudioContext ??
(window as unknown as { webkitAudioContext?: typeof AudioContext })
.webkitAudioContext;
if (!Ctx) return;
const ctx = new Ctx();
const src = ctx.createMediaStreamSource(stream);
const analyser = ctx.createAnalyser();
analyser.fftSize = 1024;
src.connect(analyser);
const buf = new Float32Array(analyser.fftSize);
haveMeter = true;
let raf = 0;
const tick = () => {
analyser.getFloatTimeDomainData(buf);
let sum = 0;
for (const v of buf) sum += v * v;
const rms = Math.sqrt(sum / buf.length);
// Speech sits well above room tone; this threshold is what
// separates "thinking" from "finished".
if (rms > 0.012) lastVoiceAt = performance.now();
setLevel(Math.min(1, rms * 9));
raf = requestAnimationFrame(tick);
};
raf = requestAnimationFrame(tick);
meter = () => {
cancelAnimationFrame(raf);
for (const t of stream.getTracks()) t.stop();
void ctx.close();
};
})
.catch(() => {
/* no meter; recognition may still work */
});
// Watchdogs. A permission prompt that is never answered — or a
// sandboxed browser that blocks capture outright — fires neither
// `onstart` nor `onerror`, and without these the caller would wait
// on a promise that never settles.
recognition.onstart = () => {
started = true;
lastVoiceAt = performance.now();
};
timers.push(
window.setTimeout(() => {
if (!started && !settled) {
failure = "no-start";
finish();
}
}, 4000),
);
/* ---- endpointing --------------------------------------------------
* Stop when the room has been quiet for a beat *and* something was
* said — not at the engine's first guess that a phrase ended.
*/
const SILENCE_MS = 2400;
const poll = window.setInterval(() => {
if (settled) return;
const quiet = performance.now() - lastVoiceAt;
const heard = finalText.trim().length > 0;
if (heard && quiet > SILENCE_MS) {
window.clearInterval(poll);
finish();
}
}, 250);
timers.push(poll as unknown as number);
// Overall ceiling, so a stuck engine cannot hold the turn forever.
timers.push(
window.setTimeout(() => {
if (!settled) {
if (!finalText.trim()) failure = failure ?? "timeout";
finish();
}
}, 60000),
);
recognition.onresult = (event: SpeechRecognitionEventLike) => {
let partial = "";
for (let i = event.resultIndex; i < event.results.length; i++) {
const result = event.results[i];
const transcript = result[0]?.transcript ?? "";
if (result.isFinal) finalText += `${transcript} `;
else partial += transcript;
}
setInterim(partial);
if (!haveMeter) {
lastVoiceAt = performance.now();
setLevel(
partial.length > 0 ? 0.5 + Math.min(partial.length / 60, 0.45) : 0.25,
);
}
};
recognition.onerror = (event: { error: string }) => {
// "no-speech" is silence, which is a legitimate answer. Everything
// else — denied, no device, service blocked — means the ear is not
// available and the caller should offer a keyboard instead.
if (event.error !== "no-speech" && event.error !== "aborted") {
failure = event.error;
finish();
return;
}
if (event.error === "aborted") finish();
};
recognition.onend = () => {
if (settled) return;
// Chrome ends the session on its own after a pause even when
// `continuous` is set. As long as the person has not asked to stop,
// pick the microphone back up instead of ending their sentence for
// them.
if (!stopping && restarts < 12) {
restarts += 1;
try {
recognition.start();
return;
} catch {
/* fall through to finish */
}
}
finish();
};
recognitionRef.current = recognition;
setListening(true);
setLevel(0);
try {
recognition.start();
} catch (e) {
failure = e instanceof Error ? e.name : "start-failed";
finish();
}
}),
[],
);
return {
canListen,
canSpeak,
speaking,
listening,
level,
speak,
listen,
interim,
stop,
muted,
setMuted,
engine,
};
}
// ---------------------------------------------------------------------------
// Minimal Web Speech surface — no @types/dom-speech-recognition dependency.
// ---------------------------------------------------------------------------
interface SpeechRecognitionAlternativeLike {
transcript: string;
}
interface SpeechRecognitionResultLike {
isFinal: boolean;
[index: number]: SpeechRecognitionAlternativeLike;
}
export interface SpeechRecognitionEventLike {
resultIndex: number;
results: ArrayLike<SpeechRecognitionResultLike>;
}
export interface SpeechRecognitionLike {
continuous: boolean;
interimResults: boolean;
lang: string;
onresult: ((event: SpeechRecognitionEventLike) => void) | null;
onerror: ((event: { error: string }) => void) | null;
onend: (() => void) | null;
onstart: (() => void) | null;
start: () => void;
stop: () => void;
}
export interface WindowWithSpeech extends Window {
SpeechRecognition?: new () => SpeechRecognitionLike;
webkitSpeechRecognition?: new () => SpeechRecognitionLike;
}