"use client";
import { useCallback, useEffect, useRef, useState } from "react";
import { CedarUnavailable, speakCedar } from "./cedar";
import { pickVoice, sentences } from "./voices";
/**
* Voice in and out, in one hook.
*
* Both halves are browser-native: `SpeechRecognition` for listening (Chrome
* and Edge only) and `speechSynthesis` for speaking (everywhere). Nothing is
* uploaded, no key is held, and no audio is retained — Jackie hears a
* transcript, not a recording.
*/
/** Which engine actually produced the last thing Jackie said. */
export type SpeechEngine = "unknown" | "cedar" | "browser" | "browser-premium";
export interface SpeechApi {
/** Recognition available — speaking usually is even when this is false. */
canListen: boolean;
canSpeak: boolean;
speaking: boolean;
listening: boolean;
/** Rough input level 0..1, for the orb. */
level: number;
/** Resolves when the utterance finishes (or immediately if muted). */
speak: (text: string) => Promise<void>;
/**
* Resolves with what was heard and, when nothing was, why. A denied or
* missing microphone is a different situation from silence, and callers
* need to tell them apart — one should fall back to typing, the other
* should just move on.
*/
listen: () => Promise<{ text: string; error: string | null }>;
/** Live partial transcript while listening. */
interim: string;
stop: () => void;
muted: boolean;
setMuted: (m: boolean) => void;
/**
* What spoke last. Surfaced so a silent fall back to a dated formant synth
* is visible rather than just sounding bad for no stated reason.
*/
engine: SpeechEngine;
}
export function useSpeech(): SpeechApi {
const [speaking, setSpeaking] = useState(false);
const [listening, setListening] = useState(false);
const [interim, setInterim] = useState("");
const [level, setLevel] = useState(0);
const [muted, setMutedState] = useState(false);
const [canListen, setCanListen] = useState(false);
const [canSpeak, setCanSpeak] = useState(false);
const [engine, setEngine] = useState<SpeechEngine>("unknown");
const recognitionRef = useRef<SpeechRecognitionLike | null>(null);
const voiceRef = useRef<SpeechSynthesisVoice | null>(null);
const levelTimer = useRef<number | null>(null);
// Mute lives in a ref as well as state so `speak` can stay referentially
// stable — consumers put it in effect deps, and an identity that changes
// every render turns those effects into a loop.
const mutedRef = useRef(false);
/** Settles the in-flight listen(), so stop() returns what was heard. */
const finishRef = useRef<(() => void) | null>(null);
/** The in-flight cedar utterance, so stop() can cut it off. */
const cedarRef = useRef<{ cancel: () => void } | null>(null);
/** Set once cedar proves unavailable, so we stop asking. */
const cedarOffRef = useRef(false);
/** Whether the chosen browser voice is a high-quality platform variant. */
const premiumRef = useRef(false);
const setMuted = useCallback((m: boolean) => {
mutedRef.current = m;
setMutedState(m);
if (m) window.speechSynthesis?.cancel();
}, []);
useEffect(() => {
const w = window as WindowWithSpeech;
setCanListen(Boolean(w.SpeechRecognition ?? w.webkitSpeechRecognition));
setCanSpeak(typeof window.speechSynthesis !== "undefined");
// Voice list loads async in most browsers.
function choose() {
const voices = window.speechSynthesis?.getVoices() ?? [];
if (voices.length === 0) return;
const { voice, premium } = pickVoice(voices);
voiceRef.current = voice;
premiumRef.current = premium;
}
choose();
window.speechSynthesis?.addEventListener("voiceschanged", choose);
return () => {
window.speechSynthesis?.removeEventListener("voiceschanged", choose);
};
}, []);
const stop = useCallback(() => {
// Resolve whatever listen() is waiting on, so an explicit stop hands back
// the words already heard rather than abandoning the promise.
finishRef.current?.();
finishRef.current = null;
cedarRef.current?.cancel();
cedarRef.current = null;
try {
recognitionRef.current?.stop();
} catch {
/* already stopped */
}
window.speechSynthesis?.cancel();
setSpeaking(false);
setListening(false);
setInterim("");
setLevel(0);
}, []);
useEffect(() => stop, [stop]);
const speak = useCallback(
(text: string) =>
new Promise<void>((resolve) => {
if (mutedRef.current) {
resolve();
return;
}
/* ---- browser synthesis, the fallback ---------------------------- */
const viaBrowser = () => {
if (!window.speechSynthesis) {
resolve();
return;
}
window.speechSynthesis.cancel();
setEngine(premiumRef.current ? "browser-premium" : "browser");
// There is no output analyser for synthesis, so the orb has to be
// driven by an approximation here — cedar gives it real amplitude.
setSpeaking(true);
levelTimer.current = window.setInterval(() => {
setLevel(0.35 + Math.random() * 0.45);
}, 110);
let settled = false;
const done = () => {
if (settled) return;
settled = true;
window.clearTimeout(guard);
if (levelTimer.current) window.clearInterval(levelTimer.current);
levelTimer.current = null;
setSpeaking(false);
setLevel(0);
resolve();
};
// Watchdog: some environments (sandboxed frames, muted OS voices,
// Safari after a background tab) accept `speak` and never fire
// `onend`. Everything downstream awaits this promise, so it has to
// settle on its own schedule — roughly speech tempo, plus slack.
const words = text.split(/\s+/).length;
const budget = Math.min(30000, 2200 + words * 420);
const guard = window.setTimeout(() => done(), budget);
// One utterance per sentence: these engines give a whole paragraph
// a single flat contour, and the gap between sentences is where a
// person would breathe.
const parts = sentences(text);
let index = 0;
const next = () => {
if (settled) return;
const part = parts[index];
index += 1;
if (part === undefined) {
done();
return;
}
const utterance = new SpeechSynthesisUtterance(part);
if (voiceRef.current) utterance.voice = voiceRef.current;
// Slightly under normal: these voices smear consonants when
// pushed, and the content is technical.
utterance.rate = 0.96;
utterance.pitch = 1.0;
utterance.onend = next;
utterance.onerror = next;
window.speechSynthesis.speak(utterance);
};
next();
};
/* ---- cedar, when it is configured -------------------------------- */
if (cedarOffRef.current) {
viaBrowser();
return;
}
setSpeaking(true);
void speakCedar(text, setLevel)
.then(async (handle) => {
setEngine("cedar");
cedarRef.current = handle;
await handle.done;
cedarRef.current = null;
setSpeaking(false);
setLevel(0);
resolve();
})
.catch((e) => {
cedarRef.current = null;
// Remember, so every later line skips the round trip. A missing
// key is a permanent condition, not a transient failure.
if (e instanceof CedarUnavailable) cedarOffRef.current = true;
setSpeaking(false);
setLevel(0);
viaBrowser();
});
}),
[],
);
const listen = useCallback(
() =>
new Promise<{ text: string; error: string | null }>((resolve) => {
const w = window as WindowWithSpeech;
const Ctor = w.SpeechRecognition ?? w.webkitSpeechRecognition;
if (!Ctor) {
resolve({ text: "", error: "unsupported" });
return;
}
const recognition = new Ctor();
// Continuous, because a person thinking mid-sentence is not finished
// speaking. With this false the engine ends at the first pause and the
// second half of the thought is simply lost.
recognition.continuous = true;
recognition.interimResults = true;
recognition.lang = "en-US";
let finalText = "";
let settled = false;
let failure: string | null = null;
let started = false;
let stopping = false;
let restarts = 0;
/** Last moment the microphone actually heard speech. */
let lastVoiceAt = performance.now();
const timers: number[] = [];
let meter: (() => void) | null = null;
const finish = () => {
if (settled) return;
settled = true;
stopping = true;
for (const t of timers) window.clearTimeout(t);
meter?.();
try {
recognition.stop();
} catch {
/* already stopped */
}
recognitionRef.current = null;
setListening(false);
setInterim("");
setLevel(0);
resolve({ text: finalText.trim(), error: failure });
};
finishRef.current = finish;
/* ---- level -------------------------------------------------------
* Amplitude from the microphone itself, not from how much text has
* arrived. It is what tells you the app can actually hear you, and it
* is what decides when you have stopped talking. If capture is
* refused we fall back to the transcript-length approximation rather
* than failing the whole attempt — recognition may still work.
*/
let haveMeter = false;
navigator.mediaDevices
?.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
})
.then((stream) => {
if (settled) {
for (const t of stream.getTracks()) t.stop();
return;
}
const Ctx =
window.AudioContext ??
(window as unknown as { webkitAudioContext?: typeof AudioContext })
.webkitAudioContext;
if (!Ctx) return;
const ctx = new Ctx();
const src = ctx.createMediaStreamSource(stream);
const analyser = ctx.createAnalyser();
analyser.fftSize = 1024;
src.connect(analyser);
const buf = new Float32Array(analyser.fftSize);
haveMeter = true;
let raf = 0;
const tick = () => {
analyser.getFloatTimeDomainData(buf);
let sum = 0;
for (const v of buf) sum += v * v;
const rms = Math.sqrt(sum / buf.length);
// Speech sits well above room tone; this threshold is what
// separates "thinking" from "finished".
if (rms > 0.012) lastVoiceAt = performance.now();
setLevel(Math.min(1, rms * 9));
raf = requestAnimationFrame(tick);
};
raf = requestAnimationFrame(tick);
meter = () => {
cancelAnimationFrame(raf);
for (const t of stream.getTracks()) t.stop();
void ctx.close();
};
})
.catch(() => {
/* no meter; recognition may still work */
});
// Watchdogs. A permission prompt that is never answered — or a
// sandboxed browser that blocks capture outright — fires neither
// `onstart` nor `onerror`, and without these the caller would wait
// on a promise that never settles.
recognition.onstart = () => {
started = true;
lastVoiceAt = performance.now();
};
timers.push(
window.setTimeout(() => {
if (!started && !settled) {
failure = "no-start";
finish();
}
}, 4000),
);
/* ---- endpointing --------------------------------------------------
* Stop when the room has been quiet for a beat *and* something was
* said — not at the engine's first guess that a phrase ended.
*/
const SILENCE_MS = 2400;
const poll = window.setInterval(() => {
if (settled) return;
const quiet = performance.now() - lastVoiceAt;
const heard = finalText.trim().length > 0;
if (heard && quiet > SILENCE_MS) {
window.clearInterval(poll);
finish();
}
}, 250);
timers.push(poll as unknown as number);
// Overall ceiling, so a stuck engine cannot hold the turn forever.
timers.push(
window.setTimeout(() => {
if (!settled) {
if (!finalText.trim()) failure = failure ?? "timeout";
finish();
}
}, 60000),
);
recognition.onresult = (event: SpeechRecognitionEventLike) => {
let partial = "";
for (let i = event.resultIndex; i < event.results.length; i++) {
const result = event.results[i];
const transcript = result[0]?.transcript ?? "";
if (result.isFinal) finalText += `${transcript} `;
else partial += transcript;
}
setInterim(partial);
if (!haveMeter) {
lastVoiceAt = performance.now();
setLevel(
partial.length > 0 ? 0.5 + Math.min(partial.length / 60, 0.45) : 0.25,
);
}
};
recognition.onerror = (event: { error: string }) => {
// "no-speech" is silence, which is a legitimate answer. Everything
// else — denied, no device, service blocked — means the ear is not
// available and the caller should offer a keyboard instead.
if (event.error !== "no-speech" && event.error !== "aborted") {
failure = event.error;
finish();
return;
}
if (event.error === "aborted") finish();
};
recognition.onend = () => {
if (settled) return;
// Chrome ends the session on its own after a pause even when
// `continuous` is set. As long as the person has not asked to stop,
// pick the microphone back up instead of ending their sentence for
// them.
if (!stopping && restarts < 12) {
restarts += 1;
try {
recognition.start();
return;
} catch {
/* fall through to finish */
}
}
finish();
};
recognitionRef.current = recognition;
setListening(true);
setLevel(0);
try {
recognition.start();
} catch (e) {
failure = e instanceof Error ? e.name : "start-failed";
finish();
}
}),
[],
);
return {
canListen,
canSpeak,
speaking,
listening,
level,
speak,
listen,
interim,
stop,
muted,
setMuted,
engine,
};
}
// ---------------------------------------------------------------------------
// Minimal Web Speech surface — no @types/dom-speech-recognition dependency.
// ---------------------------------------------------------------------------
interface SpeechRecognitionAlternativeLike {
transcript: string;
}
interface SpeechRecognitionResultLike {
isFinal: boolean;
[index: number]: SpeechRecognitionAlternativeLike;
}
export interface SpeechRecognitionEventLike {
resultIndex: number;
results: ArrayLike<SpeechRecognitionResultLike>;
}
export interface SpeechRecognitionLike {
continuous: boolean;
interimResults: boolean;
lang: string;
onresult: ((event: SpeechRecognitionEventLike) => void) | null;
onerror: ((event: { error: string }) => void) | null;
onend: (() => void) | null;
onstart: (() => void) | null;
start: () => void;
stop: () => void;
}
export interface WindowWithSpeech extends Window {
SpeechRecognition?: new () => SpeechRecognitionLike;
webkitSpeechRecognition?: new () => SpeechRecognitionLike;
}