Troubleshooting

/**
 * Report the rate the browser actually gave you, which is not always the rate
 * you asked for. Run this once, on the device and browser you are debugging.
 */
async function reportAudioContextRate() {
  const context = new AudioContext({ sampleRate: 16000 });
  try {
    if (context.state === "suspended") await context.resume();
    console.log("requested 16000, got", context.sampleRate);
    if (context.sampleRate !== 16000) {
      console.warn(
        "This browser refused the requested rate. You must resample to 16000 " +
          "before sending, or the agent will never respond."
      );
    }
    return context.sampleRate;
  } finally {
    await context.close();
  }
}

reportAudioContextRate();

/**
 * Resample mono Float32 audio to a target rate by linear interpolation and
 * convert to the 16-bit signed little-endian PCM the agent accepts.
 *
 * `tail` carries the last sample of the previous block so block boundaries do
 * not click. Pass the returned `tail` and `cursor` back in on the next call.
 *
 * @param {Float32Array} input   One block of captured audio.
 * @param {number} inputRate     The rate the AudioContext actually reported.
 * @param {number} targetRate    16000.
 * @param {{cursor: number, tail: number}} state  Carried between calls.
 * @returns {Int16Array}
 */
function resampleToInt16(input, inputRate, targetRate, state) {
  const ratio = inputRate / targetRate;
  const out = [];
  let position = state.cursor;
  while (position < input.length) {
    const index = Math.floor(position);
    const fraction = position - index;
    const current = index === 0 ? state.tail : input[index - 1];
    const next = input[index];
    const sample = current + (next - current) * fraction;
    const clamped = Math.max(-1, Math.min(1, sample));
    out.push(clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff);
    position += ratio;
  }
  state.cursor = position - input.length;
  state.tail = input[input.length - 1];
  return new Int16Array(out);
}

// Usage, inside an AudioWorkletProcessor or ScriptProcessor callback:
const resampleState = { cursor: 0, tail: 0 };
function onAudioBlock(float32Block, contextSampleRate, send) {
  const pcm = resampleToInt16(float32Block, contextSampleRate, 16000, resampleState);
  if (pcm.length) send(pcm.buffer);
}

/**
 * Convert clamped mono Float32 audio in [-1, 1] to 16-bit signed PCM.
 * The two scale factors are deliberate: Int16 spans -32768..32767, so the
 * negative side is scaled by 0x8000 and the positive side by 0x7fff.
 */
function floatToPcm16(input) {
  const output = new Int16Array(input.length);
  for (let i = 0; i < input.length; i++) {
    const sample = Math.max(-1, Math.min(1, input[i]));
    output[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
  }
  return output;
}

/** Base64-encode an ArrayBuffer without overflowing the call stack. */
function toBase64(arrayBuffer) {
  const bytes = new Uint8Array(arrayBuffer);
  let binary = "";
  const CHUNK = 0x8000;
  for (let i = 0; i < bytes.length; i += CHUNK) {
    binary += String.fromCharCode.apply(null, bytes.subarray(i, i + CHUNK));
  }
  return btoa(binary);
}

/** Send one block of already-resampled 16 kHz PCM. */
function sendPcm(ws, int16Array) {
  ws.send(JSON.stringify({ mime_type: "audio/pcm", data: toBase64(int16Array.buffer) }));
}

/**
 * Acquire a screen share plus the microphone as one MediaStream.
 *
 * getDisplayMedia's audio track is system or tab audio, so it is refused here
 * and the microphone is acquired separately. A screen session with no
 * microphone connects and then never responds.
 *
 * @param {() => void} onUserStoppedSharing Called when the browser's own
 *   "Stop sharing" control ends the track. That control does not touch the
 *   WebSocket, so nothing else will tell you.
 */
async function acquireScreenAndMicrophone(onUserStoppedSharing) {
  const display = await navigator.mediaDevices.getDisplayMedia({
    video: { width: { ideal: 1280 }, height: { ideal: 720 } },
    audio: false,
  });

  let mic;
  try {
    mic = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
  } catch (error) {
    display.getTracks().forEach((track) => track.stop());
    throw new Error(
      "Screen sharing needs the microphone too: the agent replies to speech, " +
        "so a screen session with no audio will connect and then never respond."
    );
  }

  const [videoTrack] = display.getVideoTracks();
  if (videoTrack) videoTrack.addEventListener("ended", onUserStoppedSharing);

  return {
    stream: new MediaStream([...display.getVideoTracks(), ...mic.getAudioTracks()]),
    release: () => {
      display.getTracks().forEach((track) => track.stop());
      mic.getTracks().forEach((track) => track.stop());
    },
  };
}

/**
 * Force a turn without audio, to separate a socket/agent fault from an audio
 * pipeline fault. Works on audio, camera and screen sessions.
 */
function bisectWithText(ws) {
  ws.addEventListener("message", (event) => {
    const message = JSON.parse(event.data);
    if (message.output_transcription) {
      console.log("AGENT REPLIED:", message.output_transcription.text);
    }
    if (message.parts) {
      for (const part of message.parts) {
        if (part.type === "audio/pcm") console.log("agent audio:", part.data.length, "base64 chars");
      }
    }
  });
  ws.send(JSON.stringify({ mime_type: "text/plain", data: "Say the word banana and nothing else." }));
}

curl -X POST https://app.eclatira.com/api/v1/realtime/sessions \
  -H "Authorization: Bearer $ECLATIRA_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "agent_id": "YOUR_AGENT_ID",
    "user_id": "debug-session-1",
    "source": "audio"
  }'

"""Prove the socket and agent are healthy, independent of any audio pipeline.

Usage:
    pip install websockets
    python bisect.py "wss://app.eclatira.com/ws/debug-session-1?session_token=...&source=audio"

Sends one text/plain message, which forces a turn with no audio, and prints
every message the server sends for twenty seconds.
"""

import asyncio
import json
import sys

import websockets


async def main(ws_url: str) -> None:
    async with websockets.connect(ws_url) as ws:

        async def read() -> None:
            async for raw in ws:
                message = json.loads(raw)
                kind = message.get("type")
                if kind == "error":
                    print("ERROR", message["code"], "-", message["error"])
                elif kind:
                    print(kind, message)
                    continue
                for part in message.get("parts") or []:
                    if part["type"] == "audio/pcm":
                        print(f"audio part: {len(part['data'])} base64 chars")
                    else:
                        print(part["type"], part["data"])
                if message.get("output_transcription"):
                    print("agent said:", message["output_transcription"]["text"])
                if message.get("input_transcription"):
                    print("agent heard:", message["input_transcription"]["text"])

        reader = asyncio.create_task(read())
        await ws.send(
            json.dumps({"mime_type": "text/plain", "data": "Say the word banana and nothing else."})
        )
        await asyncio.sleep(20)
        reader.cancel()


asyncio.run(main(sys.argv[1]))