Dialog-RSN-1 API

import asyncio
import os
from openai import AsyncOpenAI

client = AsyncOpenAI(
    api_key=os.environ["POLYAI_API_KEY"],
    websocket_base_url="wss://api.studio.poly.ai/v1",
)

async def main():
    async with client.realtime.connect(model="dialog-rsn-1") as conn:
        async for event in conn:
            print(event.type)  # session.created
            break

asyncio.run(main())

import WebSocket from "ws";

const ws = new WebSocket("wss://api.studio.poly.ai/v1/realtime", {
  headers: { "X-API-KEY": process.env.POLYAI_API_KEY },
});

ws.on("message", (data) => {
  const event = JSON.parse(data.toString());
  console.log(event.type); // session.created
});

ws.on("unexpected-response", (_req, res) => {
  let body = "";
  res.on("data", (chunk) => (body += chunk));
  res.on("end", () => console.error(`refused: ${res.statusCode} ${body}`));
});

"""Stream a WAV file to Dialog-RSN-1 and print the reply."""
import asyncio
import base64
import json
import os
import wave

import websockets

URL = "wss://api.studio.poly.ai/v1/realtime"
KEY = os.environ["POLYAI_API_KEY"]


async def main(path: str) -> None:
    with wave.open(path) as w:
        rate, pcm = w.getframerate(), w.readframes(w.getnframes())

    async with websockets.connect(URL, additional_headers={"X-API-KEY": KEY}) as ws:
        print((await recv(ws))["type"])  # session.created

        await ws.send(json.dumps({"type": "session.update", "session": {
            "instructions": "You work the front desk of a hotel. Keep replies to one sentence.",
            "audio": {"input": {"format": {"type": "audio/pcm", "rate": rate}}},
        }}))
        print((await recv(ws))["type"])  # session.updated

        # 20 ms of PCM16 mono per frame, as a microphone would deliver it.
        step = rate * 2 // 50
        for i in range(0, len(pcm), step):
            await ws.send(json.dumps({"type": "input_audio_buffer.append",
                                      "audio": base64.b64encode(pcm[i:i + step]).decode()}))
            await asyncio.sleep(0.02)
        # Keep sending silence: the server finds the end of the turn itself.
        silence = base64.b64encode(bytes(step)).decode()

        reply = ""
        while True:
            try:
                event = await asyncio.wait_for(recv(ws), 0.02)
            except asyncio.TimeoutError:
                await ws.send(json.dumps({"type": "input_audio_buffer.append", "audio": silence}))
                continue
            t = event["type"]
            if t == "response.output_text.delta":
                reply += event["delta"]
            elif t == "conversation.item.input_audio_transcription.completed":
                print("heard:", event["transcript"])
            elif t == "response.done":
                print("reply:", reply)
                break
            elif t == "error":
                print("error:", event["error"]["code"], event["error"]["message"])
                break


async def recv(ws):
    return json.loads(await ws.recv())


asyncio.run(main("booking_request.wav"))

// Stream a 16 kHz PCM16 mono WAV file to Dialog-RSN-1 and print the reply.
import { readFileSync } from "node:fs";
import WebSocket from "ws";

const RATE = 16000;
const pcm = readFileSync("booking_request.wav").subarray(44); // skip the WAV header
const step = (RATE * 2) / 50; // 20 ms per frame
const silence = Buffer.alloc(step).toString("base64");

const ws = new WebSocket("wss://api.studio.poly.ai/v1/realtime", {
  headers: { "X-API-KEY": process.env.POLYAI_API_KEY },
});
const send = (event: object) => ws.send(JSON.stringify(event));

let reply = "";
let keepAlive: NodeJS.Timeout | undefined;

ws.on("message", (data) => {
  const event = JSON.parse(data.toString());
  switch (event.type) {
    case "session.created":
      send({ type: "session.update", session: {
        instructions: "You work the front desk of a hotel. Keep replies to one sentence.",
        audio: { input: { format: { type: "audio/pcm", rate: RATE } } },
      }});
      break;
    case "session.updated":
      for (let i = 0; i < pcm.length; i += step) {
        send({ type: "input_audio_buffer.append", audio: pcm.subarray(i, i + step).toString("base64") });
      }
      // Keep sending silence: the server finds the end of the turn itself.
      keepAlive = setInterval(() => send({ type: "input_audio_buffer.append", audio: silence }), 20);
      break;
    case "response.output_text.delta":
      reply += event.delta;
      break;
    case "conversation.item.input_audio_transcription.completed":
      console.log("heard:", event.transcript);
      break;
    case "response.done":
      console.log("reply:", reply);
      clearInterval(keepAlive);
      ws.close();
      break;
    case "error":
      console.error("error:", event.error.code, event.error.message);
      clearInterval(keepAlive);
      ws.close();
  }
});

import json
import websockets
from websockets.exceptions import ConnectionClosed, InvalidStatus

try:
    async with websockets.connect(URL, additional_headers={"X-API-KEY": KEY}) as ws:
        async for frame in ws:
            event = json.loads(frame)
            if event["type"] == "error" and event["error"]["code"] == "session_expired":
                break  # 4410 follows; open a new connection and continue the call
except InvalidStatus as e:
    if e.response.status_code == 402:
        print("No balance: add funds in Agent Studio")  # body: {"detail": "..."}
    elif e.response.status_code == 429:
        print("At the session cap or rate limit; retry after", e.response.headers.get("Retry-After"))
    else:
        raise
except ConnectionClosed as e:
    if e.rcvd and e.rcvd.code == 4408:
        pass  # idle; reconnect when there is audio to send

ws.on("unexpected-response", (_req, res) => {
  if (res.statusCode === 402) console.error("No balance: add funds in Agent Studio");
  else if (res.statusCode === 429) console.error("Retry after", res.headers["retry-after"]);
});

ws.on("close", (code) => {
  if (code === 4410) reconnect(); // session reached its maximum duration; carry on in a new one
  if (code === 4408) {} // idle; reconnect when there is audio to send
});