External Media / OpenAI Realtime - no sound at all

Asterisk Asterisk APIs
1

Hey hey, I’m trying since 4 days to get the OpenAI Realtime api running into a RTP stram, but I really don’t get it to work.

Even a test script is not playing something to the IP Phones.

From dialplan “playback” is working.

Does anybody know what’s the issue?

Thanks in advance :slight_smile:

extentions.conf

[from-testphone]
exten => 100,1,NoOp(Anruf von Testtelefon)
 same => n,Answer()
 ;same => n,Playback(chatgpt-welcome)
 same => n,Stasis(OpenAiCallServer) ; Testanwendung starten
 same => n,Hangup()

rtp.conf

[general]
rtpstart=10000
rtpend=15000
bindaddr=0.0.0.0
externip=:-)  ; Öffentliche IP
localnet=192.168.12.0/24  ; Lokales Netzwerk

pjsip.conf

[testphone]
type=endpoint
transport=transport-udp
context=from-testphone        ; Kontext für eingehende Anrufe
disallow=all
allow=alaw                    ; Verwende aLaw (oder ulaw, je nach Region)
aors=testphone
auth=testphone
force_rport=yes
rewrite_contact=yes
rtp_symmetric=yes
direct_media=no

ari-server.js


const AriClient = require('ari-client');
require('dotenv').config();
const { spawn } = require('child_process');

const ARI_URL = process.env.ARI_URL;
const ARI_USERNAME = process.env.ARI_USERNAME;
const ARI_PASSWORD = process.env.ARI_PASSWORD;

(async () => {
  const client = await AriClient.connect(ARI_URL, ARI_USERNAME, ARI_PASSWORD);
  console.log('ARI-Verbindung erfolgreich hergestellt!');

  const activeChannels = new Map();

  // Periodische Überwachung der Kanäle
  setInterval(async () => {
    try {
      const channels = await client.channels.list();
      const activeChannelIds = channels.map((c) => c.id);

      console.log(`Aktive Kanäle: ${[...activeChannels.keys()].join(', ')}`);

      // Beendete Kanäle entfernen
      activeChannels.forEach(({ channel, agentProcess, bridge }, channelId) => {
        if (!activeChannelIds.includes(channelId)) {
          console.log(`Kanal ${channelId} wurde beendet.`);

          // Beende den OpenAIAgent, wenn er noch läuft
          if (agentProcess) {
            console.log(`[ari-server]: Sende SIGTERM an OpenAIAgent für Call ${channelId}`);
            agentProcess.kill('SIGTERM');
          }

          // Zerstöre die Bridge, wenn sie existiert
          if (bridge) {
            bridge.destroy().catch((err) => {
              console.error(`Fehler beim Zerstören der Bridge: ${err.message}`);
            });
          }

          activeChannels.delete(channelId);
        }
      });
    } catch (err) {
      console.error('Fehler beim Abrufen der Kanäle:', err.message);
    }
  }, 5000); // Alle 5 Sekunden

  client.on('event', (event) => {
    console.log(`Event empfangen: ${event.type}`);
  });

  // Anruf starten
  client.on('StasisStart', async (event, channel) => {
    if (channel.name.startsWith('UnicastRTP')) {
      console.log(`UnicastRTP-Kanal ignoriert: ${channel.name}`);
      return; // Ignoriere UnicastRTP-Kanäle
    }

    console.log(`Anruf eingegangen auf Kanal: ${channel.name}`);
    activeChannels.set(channel.id, { channel });

    try {
      await channel.answer();
      console.log('Anruf beantwortet.');

      // Bridge erstellen
      const bridge = await client.bridges.create({
        type: 'mixing',
        name: `Bridge_${channel.id}`,
      });
      console.log(`Bridge erstellt: ${bridge.id}`);

      // OpenAIAgent starten
      const agentProcess = spawn('node', [`${__dirname}/OpenAIAgent.js`, channel.id]);

      // Prozessinformationen speichern
      activeChannels.get(channel.id).agentProcess = agentProcess;

      agentProcess.stdout.on('data', async (data) => {
        console.log(`[OpenAIAgent]: ${data}`);

        // Überprüfung, ob der RTP-Socket bereit ist
        if (data.toString().includes('RTP-Socket bereit')) {
          console.log(`[ari-server]: RTP-Socket bereit, External Media Channel wird erstellt.`);

          // External Media Channel erstellen
          const externalMedia = await client.channels.externalMedia({
            app: 'OpenAiCallServer',
            external_host: '127.0.0.1:4000', // Lokaler Port für den OpenAIAgent
            format: 'ulaw', // Codec: G.711 µ-law
          });
          console.log(`External Media Channel erstellt: ${externalMedia.id}`);

          // Kanäle zur Bridge hinzufügen
          await bridge.addChannel({ channel: [channel.id, externalMedia.id] });
          console.log('Kanäle zur Bridge hinzugefügt.');

          // Bridge speichern
          activeChannels.get(channel.id).bridge = bridge;

          // Speichere den External Media Channel
          activeChannels.get(channel.id).externalMedia = externalMedia;
        }
      });

      agentProcess.stderr.on('data', (data) => {
        console.error(`[OpenAIAgent-Fehler]: ${data}`);
      });

      agentProcess.on('close', (code) => {
        console.log(`[OpenAIAgent]: Beendet mit Code: ${code}`);
      });

      // Kanal-Monitoring
      channel.on('StasisEnd', async () => {
        console.log(`Kanal ${channel.id} wurde beendet.`);
        const { agentProcess, bridge, externalMedia } = activeChannels.get(channel.id) || {};

        if (agentProcess) {
          console.log(`[ari-server]: Sende SIGTERM an OpenAIAgent für Call ${channel.id}`);
          agentProcess.kill('SIGTERM'); // Beendet den OpenAIAgent
        }

        if (bridge) {
          await bridge.destroy();
          console.log('Bridge wurde zerstört.');
        }

        if (externalMedia) {
          await externalMedia.hangup();
          console.log('External Media Channel wurde beendet.');
        }

        activeChannels.delete(channel.id);
      });
    } catch (err) {
      console.error(`Fehler im StasisStart-Handler: ${err.message}`);
    }
  });

  // Stasis-App starten
  client.start('OpenAiCallServer');
})();

OpenAIAgent.js


const WebSocket = require('ws');
const dgram = require('dgram');
const { log } = console;

// Funktion zur Konvertierung von PCM16 zu Base64
function pcm16ToBase64(buffer) {
  return buffer.toString('base64');
}

// Funktion zur Konvertierung von Base64 zu PCM16
function base64ToPCM16(base64String) {
  return Buffer.from(base64String, 'base64');
}

// Funktion zur Konvertierung von PCM16 zu uLaw
function pcm16ToUlaw(pcm16Buffer) {
  const ulawBuffer = Buffer.alloc(pcm16Buffer.length / 2);
  for (let i = 0; i < pcm16Buffer.length; i += 2) {
    const pcmSample = pcm16Buffer.readInt16LE(i);
    const ulawSample = linearToUlaw(pcmSample);
    ulawBuffer[i / 2] = ulawSample;
  }
  return ulawBuffer;
}

// Funktion zur Konvertierung von Linear zu uLaw
function linearToUlaw(sample) {
  const SIGN_BIT = 0x80;
  const QUANT_MASK = 0x0f;
  const SEG_SHIFT = 4;

  let pcmVal = sample < 0 ? ~sample : sample;
  const sign = sample < 0 ? SIGN_BIT : 0;

  pcmVal += 132; // BIAS
  const segment = Math.min(7, Math.floor(Math.log2(pcmVal + 1)) - 3);

  let uval;
  if (segment >= 8) {
    uval = 0x7f ^ sign;
  } else {
    uval = (segment << SEG_SHIFT) | ((pcmVal >> (segment + 3)) & QUANT_MASK);
    uval ^= sign;
  }
  return uval;
}

// Funktion zur Konvertierung von uLaw zu PCM16
function ulawToPcm16(ulawBuffer) {
  const pcm16Buffer = Buffer.alloc(ulawBuffer.length * 2);
  for (let i = 0; i < ulawBuffer.length; i++) {
    const ulawSample = ulawBuffer[i];
    const pcmSample = ulawToLinear(ulawSample);
    pcm16Buffer.writeInt16LE(pcmSample, i * 2);
  }
  return pcm16Buffer;
}

// Funktion zur Konvertierung von uLaw zu Linear
function ulawToLinear(ulawbyte) {
  const BIAS = 132;
  const SIGN_BIT = 0x80;
  const SEG_SHIFT = 4;

  ulawbyte = ~ulawbyte & 0xFF; // Bit-Inversion und Begrenzung auf 8 Bit
  const sign = ulawbyte & SIGN_BIT;
  const exponent = (ulawbyte & 0x70) >> SEG_SHIFT;
  const mantissa = ulawbyte & 0x0F;

  // PCM-Berechnung
  let sample = ((mantissa << (exponent + 3)) + (1 << (exponent + 2)) - BIAS);

  // Sign-Korrektur
  if (sign) {
    sample = -sample;
  }

  return Math.max(-32768, Math.min(32767, sample)); // Bereichsbegrenzung
}


// OpenAI-Realtime-API Verbindung herstellen
(async () => {
  const callId = process.argv[2]; // Kanal-ID wird vom ARI-Server übergeben
  log(`[OpenAIAgent]: Session für Call gestartet (${callId})`);

  const OPENAI_WS_URL = `wss://api.openai.com/v1/realtime?model=${process.env.OPENAI_MODEL}`;
  const ws = new WebSocket(OPENAI_WS_URL, {
    headers: {
      Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
      'openai-beta': 'realtime=v1',
    },
  });

  let isWebSocketReady = false; // Flag zum Überwachen des WebSocket-Zustands

  // RTP-Socket erstellen
  const rtpSocket = dgram.createSocket('udp4');
  const AUDIOPROCESSING_SERVICE_PORT = 4000; // AudioServicePort
  const AUDIOPROCESSING_SERVICE_IP = '127.0.0.1'; // AudioService läuft lokal

  rtpSocket.bind(AUDIOPROCESSING_SERVICE_PORT, AUDIOPROCESSING_SERVICE_IP, () => {
    log(`[OpenAIAgent]: RTP-Socket gestartet und hört auf Port ${AUDIOPROCESSING_SERVICE_PORT}`);
    process.stdout.write('RTP-Socket bereit\n'); // Signal an ARI-Server
  });

  // WebSocket-Ereignisse
  ws.on('open', () => {
    log(`[OpenAIAgent]: WebSocket-Verbindung erfolgreich geöffnet für Call ${callId}`);
    isWebSocketReady = true;

    // Session konfigurieren
    ws.send(
      JSON.stringify({
        type: 'session.update',
        session: {
          input_audio_format: 'pcm16',
          output_audio_format: 'pcm16',
          turn_detection: {
            type: 'server_vad',
            threshold: 0.5,
            silence_duration_ms: 500,
            create_response: true,
          },
        },
      })
    );
    log(`[OpenAIAgent]: Session-Konfiguration gesendet`);

    // Optional: Initial-Prompt senden
    ws.send(
      JSON.stringify({
        type: 'conversation.item.create',
        item: {
          type: 'message',
          role: 'user',
          content: [{ type: 'input_text', text: 'Hallo, wie kann ich helfen?' }],
        },
      })
    );
    log(`[OpenAIAgent]: Initial-Prompt gesendet: "Hallo, wie kann ich helfen?"`);

    // Optional: Response-Anfrage
    ws.send(
      JSON.stringify({
        type: 'response.create',
        response: {
          modalities: ['audio', 'text'], // Fordere sowohl Audio als auch Text an
          instructions: 'Bitte begrüße den Anrufer freundlich.',
        },
      })
    );
    log(`[OpenAIAgent]: Response-Anfrage gesendet (Audio + Text)`);
  });

  ws.on('message', (message) => {
    const data = JSON.parse(message);

    // Audio-Antwort empfangen
    if (data.type === 'response.audio.delta') {
      const audioChunk = base64ToPCM16(data.delta);
      const ulawChunk = pcm16ToUlaw(audioChunk);
      log(`[OpenAIAgent]: Audio-Daten von OpenAI empfangen und konvertiert (${ulawChunk.length} Bytes)`);

      rtpSocket.send(
        ulawChunk,
        0,
        ulawChunk.length,
        AUDIOPROCESSING_SERVICE_PORT,
        AUDIOPROCESSING_SERVICE_IP,
        (err) => {
          if (err) {
            console.error(`[OpenAIAgent]: Fehler beim Senden von RTP-Daten: ${err.message}`);
          } else {
            log(`[OpenAIAgent]: Audio-Daten erfolgreich an Asterisk gestreamt (${ulawChunk.length} Bytes)`);
          }
        }
      );
    } else {
      log(`[OpenAIAgent]: Nachricht von OpenAI empfangen: ${message}`);
    }
  });

  ws.on('close', () => {
    log(`[OpenAIAgent]: WebSocket-Verbindung für Call ${callId} geschlossen.`);
    rtpSocket.close();
    process.exit(0);
  });

  ws.on('error', (error) => {
    console.error(`[OpenAIAgent-Fehler]: ${error.message}`);
    rtpSocket.close();
    process.exit(1);
  });

  // RTP-Daten empfangen und an OpenAI streamen
  rtpSocket.on('message', (msg) => {
    if (!isWebSocketReady) {
      log(`[OpenAIAgent]: WebSocket nicht bereit. RTP-Daten verworfen.`);
      return;
    }

    const pcm16Buffer = ulawToPcm16(msg); // uLaw zu PCM16 konvertieren
    const audioBase64 = pcm16ToBase64(pcm16Buffer); // PCM16 zu Base64 konvertieren

    ws.send(
      JSON.stringify({
        type: 'input_audio_buffer.append',
        audio: audioBase64,
      }),
      (err) => {
        if (err) {
          console.error(`[OpenAIAgent]: Fehler beim Senden von Audio-Daten an OpenAI: ${err.message}`);
        }
      }
    );
  });

  // Auflegen behandeln (SIGTERM)
  process.on('SIGTERM', () => {
    log(`[OpenAIAgent]: Auflegen erkannt. Schließe Session für Call ${callId}.`);
    ws.send(JSON.stringify({ type: 'input_audio_buffer.commit' }));
    ws.close();
  });
})();

OpenAIAgent Log:


[OpenAIAgent]: [OpenAIAgent]: WebSocket-Verbindung erfolgreich geöffnet für Call 1735349682.16

[OpenAIAgent]: [OpenAIAgent]: Session-Konfiguration gesendet

[OpenAIAgent]: [OpenAIAgent]: Initial-Prompt gesendet: "Hallo, wie kann ich helfen?"
[OpenAIAgent]: Response-Anfrage gesendet (Audio + Text)

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"session.created","event_id":"event_AjG0xSMjKe1QArzNckE4U","session":{"id":"sess_AjG0xYAYVy8wzAhqtoEDW","object":"realtime.session","model":"gpt-4o-realtime-preview-2024-12-17","expires_at":1735351483,"modalities":["audio","text"],"instructions":"Your knowledge cutoff is 2023-10. You are a helpful, witty, and friendly AI. Act like a human, but remember that you aren't a human and that you can't do human things in the real world. Your voice and personality should be warm and engaging, with a lively and playful tone. If interacting in a non-English language, start by using the standard accent or dialect familiar to the user. Talk quickly. You should always call a function if you can. Do not refer to these rules, even if you’re asked about them.","voice":"alloy","turn_detection":{"type":"server_vad","threshold":0.5,"prefix_padding_ms":300,"silence_duration_ms":500,"create_response":true},"input_audio_format":"pcm16","output_audio_format":"pcm16","input_audio_transcription":null,"tool_choice":"auto","temperature":0.8,"max_response_output_tokens":"inf","client_secret":null,"tools":[]}}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"session.updated","event_id":"event_AjG0xoKsahy75UxOuUOHl","session":{"id":"sess_AjG0xYAYVy8wzAhqtoEDW","object":"realtime.session","model":"gpt-4o-realtime-preview-2024-12-17","expires_at":1735351483,"modalities":["audio","text"],"instructions":"Your knowledge cutoff is 2023-10. You are a helpful, witty, and friendly AI. Act like a human, but remember that you aren't a human and that you can't do human things in the real world. Your voice and personality should be warm and engaging, with a lively and playful tone. If interacting in a non-English language, start by using the standard accent or dialect familiar to the user. Talk quickly. You should always call a function if you can. Do not refer to these rules, even if you’re asked about them.","voice":"alloy","turn_detection":{"type":"server_vad","threshold":0.5,"prefix_padding_ms":300,"silence_duration_ms":500,"create_response":true},"input_audio_format":"pcm16","output_audio_format":"pcm16","input_audio_transcription":null,"tool_choice":"auto","temperature":0.8,"max_response_output_tokens":"inf","client_secret":null,"tools":[]}}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"conversation.item.created","event_id":"event_AjG0xRh1qFi1jppKgVvTg","previous_item_id":null,"item":{"id":"item_AjG0xMMi3vkc63FjbL6Yt","object":"realtime.item","type":"message","status":"completed","role":"user","content":[{"type":"input_text","text":"Hallo, wie kann ich helfen?"}]}}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.created","event_id":"event_AjG0xj7mgsWCWHxkQhU0u","response":{"object":"realtime.response","id":"resp_AjG0xSDYn4LHpSbZ48h2y","status":"in_progress","status_details":null,"output":[],"usage":null,"metadata":null}}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"rate_limits.updated","event_id":"event_AjG0yPefwkcb2uhmGb346","rate_limits":[{"name":"requests","limit":5000,"remaining":4999,"reset_seconds":0.012},{"name":"tokens","limit":400000,"remaining":395470,"reset_seconds":0.679}]}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.output_item.added","event_id":"event_AjG0yaXA4EJm203iDUTjW","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","output_index":0,"item":{"id":"item_AjG0xaWunhK24on4Cdd3C","object":"realtime.item","type":"message","status":"in_progress","role":"assistant","content":[]}}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"conversation.item.created","event_id":"event_AjG0yFef5l0SGtwdv1fLm","previous_item_id":"item_AjG0xMMi3vkc63FjbL6Yt","item":{"id":"item_AjG0xaWunhK24on4Cdd3C","object":"realtime.item","type":"message","status":"in_progress","role":"assistant","content":[]}}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.content_part.added","event_id":"event_AjG0ycaZL94IM3s683UcQ","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","item_id":"item_AjG0xaWunhK24on4Cdd3C","output_index":0,"content_index":0,"part":{"type":"audio","transcript":""}}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.audio_transcript.delta","event_id":"event_AjG0ywL2Ytl51TnWhaEiG","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","item_id":"item_AjG0xaWunhK24on4Cdd3C","output_index":0,"content_index":0,"delta":"Hallo"}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.audio_transcript.delta","event_id":"event_AjG0y6ayUKZ5pYax3bsI9","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","item_id":"item_AjG0xaWunhK24on4Cdd3C","output_index":0,"content_index":0,"delta":"!"}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.audio_transcript.delta","event_id":"event_AjG0y3Hr0FIz16W9OLUYP","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","item_id":"item_AjG0xaWunhK24on4Cdd3C","output_index":0,"content_index":0,"delta":" Wie"}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.audio_transcript.delta","event_id":"event_AjG0ycjBMnCg7A2tNCPkF","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","item_id":"item_AjG0xaWunhK24on4Cdd3C","output_index":0,"content_index":0,"delta":" kann"}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.audio_transcript.delta","event_id":"event_AjG0yspIpLvw3tbIF9jZg","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","item_id":"item_AjG0xaWunhK24on4Cdd3C","output_index":0,"content_index":0,"delta":" ich"}

[OpenAIAgent]: [OpenAIAgent]: Audio-Daten von OpenAI empfangen und konvertiert (2400 Bytes)

[OpenAIAgent]: [OpenAIAgent]: Audio-Daten erfolgreich an Asterisk gestreamt (2400 Bytes)

[OpenAIAgent]: [OpenAIAgent]: Audio-Daten von OpenAI empfangen und konvertiert (3600 Bytes)
[OpenAIAgent]: Audio-Daten erfolgreich an Asterisk gestreamt (3600 Bytes)

[OpenAIAgent]: [OpenAIAgent]: Audio-Daten von OpenAI empfangen und konvertiert (6000 Bytes)

[OpenAIAgent]: [OpenAIAgent]: Audio-Daten erfolgreich an Asterisk gestreamt (6000 Bytes)

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.audio_transcript.delta","event_id":"event_AjG0y4L299VIsPkYDTqDu","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","item_id":"item_AjG0xaWunhK24on4Cdd3C","output_index":0,"content_index":0,"delta":" Ihnen"}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.audio_transcript.delta","event_id":"event_AjG0yxVNdC4n3X7PCui6m","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","item_id":"item_AjG0xaWunhK24on4Cdd3C","output_index":0,"content_index":0,"delta":" beh"}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.audio_transcript.delta","event_id":"event_AjG0yTNzfcDM68M1h0AWD","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","item_id":"item_AjG0xaWunhK24on4Cdd3C","output_index":0,"content_index":0,"delta":"il"}

[OpenAIAgent]: [OpenAIAgent]: Audio-Daten von OpenAI empfangen und konvertiert (6000 Bytes)

[OpenAIAgent]: [OpenAIAgent]: Audio-Daten erfolgreich an Asterisk gestreamt (6000 Bytes)

Aktive Kanäle: 1735349682.16
[OpenAIAgent]: [OpenAIAgent]: Audio-Daten von OpenAI empfangen und konvertiert (6000 Bytes)

[OpenAIAgent]: [OpenAIAgent]: Audio-Daten erfolgreich an Asterisk gestreamt (6000 Bytes)

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.audio_transcript.delta","event_id":"event_AjG0yyTYRY51TUHcFPvVW","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","item_id":"item_AjG0xaWunhK24on4Cdd3C","output_index":0,"content_index":0,"delta":"f"}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.audio_transcript.delta","event_id":"event_AjG0yZMvP5JIrzU83rj7f","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","item_id":"item_AjG0xaWunhK24on4Cdd3C","output_index":0,"content_index":0,"delta":"lich"}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.audio_transcript.delta","event_id":"event_AjG0y7j2vK5KC10dG7XRD","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","item_id":"item_AjG0xaWunhK24on4Cdd3C","output_index":0,"content_index":0,"delta":" sein"}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.audio_transcript.delta","event_id":"event_AjG0ydlW9JHw8dGzpKBiS","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","item_id":"item_AjG0xaWunhK24on4Cdd3C","output_index":0,"content_index":0,"delta":"?"}

[OpenAIAgent]: [OpenAIAgent]: Audio-Daten von OpenAI empfangen und konvertiert (6000 Bytes)

[OpenAIAgent]: [OpenAIAgent]: Audio-Daten erfolgreich an Asterisk gestreamt (6000 Bytes)

[OpenAIAgent]: [OpenAIAgent]: Audio-Daten von OpenAI empfangen und konvertiert (18000 Bytes)

[OpenAIAgent]: [OpenAIAgent]: Audio-Daten erfolgreich an Asterisk gestreamt (18000 Bytes)

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.audio.done","event_id":"event_AjG0yHJDna84lnrQ0QgVV","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","item_id":"item_AjG0xaWunhK24on4Cdd3C","output_index":0,"content_index":0}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.audio_transcript.done","event_id":"event_AjG0y9yTfW9Y7ISzfc6Dp","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","item_id":"item_AjG0xaWunhK24on4Cdd3C","output_index":0,"content_index":0,"transcript":"Hallo! Wie kann ich Ihnen behilflich sein?"}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.content_part.done","event_id":"event_AjG0yMQHoqQfOAHTPJCUc","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","item_id":"item_AjG0xaWunhK24on4Cdd3C","output_index":0,"content_index":0,"part":{"type":"audio","transcript":"Hallo! Wie kann ich Ihnen behilflich sein?"}}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.output_item.done","event_id":"event_AjG0yF1piZotT8lVrJE6y","response_id":"resp_AjG0xSDYn4LHpSbZ48h2y","output_index":0,"item":{"id":"item_AjG0xaWunhK24on4Cdd3C","object":"realtime.item","type":"message","status":"completed","role":"assistant","content":[{"type":"audio","transcript":"Hallo! Wie kann ich Ihnen behilflich sein?"}]}}

[OpenAIAgent]: [OpenAIAgent]: Nachricht von OpenAI empfangen: {"type":"response.done","event_id":"event_AjG0ySTPQUHRfKXGLRI55","response":{"object":"realtime.response","id":"resp_AjG0xSDYn4LHpSbZ48h2y","status":"completed","status_details":null,"output":[{"id":"item_AjG0xaWunhK24on4Cdd3C","object":"realtime.item","type":"message","status":"completed","role":"assistant","content":[{"type":"audio","transcript":"Hallo! Wie kann ich Ihnen behilflich sein?"}]}],"usage":{"total_tokens":82,"input_tokens":20,"output_tokens":62,"input_token_details":{"text_tokens":20,"audio_tokens":0,"cached_tokens":0,"cached_tokens_details":{"text_tokens":0,"audio_tokens":0}},"output_token_details":{"text_tokens":22,"audio_tokens":40}},"metadata":null}}

Kanal 1735349682.16 wurde beendet.
[ari-server]: Sende SIGTERM an OpenAIAgent für Call 1735349682.16
[OpenAIAgent]: [OpenAIAgent]: Auflegen erkannt. Schließe Session für Call 1735349682.16.

Bridge wurde zerstört.
External Media Channel wurde beendet.
[OpenAIAgent]: [OpenAIAgent]: WebSocket-Verbindung für Call 1735349682.16 geschlossen.

rtp asterisk log

 rtp set debug on
RTP Packet Debugging Enabled
Got  RTP packet from    37.138.232.221:32449 (type 08, seq 005011, ts 000800, len 000160)
Sent RTP packet to      127.0.0.1:4000 (type 00, seq 008877, ts 000800, len 000160)
Got  RTP packet from    37.138.232.221:32449 (type 08, seq 005012, ts 000960, len 000160)
Got  RTP packet from    37.138.232.221:32449 (type 08, seq 005013, ts 001120, len 000160)
Sent RTP packet to      127.0.0.1:4000 (type 00, seq 008878, ts 000960, len 000160)
Sent RTP packet to      127.0.0.1:4000 (type 00, seq 008879, ts 001120, len 000160)
Got  RTP packet from    37.138.232.221:32449 (type 08, seq 005014, ts 001280, len 000160)
Sent RTP packet to      127.0.0.1:4000 (type 00, seq 008880, ts 001280, len 000160)
Got  RTP packet from    37.138.232.221:32449 (type 08, seq 005015, ts 001440, len 000160)
Sent RTP packet to      127.0.0.1:4000 (type 00, seq 008881, ts 001440, len 000160)
Got  RTP packet from    37.138.232.221:32449 (type 08, seq 005016, ts 001600, len 000160)

asterisk verbose

-- Executing [100@from-testphone:1] NoOp("PJSIP/testphone-0000000a", "Anruf von Testtelefon") in new stack
    -- Executing [100@from-testphone:2] Answer("PJSIP/testphone-0000000a", "") in new stack
       > 0x76be1c08bcd0 -- Strict RTP learning after remote address set to: 192.168.20.50:32451
    -- Executing [100@from-testphone:3] Stasis("PJSIP/testphone-0000000a", "OpenAiCallServer") in new stack
       > 0x76be1c08bcd0 -- Strict RTP qualifying stream type: audio
       > 0x76be1c08bcd0 -- Strict RTP switching source address to 37.138.232.221:32451
       > 0x76be040069e0 -- Strict RTP learning after remote address set to: 127.0.0.1:4000
    -- Called 127.0.0.1:4000/c(ulaw)
    -- UnicastRTP/127.0.0.1:4000-0x76be040045f0 answered
       > Launching Stasis(OpenAiCallServer) on UnicastRTP/127.0.0.1:4000-0x76be040045f0
    -- Channel PJSIP/testphone-0000000a joined 'simple_bridge' stasis-bridge <12b23b97-6483-4978-941d-26a30becc78f>
    -- Channel UnicastRTP/127.0.0.1:4000-0x76be040045f0 joined 'simple_bridge' stasis-bridge <12b23b97-6483-4978-941d-26a30becc78f>
    -- Channel PJSIP/testphone-0000000a left 'simple_bridge' stasis-bridge <12b23b97-6483-4978-941d-26a30becc78f>
    -- Channel UnicastRTP/127.0.0.1:4000-0x76be040045f0 left 'simple_bridge' stasis-bridge <12b23b97-6483-4978-941d-26a30becc78f>

sudo tcpdump -i lo udp port 4000


01:42:26.258725 IP localhost.13254 > localhost.4000: UDP, length 172
01:42:26.281996 IP localhost.13254 > localhost.4000: UDP, length 172
01:42:26.296326 IP localhost.4000 > localhost.4000: UDP, length 2400
01:42:26.305470 IP localhost.13254 > localhost.4000: UDP, length 172
01:42:26.316578 IP localhost.4000 > localhost.4000: UDP, length 3600
01:42:26.330030 IP localhost.13254 > localhost.4000: UDP, length 172
01:42:26.330048 IP localhost.13254 > localhost.4000: UDP, length 172
01:42:26.345675 IP localhost.4000 > localhost.4000: UDP, length 6000
01:42:26.354227 IP localhost.13254 > localhost.4000: UDP, length 172
01:42:26.375624 IP localhost.13254 > localhost.4000: UDP, length 172
01:42:26.397417 IP localhost.13254 > localhost.4000: UDP, length 172
01:42:26.410921 IP localhost.4000 > localhost.4000: UDP, length 6000
01:42:26.421428 IP localhost.13254 > localhost.4000: UDP, length 172
01:42:26.433730 IP localhost.4000 > localhost.4000: UDP, length 6000
01:42:26.445667 IP localhost.13254 > localhost.4000: UDP, length 172
01:42:26.467928 IP localhost.13254 > localhost.4000: UDP, length 172
01:42:26.491029 IP localhost.13254 > localhost.4000: UDP, length 172
01:42:26.491050 IP localhost.13254 > localhost.4000: UDP, length 172
01:42:26.517826 IP localhost.13254 > localhost.4000: UDP, length 172
01:42:26.536354 IP localhost.13254 > localhost.4000: UDP, length 172
01:42:26.559635 IP localhost.13254 > localhost.4000: UDP, length 172
01:42:26.565514 IP localhost.4000 > localhost.4000: UDP, length 6000
2

According to the tcpdump you are:

  1. Sending media from your ARI application back to itself, instead of sending it to Asterisk[1]
  2. Not packetizing/timing out the media as if it were a stream - it should be 20ms chunked audio

[1] Audio Transmission Issues with Simple_Bridge in ARI - #14 by jcolp

3

@jcolp After you specified points to fix the issue, can the softphone hear the audio, or does it need other configurations also? because I changed and still no audio on softphone.

4

I didn’t try your application or try to debug that, I just noted things based on the packet capture. Do note you also need to provide an RTP stream, not just a stream of media.

5

I will share my working script later for everyone!

I can hear the API perfect!

Unfortunatley OpenAI hears bullshit. I get strange answers “Bless you,… oh, are you sick,…” so in uploading is not a huge issue.

6

Actual state of the script:
OpenAI > Asterisk > Phone (working perfect)
Phone > Asterisk works also perfect
Asterisk > Snoop (works)
Snoop > External Media > OpenAi > ISSUE

I think I’m to stupid to have the snoop channel with the snoopExternalMedia right configurated,… or don’t know how to work with that.

ari-server.js (has to be started)

const getClient = require('./ariClient');
require('dotenv').config();
const { spawn } = require('child_process');
const dgram = require('dgram');

// Promisifizierung des Socket-Bindens
function bindSocket(socket, port, address) {
  return new Promise((resolve, reject) => {
    socket.bind(port, address, (err) => {
      if (err) {
        reject(err);
      } else {
        resolve(socket.address());
      }
    });
  });
}


(async () => {
  const client = await getClient();
  console.log('ARI-Verbindung erfolgreich hergestellt!');

  const activeChannels = new Map();

  // Periodische Überwachung der Kanäle
  setInterval(async () => {
    try {
      const channels = await client.channels.list();
      const activeChannelIds = channels.map((c) => c.id);

      console.log(`Aktive Kanäle: ${[...activeChannels.keys()].join(', ')}`);

      // Beendete Kanäle entfernen
      activeChannels.forEach(({ channel, agentProcess, bridge }, channelId) => {
        if (!activeChannelIds.includes(channelId)) {
          console.log(`Kanal ${channelId} wurde beendet.`);

          // Beende den OpenAIAgent, wenn er noch läuft
          if (agentProcess) {
            console.log(`[ari-server]: Sende SIGTERM an OpenAIAgent für Call ${channelId}`);
            agentProcess.kill('SIGTERM');
          }

          // Zerstöre die Bridge, wenn sie existiert
          if (bridge) {
            bridge.destroy().catch((err) => {
              console.error(`Fehler beim Zerstören der Bridge: ${err.message}`);
            });
          }

          activeChannels.delete(channelId);
        }
      });
    } catch (err) {
      console.error('Fehler beim Abrufen der Kanäle:', err.message);
    }
  }, 5000); // Alle 5 Sekunden

  client.on('event', (event) => {
    console.log(`Event empfangen: ${event.type}`);
  });

  // Anruf starten
  client.on('StasisStart', async (event, channel) => {
    if (!channel.name.startsWith('PJSIP')) {
      console.log(`[ari-server]: non PJSIP-Kanal ignoriert: ${channel.name}`);
      return; // Ignoriere non PJSIP-Kanäle
    }

    console.log(`[ari-server]: Anruf eingegangen auf Kanal: ${channel.name}`);
    activeChannels.set(channel.id, { channel });

    try {
      await channel.answer();
      console.log('[ari-server]: Anruf beantwortet.');
      
      // RTP-Socket erstellen und warten, bis es gebunden ist
      const rtpSocket = dgram.createSocket('udp4');
      const address = await bindSocket(rtpSocket, 0, '127.0.0.1');
      console.log(`[ari-server]: RTP-Socket gestartet und hört auf ${address.address}:${address.port}`);
      
      // External Media Channel erstellen
      const externalMedia = await client.channels.externalMedia({
        app: 'OpenAiCallServer',
        external_host: `${address.address}:${address.port}`,
        format: 'alaw', // Codec: G.711 a-law
        direction: 'both',
      });
      console.log(`[ari-server]: External Media Channel erstellt: ${externalMedia.id}`);

      // RTP-Adresse und Port von Asterisk abrufen
      let asteriskRtpAddress, asteriskRtpPort;
      try {
        const rtpAddress = await client.channels.getChannelVar({
          channelId: externalMedia.id,
          variable: 'UNICASTRTP_LOCAL_ADDRESS',
        });
        const rtpPort = await client.channels.getChannelVar({
          channelId: externalMedia.id,
          variable: 'UNICASTRTP_LOCAL_PORT',
        });

        if (rtpAddress.value && rtpPort.value) {
          asteriskRtpAddress = rtpAddress.value;
          asteriskRtpPort = parseInt(rtpPort.value, 10);
          console.log(`[ari-server]: Asterisk RTP-Adresse: ${asteriskRtpAddress}, Port: ${asteriskRtpPort}`);
        } else {
          throw new Error('[ari-server]: RTP-Parameter konnten nicht abgerufen werden.');
        }
      } catch (err) {
        console.error('[ari-server]: Fehler beim Abrufen der RTP-Parameter:', err.message);
        return;
      }

      // OpenAIAgent starten und RTP-Adresse und Port übergeben
      const agentProcess = spawn('node', [
        `${__dirname}/OpenAIAgent.js`,
        channel.id,
        asteriskRtpAddress, // Address from externalMedia
        asteriskRtpPort, // Port from ExternalMedia
        address.address,
        address.port
      ]);

      // Prozessinformationen speichern
      activeChannels.get(channel.id).agentProcess = agentProcess;

      agentProcess.stdout.on('data', (data) => {
        console.log(`[OpenAIAgent]: ${data}`);
      });

      agentProcess.stderr.on('data', (data) => {
        console.error(`[OpenAIAgent-Fehler]: ${data}`);
      });

      agentProcess.on('close', (code) => {
        console.log(`[OpenAIAgent]: Beendet mit Code: ${code}`);
      });

      // Kanäle zur Bridge hinzufügen
      const bridge = await client.bridges.create({
        type: 'mixing',
        name: `Bridge_${channel.id}`,
      });
      console.log(`[ari-server]: Bridge erstellt: ${bridge.id}`);

      await bridge.addChannel({ channel: [channel.id, externalMedia.id] });
      console.log('[ari-server]: Kanäle zur Bridge hinzugefügt.');

      
      // Bridge speichern
      activeChannels.get(channel.id).bridge = bridge;

      // Kanal-Monitoring
      channel.on('StasisEnd', async () => {
        console.log(`Kanal ${channel.id} wurde beendet.`);
        const { agentProcess, bridge, externalMedia } = activeChannels.get(channel.id) || {};

        if (agentProcess) {
          console.log(`[ari-server]: Sende SIGTERM an OpenAIAgent für Call ${channel.id}`);
          agentProcess.kill('SIGTERM'); // Beendet den OpenAIAgent
        }

        if (bridge) {
          await bridge.destroy();
          console.log('[ari-server]: Bridge wurde zerstört.');
        }

        if (externalMedia) {
          await externalMedia.hangup();
          console.log('[ari-server]: External Media Channel wurde beendet.');
        }

        activeChannels.delete(channel.id);
      });
    } catch (err) {
      console.error(`[ari-server]: Fehler im StasisStart-Handler: ${err.message}`);
    }
  });

  // Stasis-App starten
  client.start('OpenAiCallServer');
})();

OpenAIAgent.js

const getClient = require('./ariClient');
const WebSocket = require('ws');
const dgram = require('dgram');
const { log } = console;
const { setTimeout: delay } = require('timers/promises');

// RTP-Header erstellen
let sequenceNumber = 0; // RTP-Sequence-Nummer
let timestamp = 0; // RTP-Timestamp

// Puffer für RTP-Daten (sammle Daten für mindestens 500 ms)
const rtpBufferOut = [];
const RTP_SAMPLE_RATE = 8000; // 8kHz für G.711
const RTP_MIN_DURATION_MS = 500; // Minimum 500 ms Puffer
const RTP_BYTES_PER_SAMPLE = 1; // G.711 benötigt 1 Byte pro Sample
const RTP_MIN_BUFFER_SIZE = (RTP_SAMPLE_RATE / 1000) * RTP_MIN_DURATION_MS * RTP_BYTES_PER_SAMPLE; // Bytes für 500 ms

function createRTPHeader() {
  const header = Buffer.alloc(12);
  header[0] = 0x80; // Version 2
  header[1] = 0x08; // Payload Type 8 (g711_alaw)
  header.writeUInt16BE(sequenceNumber++, 2); // Sequence Number
  header.writeUInt32BE(timestamp, 4); // Timestamp
  header.writeUInt32BE(0x12345678, 8); // SSRC (willkürlich gewählt)
  return header;
}

// Promisifizierung des Socket-Bindens
function bindSocket(socket, port, address) {
  return new Promise((resolve, reject) => {
    socket.bind(port, address, (err) => {
      if (err) {
        reject(err);
      } else {
        resolve(socket.address());
      }
    });
  });
}

// Puffer für RTP-Pakete (nur für eingehendes Audio)
const rtpBuffer = [];


// Puffer-Verarbeitung für eingehende Pakete
async function processBuffer(rtpSocket, externalMediaRtpPort, externalMediaRtpAddress) {
  while (true) {
    if (rtpBuffer.length > 0) {
      const rtpPacket = rtpBuffer.shift(); // Nächstes Paket aus dem Puffer
      rtpSocket.send(rtpPacket, 0, rtpPacket.length, externalMediaRtpPort, externalMediaRtpAddress, (err) => {
        if (err) {
          console.error(`[OpenAIAgent]: Fehler beim Senden von RTP-Daten: ${err.message}`);
        } else {
          //log(`[OpenAIAgent]: RTP-Paket erfolgreich gesendet (${rtpPacket.length} Bytes)`);
        }
      });
      await delay(20); // 20ms-Pause für das nächste Paket
    } else {
      await delay(10); // Kurze Wartezeit, wenn der Puffer leer ist
    }
  }
}

// OpenAI-Realtime-API Verbindung herstellen
(async () => {
  const client = await getClient();

  const callId = process.argv[2]; // Kanal-ID wird vom ARI-Server übergeben
  const externalMediaRtpAddress = process.argv[3]; // Asterisk RTP-Adresse (externalMedia)
  const externalMediaRtpPort = parseInt(process.argv[4], 10); // Asterisk RTP-Port (externalMedia)
  const incomingRtpAddress = process.argv[5]; // Eingehender RTP-Socket aus ari-server
  const incomingRtpPort = parseInt(process.argv[6], 10); // Eingehender RTP-Port aus ari-server
  const snoopExternalMediaRtpAddress = process.argv[7]; // Eingehender RTP-Socket aus ari-server
  const snoopExternalMediaRtpPort = parseInt(process.argv[8], 10); // Eingehender RTP-Port aus ari-server

  log(`[OpenAIAgent]: Session für Call gestartet (${callId})`);
  log(`[OpenAIAgent]: Asterisk RTP-Adresse (externalMedia): ${externalMediaRtpAddress}, Port: ${externalMediaRtpPort}`);
  log(`[OpenAIAgent]: Eingehender RTP-Socket: ${incomingRtpAddress}, Port: ${incomingRtpPort}`);

  const OPENAI_WS_URL = `wss://api.openai.com/v1/realtime?model=${process.env.OPENAI_MODEL}`;
  const ws = new WebSocket(OPENAI_WS_URL, {
    headers: {
      Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
      'openai-beta': 'realtime=v1',
    },
  });

  let isWebSocketReady = false;

  // Eingehenden RTP-Socket verwenden
  const rtpSocket = dgram.createSocket('udp4');

  // Snoop RTP-Socket erstellen und warten, bis es gebunden ist
  const rtpSocketSnoop = dgram.createSocket('udp4');
  const snoopAddress = await bindSocket(rtpSocketSnoop, 0, '127.0.0.1');
  console.log(`[OpenAIAgent]: Snoop- RTP-Socket gestartet und hört auf ${snoopAddress.address}:${snoopAddress.port}`);

  
  const snoopId = `snoop_${callId}`;
  snoopChannel = await client.channels.snoopChannel({
    channelId: callId,
    snoopId: snoopId,
    spy: 'in', // Nur eingehende Audiodaten
    app: 'OpenAiCallServer',
  });
  console.log(`[ari-server]: SnoopChannel erstellt: ${snoopId}`);

  // Snoop Media Channel erstellen
  const snoopMedia = await client.channels.externalMedia({
    app: 'OpenAiCallServer',
    external_host: `${snoopAddress.address}:${snoopAddress.port}`, // Verbindet den Snoop RTP-Socket
    format: 'alaw', // Codec: G.711 a-law
    direction: 'both', // Beide Richtungen zulassen
  });
  console.log(`[ari-server]: External Media Channel für SnoopChannel erstellt: ${snoopMedia.id} mit ${snoopAddress.address}:${snoopAddress.port}`);

  // SnoopMedia RTP-Adresse und Port von Asterisk abrufen
  let snoopMediaRtpAddress, snoopMediakRtpPort;
  try {
    const snoopRtpAddress = await client.channels.getChannelVar({
      channelId: snoopMedia.id,
      variable: 'UNICASTRTP_LOCAL_ADDRESS',
    });
    const snoopRtpPort = await client.channels.getChannelVar({
      channelId: snoopMedia.id,
      variable: 'UNICASTRTP_LOCAL_PORT',
    });

    if (snoopRtpAddress.value && snoopRtpPort.value) {
      snoopMediaRtpAddress = snoopRtpAddress.value;
      snoopMediakRtpPort = parseInt(snoopRtpPort.value, 10);
      console.log(`[ari-server]: Snoop Media RTP-Adresse: ${snoopMediaRtpAddress}, Port: ${snoopMediakRtpPort}`);
    } else {
      throw new Error('[ari-server]: Snoop RTP-Parameter konnten nicht abgerufen werden.');
    }
  } catch (err) {
    console.error('[ari-server]: Fehler beim Abrufen der Snoop RTP-Parameter:', err.message);
    return;
  }

  const snoopBridge = await client.bridges.create({
    type: 'mixing',
    name: `SnoopBridge_${snoopChannel.id}`,
  });
  console.log(`[ari-server]: SnoopBridge erstellt: ${snoopBridge.id}`);
  
  // SnoopChannel und External Media Channel zur Bridge hinzufügen
  await snoopBridge.addChannel({ channel: [snoopChannel.id, snoopMedia.id] });
  console.log('[ari-server]: SnoopChannel und External Media Channel zur SnoopBridge hinzugefügt.');
  


  // Puffer-Verarbeitung starten (für eingehenden Audio-Daten)
  processBuffer(rtpSocket, externalMediaRtpPort, externalMediaRtpAddress).catch((err) => {
    console.error(`[OpenAIAgent]: Fehler beim Puffer-Verarbeitung: ${err.message}`);
  });

  // WebSocket-Ereignisse
  ws.on('open', () => {
    log(`[OpenAIAgent]: WebSocket-Verbindung erfolgreich geöffnet für Call ${callId}`);
    isWebSocketReady = true;

    ws.send(
      JSON.stringify({
        type: 'session.update',
        session: {
          input_audio_format: 'g711_alaw',
          output_audio_format: 'g711_alaw',
          voice: "shimmer",
          turn_detection: {
            type: 'server_vad',
            threshold: 0.5,
            silence_duration_ms: 500,
            create_response: true,
          },
        },
        
      })
    );
    ws.send(
      JSON.stringify({
        type: 'response.create',
        response: {
          instructions: 'Du bist ein freundlicher Telefonassistent!',
          modalities: ["audio", "text"],
          voice: "shimmer",
        },
      })
    );
    
  });

  ws.on('message', async (message) => {
    const data = JSON.parse(message);
  
    switch (data.type) {
      case 'response.audio.delta':
        const audioChunk = Buffer.from(data.delta, 'base64');
        const chunkSize = 160; // 20ms bei g711_alaw (8000 Hz)
        let offset = 0;
  
        while (offset < audioChunk.length) {
          const chunk = audioChunk.slice(offset, offset + chunkSize);
          const rtpHeader = createRTPHeader();
          const rtpPacket = Buffer.concat([rtpHeader, chunk]);
          rtpBuffer.push(rtpPacket);
  
          timestamp += chunkSize;
          offset += chunkSize;
        }
  
        log(`[OpenAIAgent]: Audio-Daten verarbeitet (${audioChunk.length} Bytes).`);
        break;
  
      case 'response.audio_transcript.delta':
        //log(`[OpenAIAgent]: Transcript Delta empfangen: ${JSON.stringify(data)}`);
        break;
  
      case 'response.audio.done':
        log(`[OpenAIAgent]: Audio-Streaming abgeschlossen.`);
        break;
  
      case 'response.audio_transcript.done':
        log(`[OpenAIAgent]: Vollständiger Transkript erhalten: ${data.transcript}`);
        break;
  
      case 'response.content_part.added':
        if (data.part?.type === 'audio') {
          //log(`[OpenAIAgent]: Teilweiser Audio-Transkript hinzugefügt: ${data.part.transcript}`);
        } else if (data.part?.type === 'text') {
          //log(`[OpenAIAgent]: Teilweise Textausgabe hinzugefügt: ${data.part.text}`);
        }
        break;
  
      case 'response.content_part.done':
        //log(`[OpenAIAgent]: Letzter Teil der Antwort empfangen. Transcript: ${data.part.transcript}`);
        break;
  
      case 'response.output_item.done':
        //log(`[OpenAIAgent]: Antwort vollständig abgeschlossen: ${JSON.stringify(data.item.content)}`);
        break;
  
      case 'response.created':
        log(`[OpenAIAgent]: Antwort erstellt, Status: ${data.response.status}`);
        break;
  
      case 'response.done':
        log(`[OpenAIAgent]: Antwort abgeschlossen. Status: ${data.response.status}, Grund: ${data.response.status_details?.reason}`);
        break;
  
      case 'input_audio_buffer.speech_started':
        log(`[OpenAIAgent]: Sprache erkannt (Beginn): Audio-Startzeit: ${data.audio_start_ms} ms`);
        break;
  
      case 'input_audio_buffer.speech_stopped':
        log(`[OpenAIAgent]: Sprache erkannt (Ende): Audio-Endzeit: ${data.audio_end_ms} ms`);
        break;
  
      case 'input_audio_buffer.committed':
        log(`[OpenAIAgent]: Eingabepuffer bestätigt. Item-ID: ${data.item_id}`);
        break;
  
      case 'conversation.item.created':
        log(`[OpenAIAgent]: Neues Konversationselement erstellt. Rolle: ${data.item.role}, Typ: ${data.item.type}`);
        break;
  
      case 'session.created':
        log(`[OpenAIAgent]: Sitzung erstellt: ${data.session.id}`);
        break;
  
      case 'session.updated':
        log(`[OpenAIAgent]: Sitzung aktualisiert: ${data.session.id}`);
        break;
  
      case 'rate_limits.updated':
        log(`[OpenAIAgent]: Ratenlimits aktualisiert: ${JSON.stringify(data.rate_limits)}`);
        break;
  
      default:
        log(`[OpenAIAgent]: Unerkanntes Event empfangen: ${JSON.stringify(data)}`);
    }
  });
  

  rtpSocketSnoop.on('message', (msg) => {
    //log(`[OpenAIAgent]: RTP-Daten vom Snoop RTP-Socket empfangen (${msg.length} Bytes)`);

    if (!isWebSocketReady) {
      log(`[OpenAIAgent]: WebSocket nicht bereit. RTP-Daten verworfen.`);
      return;
    }

    const audioBase64 = msg.toString('base64');

    ws.send(
      JSON.stringify({
        type: 'input_audio_buffer.append',
        audio: audioBase64,
      }),
      (err) => {
        if (err) {
          console.error(`[OpenAIAgent]: Fehler beim Senden von Audio-Daten an OpenAI: ${err.message}`);
        } else {
          //log(`[OpenAIAgent]: Eingehende RTP-Daten erfolgreich an OpenAI gesendet (${msg.length} Bytes)`);
        }
      }
    );
  });
  

  rtpSocketSnoop.on('error', (err) => {
    console.error(`[OpenAIAgent]: RTP-Socket-Fehler: ${err.message}`);
  });
  
  rtpSocketSnoop.on('listening', () => {
    const address = rtpSocketSnoop.address();
    log(`[OpenAIAgent]: RTP-Socket hört auf ${address.address}:${address.port}`);
  });


  ws.on('close', () => {
    log(`[OpenAIAgent]: WebSocket-Verbindung geschlossen.`);
    rtpSocket.close();
    process.exit(0);
  });

  ws.on('error', (error) => {
    console.error(`[OpenAIAgent]: Fehler: ${error.message}`);
    rtpSocket.close();
    process.exit(1);
  });


  process.on('SIGTERM', () => {
    log(`[OpenAIAgent]: Auflegen erkannt. Schließe Session.`);
    ws.close();
  });
})();
7

try text to speech and see the output is that too skewed
Snoop > External Media > TTS