How to update Realtime SessionConfiguration? #426

using OpenAI;
using OpenAI.Models;
using OpenAI.Realtime;
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using UnityEngine;

public class ReproSteps : MonoBehaviour {
    private OpenAIClient api = null;
    private RealtimeSession session = null;

    private float timer = 10.0f;
    private bool hasUpdated = false;

    private void Start() {
        // Create a session with an initial configuration.
        CreateSession();
    }

    private void Update() {
        // Now try to update the configuration after a few seconds.
        timer -= Time.deltaTime;
        if (timer < 0.0f && !hasUpdated) {
            UpdateSession();
            hasUpdated = true;
        }
    }

    private async void CreateSession() {
        Debug.Log("Creating session.");
        try {
            api = new OpenAIClient(GetOpenAIAuthentication()) { EnableDebug = true };
            session = await api.RealtimeEndpoint.CreateSessionAsync(GetSessionConfiguration(), destroyCancellationToken);
            await session.ReceiveUpdatesAsync<IServerEvent>(OnServerEvent, destroyCancellationToken);
        } catch (Exception e) {
            switch (e) {
                case TaskCanceledException: break;
                case OperationCanceledException: break;
                default: Debug.LogException(e); break;
            }
        } finally {
            session?.Dispose();
            session = null;
            Debug.Log("Session disposed.");
        }
    }

    private async void UpdateSession() {
        Debug.Log("Updating session configuration.");
        try {
            OpenAI.Realtime.SessionConfiguration newConfig = GetUpdatedSessionConfiguration();
            // newConfig.ClientSecret = session.Configuration.ClientSecret; // If we could do this, the error would go away.
            await session.SendAsync(new UpdateSessionRequest(newConfig), destroyCancellationToken);
        } catch (Exception e) {
            Debug.LogException(e);
        }
    }

    private OpenAIAuthentication GetOpenAIAuthentication() {
        // For now just hardcode this.
        const string apiKey = "Your API Key Here";
        const string orgKey = "Your Organisation Key Here";
        const string projKey = "Your Project Key Here";
        return new OpenAIAuthentication(apiKey, orgKey, projKey);
    }

    private OpenAI.Realtime.SessionConfiguration GetSessionConfiguration() {
        return new OpenAI.Realtime.SessionConfiguration(
            Model.GPT4oRealtime,
            modalities: Modality.Text,
            instructions: "You are a helpful AI assistant",
            inputAudioTranscriptionSettings: new OpenAI.Realtime.InputAudioTranscriptionSettings(Model.Transcribe_GPT_4o, language: "en"), // The settings we use to transcribe what the human says. Without this, the human's speech will not get transcibed. Apparently the language setting is fucking useless.
            turnDetectionSettings: new OpenAI.Realtime.ServerVAD(silenceDuration: 2000), // We want Server VAD so that the AI automatically detects when speech starts or ends.
            tools: new List<OpenAI.Tool>(),
            toolChoice: "auto");
    }

    private OpenAI.Realtime.SessionConfiguration GetUpdatedSessionConfiguration() {
        // Let's say we want to only update the instructions.
        return new OpenAI.Realtime.SessionConfiguration(instructions: "You are a fearsome dinosaur. Rawr!");
    }

    private void OnServerEvent(IServerEvent @event) {
        switch (@event) {
            case RealtimeEventError error: throw error;
            default: break;
        }
    }
}