Modernize Realtime API and full-duplex conversation flow

Examples/RealtimeExample/README.md

@@ -1,267 +1,143 @@
 # OpenAI Realtime API Example
 
-This example demonstrates how to use SwiftOpenAI's Realtime API for bidirectional voice conversations with OpenAI's GPT-4o models.
+This example demonstrates SwiftOpenAI's Realtime API support for bidirectional voice conversations with `gpt-realtime-2.1` and local function tool calling.
 
 ## Features
 
-- Real-time bidirectional audio streaming
-- Voice Activity Detection (VAD) for automatic turn-taking
-- Audio transcription of both user and AI speech
-- Function calling support
-- Interrupt handling when user starts speaking
+- Bidirectional audio streaming over WebSocket
+- Semantic VAD for automatic turn-taking
+- User and assistant transcripts
+- Local function tools with `function_call_output`
+- Playback interruption when the user starts speaking
 
 ## Requirements
 
-- iOS 15+, macOS 12+, watchOS 9+
+- iOS 15+, macOS 12+, or watchOS 9+
 - Microphone permissions
 - OpenAI API key
 
 ## Setup
 
-### 1. Add Microphone Permission
-
-Add the following to your `Info.plist`:
+Add microphone permission to your `Info.plist`:
 
 ```xml
 <key>NSMicrophoneUsageDescription</key>
 <string>We need access to your microphone for voice conversations with AI</string>
 ```
 
-### 2. macOS Sandbox Configuration
-
-If targeting macOS, enable the following in your target's Signing & Capabilities:
-
-- **App Sandbox**:
-  - Outgoing Connections (Client) ✓
-  - Audio Input ✓
-- **Hardened Runtime**:
-  - Audio Input ✓
-
-## Usage
+For macOS sandboxed apps, enable outgoing network connections and audio input.
 
-### Basic Example
+## Basic Usage
 
 ```swift
-import SwiftUI
-import OpenAI
+let service = OpenAIServiceFactory.service(apiKey: "your-api-key")
+
+let configuration = OpenAIRealtimeSessionConfiguration(
+    inputAudioFormat: .pcm16,
+    inputAudioTranscription: .init(model: Model.gptRealtimeWhisper.value, delay: .low, language: "en"),
+    instructions: "You are a concise realtime voice assistant.",
+    maxResponseOutputTokens: .int(4096),
+    modalities: [.audio],
+    outputAudioFormat: .pcm16,
+    parallelToolCalls: true,
+    reasoning: .init(effort: .low),
+    turnDetection: .init(type: .semanticVAD(eagerness: .auto, createResponse: true, interruptResponse: true)),
+    voice: "marin"
+)
+
+let session = try await service.realtimeSession(
+    model: Model.gptRealtime21.value,
+    configuration: configuration
+)
 
-struct ContentView: View {
-    let realtimeManager = RealtimeManager()
-    @State private var isActive = false
+let audioController = try await AudioController(modes: [.playback, .record])
+var isSessionReady = false
+var currentAudioItemID: String?
 
-    var body: some View {
-        Button(isActive ? "Stop" : "Start") {
-            isActive.toggle()
-            if isActive {
-                Task {
-                    try? await realtimeManager.startConversation()
-                }
-            } else {
-                Task {
-                    await realtimeManager.stopConversation()
-                }
-            }
+Task {
+    let micStream = try audioController.micStream()
+    for await buffer in micStream {
+        if isSessionReady,
+           let base64Audio = AudioUtils.base64EncodeAudioPCMBuffer(from: buffer) {
+            await session.sendMessage(OpenAIRealtimeInputAudioBufferAppend(audio: base64Audio))
         }
     }
 }
 
-@RealtimeActor
-final class RealtimeManager {
-    private var session: OpenAIRealtimeSession?
-    private var audioController: AudioController?
-
-    func startConversation() async throws {
-        // Initialize service
-        let service = OpenAIServiceFactory.service(apiKey: "your-api-key")
-
-        // Configure session
-        let config = OpenAIRealtimeSessionConfiguration(
-            inputAudioFormat: .pcm16,
-            inputAudioTranscription: .init(model: "whisper-1"),
-            instructions: "You are a helpful assistant",
-            modalities: [.audio, .text],
-            outputAudioFormat: .pcm16,
-            voice: "shimmer"
-        )
-
-        // Create session
-        session = try await service.realtimeSession(
-            model: "gpt-4o-mini-realtime-preview-2024-12-17",
-            configuration: config
-        )
-
-        // Setup audio
-        audioController = try await AudioController(modes: [.playback, .record])
-
-        // Handle microphone input
-        Task {
-            let micStream = try audioController!.micStream()
-            for await buffer in micStream {
-                if let base64Audio = AudioUtils.base64EncodeAudioPCMBuffer(from: buffer) {
-                    await session?.sendMessage(
-                        OpenAIRealtimeInputAudioBufferAppend(audio: base64Audio)
-                    )
-                }
-            }
-        }
-
-        // Handle AI responses
-        Task {
-            for await message in session!.receiver {
-                switch message {
-                case .responseAudioDelta(let audio):
-                    audioController?.playPCM16Audio(base64String: audio)
-                case .inputAudioBufferSpeechStarted:
-                    audioController?.interruptPlayback()
-                default:
-                    break
-                }
+Task {
+    for await message in session.receiver {
+        switch message {
+        case .sessionUpdated:
+            isSessionReady = true
+        case .responseAudioDelta(let itemID, _, let audio):
+            currentAudioItemID = itemID
+            audioController.playPCM16Audio(base64String: audio, itemID: itemID)
+        case .inputAudioBufferSpeechStarted:
+            if let currentAudioItemID,
+               let audioEndMS = audioController.interruptPlayback() {
+                await session.sendMessage(OpenAIRealtimeConversationItemTruncate(
+                    itemID: currentAudioItemID,
+                    audioEndMS: audioEndMS
+                ))
             }
+            currentAudioItemID = nil
+        default:
+            break
         }
     }
-
-    func stopConversation() {
-        audioController?.stop()
-        session?.disconnect()
-    }
-}
-```
-
-## Configuration Options
-
-### Voice Options
-
-- `alloy` - Neutral and balanced
-- `echo` - Friendly and warm
-- `shimmer` - Gentle and calming
-
-### Turn Detection
-
-#### Server VAD (Voice Activity Detection)
-
-```swift
-turnDetection: .init(type: .serverVAD(
-    prefixPaddingMs: 300,  // Audio to include before speech
-    silenceDurationMs: 500, // Silence duration to detect end
-    threshold: 0.5         // Activation threshold (0.0-1.0)
-))
-```
-
-#### Semantic VAD
-
-```swift
-turnDetection: .init(type: .semanticVAD(
-    eagerness: .medium  // .low, .medium, or .high
-))
-```
-
-### Modalities
-
-```swift
-modalities: [.audio, .text]  // Both audio and text
-modalities: [.text]          // Text only (disables audio)
-```
-
-## Handling Different Events
-
-```swift
-for await message in session.receiver {
-    switch message {
-    case .error(let error):
-        print("Error: \(error ?? "Unknown")")
-
-    case .sessionCreated:
-        print("Session started")
-
-    case .sessionUpdated:
-        // Trigger first response if AI speaks first
-        await session.sendMessage(OpenAIRealtimeResponseCreate())
-
-    case .responseAudioDelta(let base64Audio):
-        audioController.playPCM16Audio(base64String: base64Audio)
-
-    case .inputAudioBufferSpeechStarted:
-        // User started speaking, interrupt AI
-        audioController.interruptPlayback()
-
-    case .responseTranscriptDone(let transcript):
-        print("AI said: \(transcript)")
-
-    case .inputAudioTranscriptionCompleted(let transcript):
-        print("User said: \(transcript)")
-
-    case .responseFunctionCallArgumentsDone(let name, let args, let callId):
-        print("Function \(name) called with: \(args)")
-        // Handle function call and return result
-
-    default:
-        break
-    }
 }
 ```
 
 ## Function Calling
 
-Add tools to your configuration:
+Add a function tool to the session:
 
 ```swift
-let config = OpenAIRealtimeSessionConfiguration(
-    tools: [
-        .init(
-            name: "get_weather",
-            description: "Get the current weather for a location",
-            parameters: [
-                "type": "object",
-                "properties": [
-                    "location": [
-                        "type": "string",
-                        "description": "City name"
-                    ]
-                ],
-                "required": ["location"]
+let tool = OpenAIRealtimeSessionConfiguration.RealtimeTool.function(.init(
+    name: "get_demo_context",
+    description: "Get current context from this SwiftOpenAI Realtime example.",
+    parameters: [
+        "type": "object",
+        "properties": [
+            "topic": [
+                "type": "string",
+                "enum": ["time", "session"]
             ]
-        )
-    ],
-    toolChoice: .auto
-)
+        ],
+        "required": ["topic"],
+        "additionalProperties": false
+    ]
+))
 ```
 
-Handle function calls in the message loop:
+Handle the tool call and continue the response:
 
 ```swift
-case .responseFunctionCallArgumentsDone(let name, let args, let callId):
-    // Parse arguments and execute function
-    let result = handleFunction(name: name, args: args)
-
-    // Return result to OpenAI
-    await session.sendMessage(
-        OpenAIRealtimeConversationItemCreate(
-            item: .init(role: "function", text: result)
-        )
-    )
+var pendingToolResponseID: String?
+
+case .responseFunctionCallArgumentsDone(let name, let arguments, let callID, _, let responseID):
+    let output = handleFunctionCall(name: name, arguments: arguments)
+    await session.sendMessage(OpenAIRealtimeFunctionCallOutput(callID: callID, output: output))
+    pendingToolResponseID = responseID
+
+case .responseDone(let responseID, let status, _):
+    if pendingToolResponseID == responseID {
+        pendingToolResponseID = nil
+        if status == "completed" {
+            await session.sendMessage(OpenAIRealtimeResponseCreate())
+        }
+    }
 ```
 
 ## Troubleshooting
 
-### No Audio Output
-
-- Check that `.playback` mode is included in AudioController initialization
-- Verify audio permissions are granted
-- Ensure `outputAudioFormat` is set to `.pcm16`
-
-### No Microphone Input
-
-- Check that `.record` mode is included in AudioController initialization
-- Verify microphone permissions in Info.plist
-- Check System Settings > Privacy & Security > Microphone
-
-### WebSocket Connection Fails
-
-- Verify API key is correct
-- Check that `openai-beta: realtime=v1` header is included (SwiftOpenAI handles this automatically)
-- Ensure you're using a compatible model (gpt-4o-mini-realtime-preview or newer)
+- Verify the API key has access to Realtime models.
+- Use `Model.gptRealtime21.value` for the latest voice-agent model.
+- Ensure microphone permissions are granted.
+- Keep capture and playback on the same `AudioController`; separate audio graphs prevent echo cancellation from using model playback as its reference.
+- On macOS, enable outgoing connections and audio input in sandbox settings.
 
 ## Resources
 
-- [OpenAI Realtime API Documentation](https://platform.openai.com/docs/api-reference/realtime)
-- [SwiftOpenAI GitHub](https://github.com/jamesrochabrun/SwiftOpenAI)
+- [OpenAI Realtime API documentation](https://developers.openai.com/api/docs/guides/realtime)
+- [Realtime with tools](https://developers.openai.com/api/docs/guides/realtime-mcp)