From 67f009b5f4039fe1bf0dcc7eef4a089649fe605a Mon Sep 17 00:00:00 2001 From: jamesrochabrun Date: Fri, 10 Jul 2026 11:08:54 -0700 Subject: [PATCH 1/2] Modernize Realtime conversation handling --- Examples/RealtimeExample/README.md | 310 ++++------- .../RealtimeExample/RealtimeExample.swift | 131 ++++- .../project.pbxproj | 40 ++ .../SwiftOpenAIExample/AIProxyIntroView.swift | 2 +- .../SwiftOpenAIExample/OptionsListView.swift | 2 +- .../SwiftOpenAIExample/RealtimeDemo/README.md | 19 + .../RealtimeDemo/RealtimeConversation.swift | 204 ++++++++ .../RealtimeConversationControls.swift | 58 +++ .../RealtimeConversationDemoView.swift | 56 ++ .../RealtimeConversationProvider.swift | 483 ++++++++++++++++++ .../RealtimeConversationStatusView.swift | 63 +++ .../RealtimeDemo/RealtimeMicrophoneGate.swift | 22 + .../RealtimeDemo/RealtimeTranscriptRow.swift | 77 +++ .../RealtimeTurnCoordinator.swift | 88 ++++ .../SwiftOpenAIExampleTests.swift | 112 ++++ README.md | 159 +++--- Sources/OpenAI/AIProxy/AIProxyService.swift | 7 + .../Azure/DefaultOpenAIAzureService.swift | 7 + .../LocalModelService/LocalModelService.swift | 5 + .../OpenAI/Private/Audio/AudioPCMPlayer.swift | 95 +++- .../Audio/MicrophonePCMSampleVendorAE.swift | 24 +- .../Audio/MicrophonePCMSampleVendorAT.swift | 10 +- .../MicrophonePCMSampleVendorCommon.swift | 73 ++- .../OpenAI/Private/Networking/OpenAIAPI.swift | 10 + .../Realtime/OpenAIRealtimeSession.swift | 204 ++++++-- Sources/OpenAI/Public/Parameters/Model.swift | 25 + ...OpenAIRealtimeClientSecretParameters.swift | 48 ++ ...OpenAIRealtimeConversationItemCreate.swift | 7 +- ...enAIRealtimeConversationItemTruncate.swift | 26 + .../OpenAIRealtimeFunctionCallOutput.swift | 44 ++ .../OpenAIRealtimeMCPApprovalResponse.swift | 59 +++ .../OpenAIRealtimeResponseCreate.swift | 59 ++- .../OpenAIRealtimeSessionConfiguration.swift | 219 +++++++- .../Realtime/OpenAIRealtimeClientSecret.swift | 25 + .../Realtime/OpenAIRealtimeMessage.swift | 55 +- .../Public/Service/DefaultOpenAIService.swift | 19 +- .../OpenAI/Public/Service/OpenAIService.swift | 9 +- .../Public/Shared/AudioController.swift | 102 ++-- .../Public/Shared/OpenAIJSONValue.swift | 36 ++ Sources/OpenAI/Public/Shared/Tool.swift | 38 +- .../RealtimeAudioConversionTests.swift | 34 ++ .../OpenAITests/RealtimeParameterTests.swift | 129 +++++ 42 files changed, 2706 insertions(+), 489 deletions(-) create mode 100644 Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/README.md create mode 100644 Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversation.swift create mode 100644 Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversationControls.swift create mode 100644 Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversationDemoView.swift create mode 100644 Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversationProvider.swift create mode 100644 Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversationStatusView.swift create mode 100644 Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeMicrophoneGate.swift create mode 100644 Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeTranscriptRow.swift create mode 100644 Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeTurnCoordinator.swift create mode 100644 Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeClientSecretParameters.swift create mode 100644 Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeConversationItemTruncate.swift create mode 100644 Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeFunctionCallOutput.swift create mode 100644 Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeMCPApprovalResponse.swift create mode 100644 Sources/OpenAI/Public/ResponseModels/Realtime/OpenAIRealtimeClientSecret.swift create mode 100644 Tests/OpenAITests/RealtimeAudioConversionTests.swift create mode 100644 Tests/OpenAITests/RealtimeParameterTests.swift diff --git a/Examples/RealtimeExample/README.md b/Examples/RealtimeExample/README.md index b5a77417..5c2c0473 100644 --- a/Examples/RealtimeExample/README.md +++ b/Examples/RealtimeExample/README.md @@ -1,267 +1,143 @@ # OpenAI Realtime API Example -This example demonstrates how to use SwiftOpenAI's Realtime API for bidirectional voice conversations with OpenAI's GPT-4o models. +This example demonstrates SwiftOpenAI's Realtime API support for bidirectional voice conversations with `gpt-realtime-2.1` and local function tool calling. ## Features -- Real-time bidirectional audio streaming -- Voice Activity Detection (VAD) for automatic turn-taking -- Audio transcription of both user and AI speech -- Function calling support -- Interrupt handling when user starts speaking +- Bidirectional audio streaming over WebSocket +- Semantic VAD for automatic turn-taking +- User and assistant transcripts +- Local function tools with `function_call_output` +- Playback interruption when the user starts speaking ## Requirements -- iOS 15+, macOS 12+, watchOS 9+ +- iOS 15+, macOS 12+, or watchOS 9+ - Microphone permissions - OpenAI API key ## Setup -### 1. Add Microphone Permission - -Add the following to your `Info.plist`: +Add microphone permission to your `Info.plist`: ```xml NSMicrophoneUsageDescription We need access to your microphone for voice conversations with AI ``` -### 2. macOS Sandbox Configuration - -If targeting macOS, enable the following in your target's Signing & Capabilities: - -- **App Sandbox**: - - Outgoing Connections (Client) ✓ - - Audio Input ✓ -- **Hardened Runtime**: - - Audio Input ✓ - -## Usage +For macOS sandboxed apps, enable outgoing network connections and audio input. -### Basic Example +## Basic Usage ```swift -import SwiftUI -import OpenAI +let service = OpenAIServiceFactory.service(apiKey: "your-api-key") + +let configuration = OpenAIRealtimeSessionConfiguration( + inputAudioFormat: .pcm16, + inputAudioTranscription: .init(model: Model.gptRealtimeWhisper.value, delay: .low, language: "en"), + instructions: "You are a concise realtime voice assistant.", + maxResponseOutputTokens: .int(4096), + modalities: [.audio], + outputAudioFormat: .pcm16, + parallelToolCalls: true, + reasoning: .init(effort: .low), + turnDetection: .init(type: .semanticVAD(eagerness: .auto, createResponse: true, interruptResponse: true)), + voice: "marin" +) + +let session = try await service.realtimeSession( + model: Model.gptRealtime21.value, + configuration: configuration +) -struct ContentView: View { - let realtimeManager = RealtimeManager() - @State private var isActive = false +let audioController = try await AudioController(modes: [.playback, .record]) +var isSessionReady = false +var currentAudioItemID: String? - var body: some View { - Button(isActive ? "Stop" : "Start") { - isActive.toggle() - if isActive { - Task { - try? await realtimeManager.startConversation() - } - } else { - Task { - await realtimeManager.stopConversation() - } - } +Task { + let micStream = try audioController.micStream() + for await buffer in micStream { + if isSessionReady, + let base64Audio = AudioUtils.base64EncodeAudioPCMBuffer(from: buffer) { + await session.sendMessage(OpenAIRealtimeInputAudioBufferAppend(audio: base64Audio)) } } } -@RealtimeActor -final class RealtimeManager { - private var session: OpenAIRealtimeSession? - private var audioController: AudioController? - - func startConversation() async throws { - // Initialize service - let service = OpenAIServiceFactory.service(apiKey: "your-api-key") - - // Configure session - let config = OpenAIRealtimeSessionConfiguration( - inputAudioFormat: .pcm16, - inputAudioTranscription: .init(model: "whisper-1"), - instructions: "You are a helpful assistant", - modalities: [.audio, .text], - outputAudioFormat: .pcm16, - voice: "shimmer" - ) - - // Create session - session = try await service.realtimeSession( - model: "gpt-4o-mini-realtime-preview-2024-12-17", - configuration: config - ) - - // Setup audio - audioController = try await AudioController(modes: [.playback, .record]) - - // Handle microphone input - Task { - let micStream = try audioController!.micStream() - for await buffer in micStream { - if let base64Audio = AudioUtils.base64EncodeAudioPCMBuffer(from: buffer) { - await session?.sendMessage( - OpenAIRealtimeInputAudioBufferAppend(audio: base64Audio) - ) - } - } - } - - // Handle AI responses - Task { - for await message in session!.receiver { - switch message { - case .responseAudioDelta(let audio): - audioController?.playPCM16Audio(base64String: audio) - case .inputAudioBufferSpeechStarted: - audioController?.interruptPlayback() - default: - break - } +Task { + for await message in session.receiver { + switch message { + case .sessionUpdated: + isSessionReady = true + case .responseAudioDelta(let itemID, _, let audio): + currentAudioItemID = itemID + audioController.playPCM16Audio(base64String: audio, itemID: itemID) + case .inputAudioBufferSpeechStarted: + if let currentAudioItemID, + let audioEndMS = audioController.interruptPlayback() { + await session.sendMessage(OpenAIRealtimeConversationItemTruncate( + itemID: currentAudioItemID, + audioEndMS: audioEndMS + )) } + currentAudioItemID = nil + default: + break } } - - func stopConversation() { - audioController?.stop() - session?.disconnect() - } -} -``` - -## Configuration Options - -### Voice Options - -- `alloy` - Neutral and balanced -- `echo` - Friendly and warm -- `shimmer` - Gentle and calming - -### Turn Detection - -#### Server VAD (Voice Activity Detection) - -```swift -turnDetection: .init(type: .serverVAD( - prefixPaddingMs: 300, // Audio to include before speech - silenceDurationMs: 500, // Silence duration to detect end - threshold: 0.5 // Activation threshold (0.0-1.0) -)) -``` - -#### Semantic VAD - -```swift -turnDetection: .init(type: .semanticVAD( - eagerness: .medium // .low, .medium, or .high -)) -``` - -### Modalities - -```swift -modalities: [.audio, .text] // Both audio and text -modalities: [.text] // Text only (disables audio) -``` - -## Handling Different Events - -```swift -for await message in session.receiver { - switch message { - case .error(let error): - print("Error: \(error ?? "Unknown")") - - case .sessionCreated: - print("Session started") - - case .sessionUpdated: - // Trigger first response if AI speaks first - await session.sendMessage(OpenAIRealtimeResponseCreate()) - - case .responseAudioDelta(let base64Audio): - audioController.playPCM16Audio(base64String: base64Audio) - - case .inputAudioBufferSpeechStarted: - // User started speaking, interrupt AI - audioController.interruptPlayback() - - case .responseTranscriptDone(let transcript): - print("AI said: \(transcript)") - - case .inputAudioTranscriptionCompleted(let transcript): - print("User said: \(transcript)") - - case .responseFunctionCallArgumentsDone(let name, let args, let callId): - print("Function \(name) called with: \(args)") - // Handle function call and return result - - default: - break - } } ``` ## Function Calling -Add tools to your configuration: +Add a function tool to the session: ```swift -let config = OpenAIRealtimeSessionConfiguration( - tools: [ - .init( - name: "get_weather", - description: "Get the current weather for a location", - parameters: [ - "type": "object", - "properties": [ - "location": [ - "type": "string", - "description": "City name" - ] - ], - "required": ["location"] +let tool = OpenAIRealtimeSessionConfiguration.RealtimeTool.function(.init( + name: "get_demo_context", + description: "Get current context from this SwiftOpenAI Realtime example.", + parameters: [ + "type": "object", + "properties": [ + "topic": [ + "type": "string", + "enum": ["time", "session"] ] - ) - ], - toolChoice: .auto -) + ], + "required": ["topic"], + "additionalProperties": false + ] +)) ``` -Handle function calls in the message loop: +Handle the tool call and continue the response: ```swift -case .responseFunctionCallArgumentsDone(let name, let args, let callId): - // Parse arguments and execute function - let result = handleFunction(name: name, args: args) - - // Return result to OpenAI - await session.sendMessage( - OpenAIRealtimeConversationItemCreate( - item: .init(role: "function", text: result) - ) - ) +var pendingToolResponseID: String? + +case .responseFunctionCallArgumentsDone(let name, let arguments, let callID, _, let responseID): + let output = handleFunctionCall(name: name, arguments: arguments) + await session.sendMessage(OpenAIRealtimeFunctionCallOutput(callID: callID, output: output)) + pendingToolResponseID = responseID + +case .responseDone(let responseID, let status, _): + if pendingToolResponseID == responseID { + pendingToolResponseID = nil + if status == "completed" { + await session.sendMessage(OpenAIRealtimeResponseCreate()) + } + } ``` ## Troubleshooting -### No Audio Output - -- Check that `.playback` mode is included in AudioController initialization -- Verify audio permissions are granted -- Ensure `outputAudioFormat` is set to `.pcm16` - -### No Microphone Input - -- Check that `.record` mode is included in AudioController initialization -- Verify microphone permissions in Info.plist -- Check System Settings > Privacy & Security > Microphone - -### WebSocket Connection Fails - -- Verify API key is correct -- Check that `openai-beta: realtime=v1` header is included (SwiftOpenAI handles this automatically) -- Ensure you're using a compatible model (gpt-4o-mini-realtime-preview or newer) +- Verify the API key has access to Realtime models. +- Use `Model.gptRealtime21.value` for the latest voice-agent model. +- Ensure microphone permissions are granted. +- Keep capture and playback on the same `AudioController`; separate audio graphs prevent echo cancellation from using model playback as its reference. +- On macOS, enable outgoing connections and audio input in sandbox settings. ## Resources -- [OpenAI Realtime API Documentation](https://platform.openai.com/docs/api-reference/realtime) -- [SwiftOpenAI GitHub](https://github.com/jamesrochabrun/SwiftOpenAI) +- [OpenAI Realtime API documentation](https://developers.openai.com/api/docs/guides/realtime) +- [Realtime with tools](https://developers.openai.com/api/docs/guides/realtime-mcp) diff --git a/Examples/RealtimeExample/RealtimeExample.swift b/Examples/RealtimeExample/RealtimeExample.swift index 53bfc1d0..5004623d 100644 --- a/Examples/RealtimeExample/RealtimeExample.swift +++ b/Examples/RealtimeExample/RealtimeExample.swift @@ -2,11 +2,12 @@ // RealtimeExample.swift // SwiftOpenAI // -// Example implementation of OpenAI Realtime API for bidirectional voice conversation +// Example implementation of OpenAI Realtime API for bidirectional voice conversation and tool calling // import AVFoundation -import OpenAI +import Foundation +import SwiftOpenAI import SwiftUI // MARK: - RealtimeExampleView @@ -63,31 +64,56 @@ final class RealtimeManager { // Set to false if you want your user to speak first let aiSpeaksFirst = true - let audioController = try await AudioController(modes: [.playback, .record]) - let micStream = try audioController.micStream() - // Configure the realtime session let configuration = OpenAIRealtimeSessionConfiguration( inputAudioFormat: .pcm16, - inputAudioTranscription: .init(model: "whisper-1"), + inputAudioTranscription: .init(model: Model.gptRealtimeWhisper.value, delay: .low, language: "en"), instructions: "You are a helpful, witty, and friendly AI assistant. " + "Your voice and personality should be warm and engaging, " + - "with a lively and playful tone. Talk quickly.", + "with a lively and playful tone. Talk quickly. " + + "When asked about the current demo context, call get_demo_context before answering.", maxResponseOutputTokens: .int(4096), - modalities: [.audio, .text], + modalities: [.audio], outputAudioFormat: .pcm16, - temperature: 0.7, + parallelToolCalls: true, + reasoning: .init(effort: .low), + tools: [ + .function(.init( + name: "get_demo_context", + description: "Get current context from this SwiftOpenAI Realtime example.", + parameters: [ + "type": "object", + "properties": [ + "topic": [ + "type": "string", + "description": "The context to retrieve.", + "enum": ["time", "session"], + ], + ], + "required": ["topic"], + "additionalProperties": false, + ])), + ], + toolChoice: .auto, turnDetection: .init( - type: .semanticVAD(eagerness: .medium)), - voice: "shimmer") + type: .semanticVAD(eagerness: .auto, createResponse: true, interruptResponse: true)), + voice: "marin") // Create the realtime session let realtimeSession = try await service.realtimeSession( - model: "gpt-4o-mini-realtime-preview-2024-12-17", + model: Model.gptRealtime21.value, configuration: configuration) + // Install the input tap only after the buffered session exists. `micStream()` starts the + // fully configured capture/playback graph before either event task begins consuming data. + let audioController = try await AudioController(modes: [.playback, .record]) + let micStream = try audioController.micStream() + // Send audio from the microphone to OpenAI once OpenAI is ready for it var isOpenAIReadyForAudio = false + var currentResponseID: String? + var pendingToolResponseID: String? + var currentAudioItemID: String? Task { for await buffer in micStream { if @@ -109,31 +135,62 @@ final class RealtimeManager { realtimeSession.disconnect() case .sessionUpdated: + isOpenAIReadyForAudio = true if aiSpeaksFirst { await realtimeSession.sendMessage(OpenAIRealtimeResponseCreate()) - } else { - isOpenAIReadyForAudio = true } - case .responseAudioDelta(let base64String): - audioController.playPCM16Audio(base64String: base64String) + case .responseAudioDelta(let itemID, _, let base64String): + currentAudioItemID = itemID + audioController.playPCM16Audio(base64String: base64String, itemID: itemID) case .inputAudioBufferSpeechStarted: - // User started speaking, interrupt AI playback - audioController.interruptPlayback() + if + let assistantItemID = currentAudioItemID, + let audioEndMS = audioController.interruptPlayback() + { + await realtimeSession.sendMessage(OpenAIRealtimeConversationItemTruncate( + itemID: assistantItemID, + audioEndMS: audioEndMS)) + } + currentAudioItemID = nil - case .responseCreated: - isOpenAIReadyForAudio = true + case .responseCreated(let responseID): + currentResponseID = responseID - case .responseTranscriptDone(let transcript): + case .responseDone(let responseID, let status, _): + if Self.responseIDsMatch(currentResponseID, responseID) { + currentResponseID = nil + } + if + pendingToolResponseID != nil, + Self.responseIDsMatch(pendingToolResponseID, responseID) + { + pendingToolResponseID = nil + if status == "completed" { + await realtimeSession.sendMessage(OpenAIRealtimeResponseCreate()) + } + } + + case .responseTranscriptDone(_, _, let transcript): print("AI said: \(transcript)") - case .inputAudioTranscriptionCompleted(let transcript): + case .inputAudioTranscriptionCompleted(_, let transcript): print("User said: \(transcript)") - case .responseFunctionCallArgumentsDone(let name, let arguments, let callId): + case .responseFunctionCallArgumentsDone( + let name, + let arguments, + let callId, + _, + let responseID): print("Function call: \(name) with args: \(arguments)") - // Handle function calls here + let output = Self.handleFunctionCall(name: name, arguments: arguments) + await realtimeSession.sendMessage( + OpenAIRealtimeFunctionCallOutput(callID: callId, output: output)) + if currentResponseID != nil { + pendingToolResponseID = responseID ?? currentResponseID + } default: break @@ -155,6 +212,24 @@ final class RealtimeManager { private var realtimeSession: OpenAIRealtimeSession? private var audioController: AudioController? + private static func responseIDsMatch(_ lhs: String?, _ rhs: String?) -> Bool { + guard let lhs, let rhs else { return true } + return lhs == rhs + } + + private static func handleFunctionCall(name: String, arguments _: String) -> String { + guard name == "get_demo_context" else { + return #"{"error":"Unknown function"}"# + } + + let payload: [String: String] = [ + "model": Model.gptRealtime21.value, + "current_time": ISO8601DateFormatter().string(from: Date()), + "session_state": "connected", + ] + let data = try? JSONSerialization.data(withJSONObject: payload, options: [.sortedKeys]) + return data.flatMap { String(data: $0, encoding: .utf8) } ?? #"{"error":"Encoding failed"}"# + } } // MARK: - Basic Usage Example @@ -177,14 +252,14 @@ final class RealtimeManager { // let config = OpenAIRealtimeSessionConfiguration( // inputAudioFormat: .pcm16, // instructions: "You are a helpful assistant", -// modalities: [.audio, .text], +// modalities: [.audio], // outputAudioFormat: .pcm16, -// voice: "shimmer" +// voice: "marin" // ) // // 5. Create the realtime session: // let session = try await service.realtimeSession( -// model: "gpt-4o-mini-realtime-preview-2024-12-17", +// model: Model.gptRealtime21.value, // configuration: config // ) // @@ -203,7 +278,7 @@ final class RealtimeManager { // 8. Listen for and play responses: // for await message in session.receiver { // switch message { -// case .responseAudioDelta(let base64Audio): +// case .responseAudioDelta(_, _, let base64Audio): // audioController.playPCM16Audio(base64String: base64Audio) // default: // break diff --git a/Examples/SwiftOpenAIExample/SwiftOpenAIExample.xcodeproj/project.pbxproj b/Examples/SwiftOpenAIExample/SwiftOpenAIExample.xcodeproj/project.pbxproj index 2e342564..a068b277 100644 --- a/Examples/SwiftOpenAIExample/SwiftOpenAIExample.xcodeproj/project.pbxproj +++ b/Examples/SwiftOpenAIExample/SwiftOpenAIExample.xcodeproj/project.pbxproj @@ -18,6 +18,14 @@ 7B1268072B08247C00400694 /* AssistantConfigurationProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B1268062B08247C00400694 /* AssistantConfigurationProvider.swift */; }; 7B2B6D562DF434670059B4BB /* ResponseStreamDemoView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B2B6D552DF434670059B4BB /* ResponseStreamDemoView.swift */; }; 7B2B6D582DF4347E0059B4BB /* ResponseStreamProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B2B6D572DF4347E0059B4BB /* ResponseStreamProvider.swift */; }; + 7B6A10012E00000100000001 /* RealtimeConversationDemoView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B6A10032E00000100000003 /* RealtimeConversationDemoView.swift */; }; + 7B6A10022E00000100000002 /* RealtimeConversationProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B6A10042E00000100000004 /* RealtimeConversationProvider.swift */; }; + 7B6A10062E00000100000006 /* RealtimeConversation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B6A100A2E0000010000000A /* RealtimeConversation.swift */; }; + 7B6A10072E00000100000007 /* RealtimeConversationStatusView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B6A100B2E0000010000000B /* RealtimeConversationStatusView.swift */; }; + 7B6A10082E00000100000008 /* RealtimeConversationControls.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B6A100C2E0000010000000C /* RealtimeConversationControls.swift */; }; + 7B6A10092E00000100000009 /* RealtimeTranscriptRow.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B6A100D2E0000010000000D /* RealtimeTranscriptRow.swift */; }; + 7B6A100E2E0000010000000E /* RealtimeTurnCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B6A100F2E0000010000000F /* RealtimeTurnCoordinator.swift */; }; + 7B6A10102E00000100000010 /* RealtimeMicrophoneGate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B6A10112E00000100000011 /* RealtimeMicrophoneGate.swift */; }; 7B3DDCC52BAAA722004B5C96 /* AssistantsListDemoView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B3DDCC42BAAA722004B5C96 /* AssistantsListDemoView.swift */; }; 7B3DDCC72BAAAD34004B5C96 /* AssistantThreadConfigurationProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B3DDCC62BAAAD34004B5C96 /* AssistantThreadConfigurationProvider.swift */; }; 7B3DDCC92BAAAF96004B5C96 /* AssistantStreamDemoScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B3DDCC82BAAAF96004B5C96 /* AssistantStreamDemoScreen.swift */; }; @@ -103,6 +111,14 @@ 7B1268062B08247C00400694 /* AssistantConfigurationProvider.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AssistantConfigurationProvider.swift; sourceTree = ""; }; 7B2B6D552DF434670059B4BB /* ResponseStreamDemoView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ResponseStreamDemoView.swift; sourceTree = ""; }; 7B2B6D572DF4347E0059B4BB /* ResponseStreamProvider.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ResponseStreamProvider.swift; sourceTree = ""; }; + 7B6A10032E00000100000003 /* RealtimeConversationDemoView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RealtimeConversationDemoView.swift; sourceTree = ""; }; + 7B6A10042E00000100000004 /* RealtimeConversationProvider.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RealtimeConversationProvider.swift; sourceTree = ""; }; + 7B6A100A2E0000010000000A /* RealtimeConversation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RealtimeConversation.swift; sourceTree = ""; }; + 7B6A100B2E0000010000000B /* RealtimeConversationStatusView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RealtimeConversationStatusView.swift; sourceTree = ""; }; + 7B6A100C2E0000010000000C /* RealtimeConversationControls.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RealtimeConversationControls.swift; sourceTree = ""; }; + 7B6A100D2E0000010000000D /* RealtimeTranscriptRow.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RealtimeTranscriptRow.swift; sourceTree = ""; }; + 7B6A100F2E0000010000000F /* RealtimeTurnCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RealtimeTurnCoordinator.swift; sourceTree = ""; }; + 7B6A10112E00000100000011 /* RealtimeMicrophoneGate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RealtimeMicrophoneGate.swift; sourceTree = ""; }; 7B3DDCC42BAAA722004B5C96 /* AssistantsListDemoView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AssistantsListDemoView.swift; sourceTree = ""; }; 7B3DDCC62BAAAD34004B5C96 /* AssistantThreadConfigurationProvider.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AssistantThreadConfigurationProvider.swift; sourceTree = ""; }; 7B3DDCC82BAAAF96004B5C96 /* AssistantStreamDemoScreen.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AssistantStreamDemoScreen.swift; sourceTree = ""; }; @@ -318,6 +334,21 @@ path = LocalChatDemo; sourceTree = ""; }; + 7B6A10052E00000100000005 /* RealtimeDemo */ = { + isa = PBXGroup; + children = ( + 7B6A100A2E0000010000000A /* RealtimeConversation.swift */, + 7B6A10032E00000100000003 /* RealtimeConversationDemoView.swift */, + 7B6A10042E00000100000004 /* RealtimeConversationProvider.swift */, + 7B6A100B2E0000010000000B /* RealtimeConversationStatusView.swift */, + 7B6A100C2E0000010000000C /* RealtimeConversationControls.swift */, + 7B6A100D2E0000010000000D /* RealtimeTranscriptRow.swift */, + 7B6A100F2E0000010000000F /* RealtimeTurnCoordinator.swift */, + 7B6A10112E00000100000011 /* RealtimeMicrophoneGate.swift */, + ); + path = RealtimeDemo; + sourceTree = ""; + }; 7B72399E2AF625B700646679 /* ChatStreamFluidConversationDemo */ = { isa = PBXGroup; children = ( @@ -393,6 +424,7 @@ isa = PBXGroup; children = ( 7B2B6D542DF434550059B4BB /* ResponseAPIDemo */, + 7B6A10052E00000100000005 /* RealtimeDemo */, 7BA788CC2AE23A48008825D5 /* SwiftOpenAIExampleApp.swift */, 7BE802572D2877D30080E06A /* PredictedOutputsDemo */, 7B50DD292C2A9D1D0070A64D /* LocalChatDemo */, @@ -663,6 +695,13 @@ 7BBE7EDE2B03718E0096A693 /* ChatFunctionCallProvider.swift in Sources */, 7B7239A62AF628F800646679 /* ChatDisplayMessageView.swift in Sources */, 7B99C2ED2C071B1600E701B3 /* FilesPickerProvider.swift in Sources */, + 7B6A10012E00000100000001 /* RealtimeConversationDemoView.swift in Sources */, + 7B6A10062E00000100000006 /* RealtimeConversation.swift in Sources */, + 7B6A10072E00000100000007 /* RealtimeConversationStatusView.swift in Sources */, + 7B6A10082E00000100000008 /* RealtimeConversationControls.swift in Sources */, + 7B6A10092E00000100000009 /* RealtimeTranscriptRow.swift in Sources */, + 7B6A100E2E0000010000000E /* RealtimeTurnCoordinator.swift in Sources */, + 7B6A10102E00000100000010 /* RealtimeMicrophoneGate.swift in Sources */, 7B7239A02AF625F200646679 /* ChatFluidConversationProvider.swift in Sources */, 7BA788CF2AE23A48008825D5 /* ApiKeyIntroView.swift in Sources */, 7BA788CD2AE23A48008825D5 /* SwiftOpenAIExampleApp.swift in Sources */, @@ -694,6 +733,7 @@ 7B436B992AE25052003CE281 /* ContentLoader.swift in Sources */, 7B436BC12AE7B01F003CE281 /* ModerationProvider.swift in Sources */, 7B436BBC2AE7ABD3003CE281 /* ModelsProvider.swift in Sources */, + 7B6A10022E00000100000002 /* RealtimeConversationProvider.swift in Sources */, 7B2B6D562DF434670059B4BB /* ResponseStreamDemoView.swift in Sources */, 7B2B6D582DF4347E0059B4BB /* ResponseStreamProvider.swift in Sources */, 7B436BA62AE77F37003CE281 /* Embeddingsprovider.swift in Sources */, diff --git a/Examples/SwiftOpenAIExample/SwiftOpenAIExample/AIProxyIntroView.swift b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/AIProxyIntroView.swift index 34f27dc0..5b19b568 100644 --- a/Examples/SwiftOpenAIExample/SwiftOpenAIExample/AIProxyIntroView.swift +++ b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/AIProxyIntroView.swift @@ -25,7 +25,7 @@ struct AIProxyIntroView: View { NavigationLink(destination: OptionsListView( openAIService: aiproxyService, - options: OptionsListView.APIOption.allCases.filter { $0 != .localChat })) + options: OptionsListView.APIOption.allCases.filter { $0 != .localChat && $0 != .realTimeAPI })) { Text("Continue") .padding() diff --git a/Examples/SwiftOpenAIExample/SwiftOpenAIExample/OptionsListView.swift b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/OptionsListView.swift index 6874015b..0d2fc49d 100644 --- a/Examples/SwiftOpenAIExample/SwiftOpenAIExample/OptionsListView.swift +++ b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/OptionsListView.swift @@ -96,7 +96,7 @@ struct OptionsListView: View { case .configureAssistant: AssistantConfigurationDemoView(service: openAIService) case .realTimeAPI: - Text("WIP") + RealtimeConversationDemoView(service: openAIService) case .responseStream: ResponseStreamDemoView(service: openAIService) } diff --git a/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/README.md b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/README.md new file mode 100644 index 00000000..d2892669 --- /dev/null +++ b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/README.md @@ -0,0 +1,19 @@ +# Realtime demo architecture + +The demo separates conversation state from transport and UI so its core can move into another app without bringing the demo screen with it. + +- `RealtimeConversation` is a deterministic reducer. It merges transcript deltas by server `item_id`, applies final transcripts, and maintains server conversation order. +- `RealtimeConversationProvider` owns the session, audio controller, event loop, tool execution, and the small connection state machine. +- The SwiftUI files only render observable state and forward user actions. + +## Event rules + +1. Request microphone permission explicitly. Create the Realtime session first, then configure the audio controller and install its microphone tap before starting the audio engine. The SDK buffers session events during that setup so `session.created` and `session.updated` are not lost. +2. Consume microphone buffers on `RealtimeActor`, not the main actor. Open a separate microphone gate after `session.updated` and keep it open while the model speaks. `AudioController` uses one `AVAudioEngine` voice-processing graph for record and playback, giving echo cancellation the model-audio reference it needs. +3. Use `item_id` as UI identity. Append delta events to the matching item and replace them with the terminal transcript from the corresponding `done` event. +4. Enable `interrupt_response`. On `input_audio_buffer.speech_started`, stop local playback immediately and send `conversation.item.truncate` with the heard playback duration. +5. Send function output as a conversation item, then bind the pending continuation to the tool call's `response_id`. Only a matching `response.done` with `completed` status may send `response.create`; cancellation yields to the user's VAD-created turn. +6. Treat function-argument deltas, item completion, audio completion, and rate-limit updates as normal lifecycle events. Treat an ended receiver stream as a disconnect and tear down microphone, playback, and session state together. +7. Do not report the session as listening until the microphone produces its first buffer. A startup watchdog should turn a silent audio graph into a visible error instead of leaving the UI stuck in a false listening state. + +When migrating, keep the reducer and event rules intact. Replace the provider's tool implementation and the SwiftUI views with application-specific versions. diff --git a/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversation.swift b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversation.swift new file mode 100644 index 00000000..74d55c38 --- /dev/null +++ b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversation.swift @@ -0,0 +1,204 @@ +// +// RealtimeConversation.swift +// SwiftOpenAIExample +// + +import Foundation + +/// A transport-independent projection of Realtime events into conversation rows. +/// Other apps can reuse this reducer while replacing the surrounding UI and tool layer. +struct RealtimeConversation { + struct Entry: Identifiable, Equatable { + enum Role: Equatable { + case assistant + case tool + case user + } + + enum State: Equatable { + case streaming + case complete + } + + let id: String + let role: Role + var text: String + var state: State + } + + private(set) var entries = [Entry]() + + mutating func reset() { + entries.removeAll(keepingCapacity: true) + currentAssistantID = nil + currentUserID = nil + fallbackID = 0 + } + + mutating func beginUserTurn(itemID: String?) { + let id = resolvedID(itemID, role: .user, responseID: nil) + currentUserID = id + upsert(id: id, role: .user, state: .streaming) + } + + mutating func appendUserTranscript(_ delta: String, itemID: String?) { + let id = resolvedID(itemID, role: .user, responseID: nil) + currentUserID = id + append(delta, to: id, role: .user) + } + + mutating func finishUserTranscript(_ transcript: String, itemID: String?) { + let id = resolvedID(itemID, role: .user, responseID: nil) + finish(text: transcript, id: id, role: .user) + if currentUserID == id { + currentUserID = nil + } + } + + mutating func appendUserText(_ text: String, itemID: String) { + finish(text: text, id: itemID, role: .user) + } + + mutating func beginAssistantTurn(itemID: String?, responseID: String?) { + let id = resolvedID(itemID, role: .assistant, responseID: responseID) + currentAssistantID = id + upsert(id: id, role: .assistant, state: .streaming) + } + + mutating func appendAssistantTranscript(_ delta: String, itemID: String?, responseID: String?) { + let id = resolvedID(itemID, role: .assistant, responseID: responseID) + currentAssistantID = id + append(delta, to: id, role: .assistant) + } + + mutating func finishAssistantTranscript(_ transcript: String, itemID: String?, responseID: String?) { + let id = resolvedID(itemID, role: .assistant, responseID: responseID) + finish(text: transcript, id: id, role: .assistant) + if currentAssistantID == id { + currentAssistantID = nil + } + } + + mutating func finishAssistantItem(itemID: String, text: String?) { + guard let text, !text.isEmpty else { return } + finish(text: text, id: itemID, role: .assistant) + if currentAssistantID == itemID { + currentAssistantID = nil + } + } + + mutating func finishResponse() { + guard let currentAssistantID else { return } + + if let index = entries.firstIndex(where: { $0.id == currentAssistantID }) { + if entries[index].text.isEmpty { + entries.remove(at: index) + } else { + entries[index].state = .complete + } + } + + self.currentAssistantID = nil + } + + mutating func recordToolCall(name: String, callID: String) { + finish(text: "Used \(name)", id: callID, role: .tool) + } + + mutating func registerItem( + itemID: String, + type: String, + role: String?, + previousItemID: String?) + { + guard type == "message" else { return } + + switch role { + case "assistant": + beginAssistantTurn(itemID: itemID, responseID: nil) + case "user": + beginUserTurn(itemID: itemID) + default: + return + } + + move(itemID: itemID, after: previousItemID) + } + + private var currentAssistantID: String? + private var currentUserID: String? + private var fallbackID = 0 + + private mutating func resolvedID( + _ itemID: String?, + role: Entry.Role, + responseID: String?) + -> String + { + if let itemID { + return itemID + } + if role == .assistant, let responseID { + return "response-\(responseID)" + } + if role == .assistant, let currentAssistantID { + return currentAssistantID + } + if role == .user, let currentUserID { + return currentUserID + } + + fallbackID += 1 + return "local-\(role)-\(fallbackID)" + } + + private mutating func upsert(id: String, role: Entry.Role, state: Entry.State) { + guard !entries.contains(where: { $0.id == id }) else { return } + entries.append(.init(id: id, role: role, text: "", state: state)) + } + + private mutating func append(_ delta: String, to id: String, role: Entry.Role) { + guard !delta.isEmpty else { return } + if let index = entries.firstIndex(where: { $0.id == id }) { + entries[index].text += delta + entries[index].state = .streaming + } else { + entries.append(.init(id: id, role: role, text: delta, state: .streaming)) + } + } + + private mutating func finish(text: String, id: String, role: Entry.Role) { + let text = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty else { + entries.removeAll { $0.id == id } + return + } + + if let index = entries.firstIndex(where: { $0.id == id }) { + entries[index].text = text + entries[index].state = .complete + } else { + entries.append(.init(id: id, role: role, text: text, state: .complete)) + } + } + + private mutating func move(itemID: String, after previousItemID: String?) { + guard let sourceIndex = entries.firstIndex(where: { $0.id == itemID }) else { return } + + guard let previousItemID else { + if sourceIndex != entries.startIndex { + let entry = entries.remove(at: sourceIndex) + entries.insert(entry, at: entries.startIndex) + } + return + } + guard + let previousIndex = entries.firstIndex(where: { $0.id == previousItemID }), + sourceIndex != previousIndex + 1 + else { return } + + let entry = entries.remove(at: sourceIndex) + let adjustedPreviousIndex = entries.firstIndex(where: { $0.id == previousItemID }) ?? previousIndex + entries.insert(entry, at: min(adjustedPreviousIndex + 1, entries.endIndex)) + } +} diff --git a/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversationControls.swift b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversationControls.swift new file mode 100644 index 00000000..01526195 --- /dev/null +++ b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversationControls.swift @@ -0,0 +1,58 @@ +// +// RealtimeConversationControls.swift +// SwiftOpenAIExample +// + +import SwiftUI + +struct RealtimeConversationControls: View { + let provider: RealtimeConversationProvider + + var body: some View { + VStack(spacing: 10) { + if let errorMessage = provider.errorMessage { + Text(errorMessage) + .font(.caption) + .foregroundStyle(.red) + .frame(maxWidth: .infinity, alignment: .leading) + .accessibilityLabel("Realtime error: \(errorMessage)") + } + + HStack(spacing: 12) { + Button( + provider.isConnected ? "Stop" : "Start", + systemImage: provider.isConnected ? "stop.fill" : "mic.fill", + action: toggleConversation) + .frame(maxWidth: .infinity) + .buttonStyle(.borderedProminent) + .disabled(provider.phase == .connecting) + + Button("Ask Tool", systemImage: "wrench.and.screwdriver", action: askTool) + .frame(maxWidth: .infinity) + .buttonStyle(.bordered) + .disabled(!provider.canSendToolPrompt) + } + } + .padding() + .background(.background) + .overlay(alignment: .top) { + Divider() + } + } + + private func toggleConversation() { + Task { + if provider.isConnected { + await provider.stop() + } else { + await provider.start() + } + } + } + + private func askTool() { + Task { + await provider.askToolPrompt() + } + } +} diff --git a/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversationDemoView.swift b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversationDemoView.swift new file mode 100644 index 00000000..281c8e8c --- /dev/null +++ b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversationDemoView.swift @@ -0,0 +1,56 @@ +// +// RealtimeConversationDemoView.swift +// SwiftOpenAIExample +// + +import SwiftOpenAI +import SwiftUI + +struct RealtimeConversationDemoView: View { + init(service: OpenAIService) { + _provider = State(initialValue: RealtimeConversationProvider(service: service)) + } + + var body: some View { + VStack(spacing: 0) { + RealtimeConversationStatusView(phase: provider.phase) + + ScrollViewReader { proxy in + ScrollView { + LazyVStack(spacing: 10) { + ForEach(provider.conversation.entries) { entry in + RealtimeTranscriptRow(entry: entry) + .id(entry.id) + } + } + .padding() + } + .overlay { + if provider.conversation.entries.isEmpty { + ContentUnavailableView( + "Ready for a conversation", + systemImage: "waveform", + description: Text("Start the session, then speak naturally.")) + } + } + .onChange(of: provider.conversation.entries.last) { _, entry in + guard let entry else { return } + proxy.scrollTo(entry.id, anchor: .bottom) + } + } + + RealtimeConversationControls(provider: provider) + } + .navigationTitle("Realtime") + .navigationBarTitleDisplayMode(.inline) + .onDisappear(perform: stopConversation) + } + + @State private var provider: RealtimeConversationProvider + + private func stopConversation() { + Task { + await provider.stop() + } + } +} diff --git a/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversationProvider.swift b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversationProvider.swift new file mode 100644 index 00000000..b0d4529b --- /dev/null +++ b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversationProvider.swift @@ -0,0 +1,483 @@ +// +// RealtimeConversationProvider.swift +// SwiftOpenAIExample +// + +import AVFoundation +import Foundation +import Observation +import SwiftOpenAI + +@MainActor +@Observable +final class RealtimeConversationProvider { + init(service: OpenAIService) { + self.service = service + } + + enum Phase: Equatable { + case connecting + case idle + case listening + case responding + } + + private(set) var conversation = RealtimeConversation() + private(set) var errorMessage: String? + private(set) var phase = Phase.idle + + var canSendToolPrompt: Bool { + phase == .listening + } + + var isConnected: Bool { + phase == .listening || phase == .responding + } + + func start() async { + guard phase == .idle else { return } + + phase = .connecting + errorMessage = nil + conversation.reset() + hasReceivedMicrophoneBuffer = false + turnCoordinator.reset() + await microphoneGate.close() + + guard await Self.requestMicrophonePermission() else { + errorMessage = "Microphone permission is required for realtime voice." + phase = .idle + return + } + + do { + let realtimeSession = try await service.realtimeSession( + model: Self.model, + configuration: Self.sessionConfiguration) + session = realtimeSession + + let audioController = try await AudioController(modes: [.playback, .record]) + self.audioController = audioController + let microphoneStream = try await audioController.micStream() + + startReceivingEvents(audioController: audioController, session: realtimeSession) + startMicrophoneStream(microphoneStream, session: realtimeSession) + startMicrophoneWatchdog() + } catch { + errorMessage = error.localizedDescription + await tearDownResources() + phase = .idle + } + } + + func stop() async { + guard phase != .idle || session != nil || audioController != nil else { return } + phase = .idle + errorMessage = nil + await tearDownResources() + } + + func askToolPrompt() async { + guard let session, canSendToolPrompt else { return } + + let itemID = "item_demo_\(UUID().uuidString.replacing("-", with: ""))" + let visiblePrompt = "What is the current demo context?" + conversation.appendUserText(visiblePrompt, itemID: itemID) + turnCoordinator.responseDidStart() + phase = .responding + + await session.sendMessage(OpenAIRealtimeConversationItemCreate( + item: .init( + id: itemID, + role: "user", + text: "Use the get_demo_context tool and summarize the current demo context."))) + await session.sendMessage(OpenAIRealtimeResponseCreate()) + } + + private static let model = Model.gptRealtime21.value + + private static let sessionConfiguration = OpenAIRealtimeSessionConfiguration( + inputAudioFormat: .pcm16, + inputAudioTranscription: .init(model: Model.gptRealtimeWhisper.value, delay: .low, language: "en"), + instructions: """ + You are a concise realtime voice assistant in the SwiftOpenAI demo app. + When the user asks about the current time, device, app state, or demo context, call get_demo_context before answering. + Keep spoken answers brief and natural. + """, + maxResponseOutputTokens: .int(4096), + modalities: [.audio], + outputAudioFormat: .pcm16, + parallelToolCalls: true, + reasoning: .init(effort: .low), + tools: [.function(demoContextTool)], + toolChoice: .auto, + turnDetection: .init(type: .semanticVAD(eagerness: .auto, createResponse: true, interruptResponse: true)), + voice: "marin") + + private static let demoContextTool = OpenAIRealtimeSessionConfiguration.FunctionTool( + name: "get_demo_context", + description: "Get current context from the SwiftOpenAI realtime demo app.", + parameters: [ + "type": "object", + "properties": [ + "topic": [ + "type": "string", + "description": "The context to retrieve.", + "enum": ["time", "device", "session"], + ], + ], + "required": ["topic"], + "additionalProperties": false, + ]) + + private let service: OpenAIService + private var audioController: AudioController? + private var currentAudioItemID: String? + private var hasReceivedMicrophoneBuffer = false + private let microphoneGate = RealtimeMicrophoneGate() + private var micTask: Task? + private var microphoneWatchdogTask: Task? + private var playbackDrainTask: Task? + private var receiveTask: Task? + private var session: OpenAIRealtimeSession? + private var turnCoordinator = RealtimeTurnCoordinator() + + private static func executeTool(name: String, arguments: String) -> String { + guard name == "get_demo_context" else { + return #"{"error":"Unknown tool"}"# + } + + let topic = decodeTopic(from: arguments) ?? "session" + let payload: [String: String] = [ + "topic": topic, + "model": model, + "current_time": ISO8601DateFormatter().string(from: .now), + "session_state": "connected", + "audio_transport": "URLSessionWebSocketTask", + ] + + guard + let data = try? JSONSerialization.data(withJSONObject: payload, options: [.sortedKeys]), + let json = String(data: data, encoding: .utf8) + else { + return #"{"error":"Could not encode tool output"}"# + } + return json + } + + private static func decodeTopic(from arguments: String) -> String? { + guard + let data = arguments.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] + else { + return nil + } + return object["topic"] as? String + } + + private static func responseError(from statusDetails: [String: OpenAIJSONValue]?) -> String? { + guard let statusDetails else { return nil } + if + case .object(let error)? = statusDetails["error"], + case .string(let message)? = error["message"] + { + return message + } + guard + case .object(let nestedStatusDetails)? = statusDetails["status_details"], + case .object(let nestedError)? = nestedStatusDetails["error"], + case .string(let message)? = nestedError["message"] + else { + return nil + } + return message + } + + private static func transcript(from content: [[String: OpenAIJSONValue]]?) -> String? { + content?.compactMap { part in + if case .string(let transcript)? = part["transcript"] { + return transcript + } + if case .string(let text)? = part["text"] { + return text + } + return nil + }.joined() + } + + private static func requestMicrophonePermission() async -> Bool { + #if os(macOS) + let currentPermission = await AVAudioApplication.shared.recordPermission + if currentPermission == .granted { + return true + } + return await AVAudioApplication.requestRecordPermission() + #else + switch AVAudioSession.sharedInstance().recordPermission { + case .granted: + return true + + case .denied: + return false + + case .undetermined: + return await withCheckedContinuation { continuation in + AVAudioSession.sharedInstance().requestRecordPermission { granted in + continuation.resume(returning: granted) + } + } + + @unknown default: + return false + } + #endif + } + + private func startMicrophoneStream( + _ microphoneStream: AsyncStream, + session: OpenAIRealtimeSession) + { + let microphoneGate = microphoneGate + micTask = Task { @RealtimeActor [weak self] in + var hasReportedFirstBuffer = false + for await buffer in microphoneStream { + guard !Task.isCancelled else { return } + if !hasReportedFirstBuffer { + hasReportedFirstBuffer = true + await self?.microphoneDidProduceBuffer() + } + guard microphoneGate.isOpen else { continue } + guard let base64Audio = AudioUtils.base64EncodeAudioPCMBuffer(from: buffer) else { continue } + await session.sendMessage(OpenAIRealtimeInputAudioBufferAppend(audio: base64Audio)) + } + } + } + + private func startMicrophoneWatchdog() { + microphoneWatchdogTask?.cancel() + microphoneWatchdogTask = Task { [weak self] in + do { + try await Task.sleep(for: .seconds(5)) + } catch { + return + } + guard let self, !hasReceivedMicrophoneBuffer, phase != .idle else { return } + errorMessage = "The microphone started but did not produce audio buffers." + await tearDownResources() + phase = .idle + } + } + + private func microphoneDidProduceBuffer() { + guard !hasReceivedMicrophoneBuffer else { return } + hasReceivedMicrophoneBuffer = true + microphoneWatchdogTask?.cancel() + microphoneWatchdogTask = nil + updateConnectedPhase() + } + + private func updateConnectedPhase() { + guard session != nil else { return } + if turnCoordinator.isResponseInProgress || turnCoordinator.isWaitingForPlayback { + phase = .responding + } else if turnCoordinator.isSessionReady, hasReceivedMicrophoneBuffer { + phase = .listening + } else { + phase = .connecting + } + } + + private func startReceivingEvents( + audioController: AudioController, + session: OpenAIRealtimeSession) + { + receiveTask = Task { [weak self] in + for await message in session.receiver { + guard !Task.isCancelled, let self else { return } + await handle(message, audioController: audioController, session: session) + } + + guard !Task.isCancelled, let self, phase != .idle else { return } + await handleUnexpectedDisconnect(message: "Realtime connection closed.") + } + } + + private func handle( + _ message: OpenAIRealtimeMessage, + audioController: AudioController, + session: OpenAIRealtimeSession) + async + { + switch message { + case .error(let message): + errorMessage = message ?? "Unknown Realtime error" + _ = turnCoordinator.responseDidFinish(responseID: nil, status: "failed") + beginPlaybackDrain(audioController: audioController) + + case .disconnected(let message): + await handleUnexpectedDisconnect(message: message) + + case .sessionUpdated: + errorMessage = nil + turnCoordinator.sessionDidBecomeReady() + await microphoneGate.open() + updateConnectedPhase() + + case .responseCreated(let responseID): + errorMessage = nil + playbackDrainTask?.cancel() + playbackDrainTask = nil + turnCoordinator.responseDidStart(responseID: responseID) + phase = .responding + + case .responseAudioDelta(let itemID, _, let base64Audio): + currentAudioItemID = itemID + await audioController.playPCM16Audio(base64String: base64Audio, itemID: itemID) + + case .responseAudioDone: + break + + case .inputAudioBufferSpeechStarted(let itemID, _): + conversation.beginUserTurn(itemID: itemID) + let audioEndMS = await audioController.interruptPlayback() + if let currentAudioItemID, let audioEndMS { + await session.sendMessage(OpenAIRealtimeConversationItemTruncate( + itemID: currentAudioItemID, + audioEndMS: audioEndMS)) + } + currentAudioItemID = nil + + case .inputAudioTranscriptionDelta(let itemID, let delta): + conversation.appendUserTranscript(delta, itemID: itemID) + + case .inputAudioTranscriptionCompleted(let itemID, let transcript): + conversation.finishUserTranscript(transcript, itemID: itemID) + + case .responseOutputItemAdded(let itemID, let type): + if type == "message" { + conversation.beginAssistantTurn(itemID: itemID, responseID: nil) + } + + case .responseTranscriptDelta(let itemID, let responseID, let delta), + .responseTextDelta(let itemID, let responseID, let delta): + conversation.appendAssistantTranscript(delta, itemID: itemID, responseID: responseID) + + case .responseTranscriptDone(let itemID, let responseID, let transcript): + conversation.finishAssistantTranscript(transcript, itemID: itemID, responseID: responseID) + + case .responseTextDone(let itemID, let responseID, let text): + conversation.finishAssistantTranscript(text, itemID: itemID, responseID: responseID) + + case .responseOutputItemDone(let itemID, let type, let content): + if type == "message" { + conversation.finishAssistantItem(itemID: itemID, text: Self.transcript(from: content)) + } + + case .conversationItemCreated(let itemID, let type, let role, let previousItemID): + conversation.registerItem( + itemID: itemID, + type: type, + role: role, + previousItemID: previousItemID) + + case .conversationItemDone(let itemID, let type, let role, let previousItemID, let content): + conversation.registerItem( + itemID: itemID, + type: type, + role: role, + previousItemID: previousItemID) + let transcript = Self.transcript(from: content) + if role == "assistant" { + conversation.finishAssistantItem(itemID: itemID, text: transcript) + } else if role == "user", let transcript { + conversation.finishUserTranscript(transcript, itemID: itemID) + } + + case .responseFunctionCallArgumentsDelta, .rateLimitsUpdated: + break + + case .responseFunctionCallArgumentsDone(let name, let arguments, let callID, _, let responseID): + let output = Self.executeTool(name: name, arguments: arguments) + conversation.recordToolCall(name: name, callID: callID) + await session.sendMessage(OpenAIRealtimeFunctionCallOutput(callID: callID, output: output)) + turnCoordinator.toolOutputWasSent(responseID: responseID) + + case .responseDone(let responseID, let status, let statusDetails): + conversation.finishResponse() + let completion = turnCoordinator.responseDidFinish(responseID: responseID, status: status) + + if status == "failed" || status == "incomplete" { + errorMessage = Self.responseError(from: statusDetails) ?? "Response \(status)." + } + + if completion.shouldCreateToolContinuation { + await sendToolContinuation(session: session) + } else if completion.didFinishCurrentResponse { + beginPlaybackDrain(audioController: audioController) + } + + default: + break + } + } + + private func sendToolContinuation(session: OpenAIRealtimeSession) async { + turnCoordinator.responseDidStart() + phase = .responding + await session.sendMessage(OpenAIRealtimeResponseCreate()) + } + + private func beginPlaybackDrain(audioController: AudioController) { + playbackDrainTask?.cancel() + turnCoordinator.playbackDrainDidStart() + phase = .responding + playbackDrainTask = Task { [weak self] in + await audioController.waitUntilPlaybackFinishes() + guard !Task.isCancelled, let self else { return } + turnCoordinator.playbackDidFinish() + playbackDrainTask = nil + currentAudioItemID = nil + if turnCoordinator.canStreamMicrophone, hasReceivedMicrophoneBuffer, session != nil { + phase = .listening + } else if !turnCoordinator.isSessionReady, session != nil { + phase = .connecting + } + } + } + + private func handleUnexpectedDisconnect(message: String?) async { + if let message, !message.isEmpty { + errorMessage = message + } + phase = .idle + await tearDownResources(disconnectSession: false) + } + + private func tearDownResources(disconnectSession: Bool = true) async { + currentAudioItemID = nil + hasReceivedMicrophoneBuffer = false + turnCoordinator.reset() + await microphoneGate.close() + + micTask?.cancel() + microphoneWatchdogTask?.cancel() + playbackDrainTask?.cancel() + receiveTask?.cancel() + micTask = nil + microphoneWatchdogTask = nil + playbackDrainTask = nil + receiveTask = nil + + let audioController = audioController + let session = session + self.audioController = nil + self.session = nil + + if let audioController { + await audioController.stop() + } + if disconnectSession, let session { + await session.disconnect() + } + } +} diff --git a/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversationStatusView.swift b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversationStatusView.swift new file mode 100644 index 00000000..a28e65cd --- /dev/null +++ b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeConversationStatusView.swift @@ -0,0 +1,63 @@ +// +// RealtimeConversationStatusView.swift +// SwiftOpenAIExample +// + +import SwiftUI + +struct RealtimeConversationStatusView: View { + let phase: RealtimeConversationProvider.Phase + + var body: some View { + HStack(spacing: 8) { + Circle() + .fill(indicatorColor) + .frame(width: 10, height: 10) + .accessibilityHidden(true) + + Text(title) + .font(.headline) + + Spacer() + + if phase == .connecting { + ProgressView() + .controlSize(.small) + .accessibilityLabel("Connecting") + } else if phase == .listening { + Image(systemName: "waveform") + .foregroundStyle(.secondary) + .accessibilityHidden(true) + } + } + .padding() + .background(.thinMaterial) + .accessibilityElement(children: .combine) + } + + private var indicatorColor: Color { + switch phase { + case .listening: + .green + case .responding: + .blue + case .connecting: + .orange + case .idle: + .secondary + } + } + + private var title: String { + switch phase { + case .connecting: + "Connecting" + case .idle: + "Idle" + case .listening: + "Listening" + case .responding: + "Assistant responding" + } + } +} diff --git a/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeMicrophoneGate.swift b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeMicrophoneGate.swift new file mode 100644 index 00000000..423b1b55 --- /dev/null +++ b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeMicrophoneGate.swift @@ -0,0 +1,22 @@ +// +// RealtimeMicrophoneGate.swift +// SwiftOpenAIExample +// + +import SwiftOpenAI + +/// Keeps high-frequency microphone work off the main actor while the UI owns conversation state. +@RealtimeActor +final class RealtimeMicrophoneGate { + nonisolated init() { } + + private(set) var isOpen = false + + func close() { + isOpen = false + } + + func open() { + isOpen = true + } +} diff --git a/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeTranscriptRow.swift b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeTranscriptRow.swift new file mode 100644 index 00000000..b6b42407 --- /dev/null +++ b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeTranscriptRow.swift @@ -0,0 +1,77 @@ +// +// RealtimeTranscriptRow.swift +// SwiftOpenAIExample +// + +import SwiftUI + +struct RealtimeTranscriptRow: View { + let entry: RealtimeConversation.Entry + + var body: some View { + HStack { + if entry.role == .user { + Spacer(minLength: 40) + bubble + } else { + bubble + Spacer(minLength: 40) + } + } + .accessibilityElement(children: .combine) + .accessibilityLabel("\(speaker): \(displayText)") + } + + private var bubble: some View { + HStack(spacing: 8) { + if entry.state == .streaming { + ProgressView() + .controlSize(.small) + .tint(foregroundStyle) + .accessibilityHidden(true) + } + + Text(displayText) + .font(.body) + .textSelection(.enabled) + } + .padding(.horizontal, 12) + .padding(.vertical, 8) + .foregroundStyle(foregroundStyle) + .background(backgroundStyle) + .clipShape(.rect(cornerRadius: 12)) + } + + private var displayText: String { + guard !entry.text.isEmpty else { + return entry.role == .user ? "Listening…" : "Thinking…" + } + return entry.text + } + + private var speaker: String { + switch entry.role { + case .assistant: + "Assistant" + case .tool: + "Tool" + case .user: + "You" + } + } + + private var foregroundStyle: Color { + entry.role == .user ? .white : .primary + } + + private var backgroundStyle: Color { + switch entry.role { + case .assistant: + .secondary.opacity(0.14) + case .tool: + .green.opacity(0.18) + case .user: + .blue + } + } +} diff --git a/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeTurnCoordinator.swift b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeTurnCoordinator.swift new file mode 100644 index 00000000..ba2e4120 --- /dev/null +++ b/Examples/SwiftOpenAIExample/SwiftOpenAIExample/RealtimeDemo/RealtimeTurnCoordinator.swift @@ -0,0 +1,88 @@ +// +// RealtimeTurnCoordinator.swift +// SwiftOpenAIExample +// + +/// Owns response and tool-continuation state independently from transport and UI concerns. +struct RealtimeTurnCoordinator { + struct ResponseCompletion: Equatable { + let didFinishCurrentResponse: Bool + let shouldCreateToolContinuation: Bool + } + + private(set) var isResponseInProgress = false + private(set) var isWaitingForPlayback = false + private(set) var isSessionReady = false + + /// Full-duplex capture remains open while the model speaks so server VAD can detect barge-in. + var canStreamMicrophone: Bool { + isSessionReady + } + + mutating func reset() { + currentResponseID = nil + hasPendingToolContinuation = false + isResponseInProgress = false + isWaitingForPlayback = false + isSessionReady = false + pendingToolResponseID = nil + } + + mutating func sessionDidBecomeReady() { + isSessionReady = true + } + + mutating func responseDidStart(responseID: String? = nil) { + currentResponseID = responseID ?? currentResponseID + isResponseInProgress = true + isWaitingForPlayback = false + } + + mutating func toolOutputWasSent(responseID: String?) { + guard isResponseInProgress else { return } + hasPendingToolContinuation = true + pendingToolResponseID = responseID ?? currentResponseID + } + + mutating func responseDidFinish( + responseID: String?, + status: String) + -> ResponseCompletion + { + let didFinishCurrentResponse = isResponseInProgress + && responseIDsMatch(currentResponseID, responseID) + if didFinishCurrentResponse { + currentResponseID = nil + isResponseInProgress = false + } + + let didFinishToolResponse = hasPendingToolContinuation + && responseIDsMatch(pendingToolResponseID, responseID) + let shouldCreateToolContinuation = didFinishToolResponse && status == "completed" + if didFinishToolResponse { + hasPendingToolContinuation = false + pendingToolResponseID = nil + } + + return ResponseCompletion( + didFinishCurrentResponse: didFinishCurrentResponse, + shouldCreateToolContinuation: shouldCreateToolContinuation) + } + + mutating func playbackDrainDidStart() { + isWaitingForPlayback = true + } + + mutating func playbackDidFinish() { + isWaitingForPlayback = false + } + + private var currentResponseID: String? + private var hasPendingToolContinuation = false + private var pendingToolResponseID: String? + + private func responseIDsMatch(_ lhs: String?, _ rhs: String?) -> Bool { + guard let lhs, let rhs else { return true } + return lhs == rhs + } +} diff --git a/Examples/SwiftOpenAIExample/SwiftOpenAIExampleTests/SwiftOpenAIExampleTests.swift b/Examples/SwiftOpenAIExample/SwiftOpenAIExampleTests/SwiftOpenAIExampleTests.swift index 94e0981e..6abff67b 100644 --- a/Examples/SwiftOpenAIExample/SwiftOpenAIExampleTests/SwiftOpenAIExampleTests.swift +++ b/Examples/SwiftOpenAIExample/SwiftOpenAIExampleTests/SwiftOpenAIExampleTests.swift @@ -6,6 +6,7 @@ // import XCTest +@testable import SwiftOpenAIExample final class SwiftOpenAIExampleTests: XCTestCase { override func setUpWithError() throws { @@ -24,6 +25,117 @@ final class SwiftOpenAIExampleTests: XCTestCase { // Mark your test async to allow awaiting for asynchronous code to complete. Check the results with assertions afterwards. } + func testRealtimeConversationReducesTranscriptDeltasIntoStableRows() { + var conversation = RealtimeConversation() + + conversation.beginUserTurn(itemID: "user-1") + conversation.appendUserTranscript("Hello", itemID: "user-1") + conversation.appendUserTranscript(" there", itemID: "user-1") + conversation.finishUserTranscript("Hello there", itemID: "user-1") + + XCTAssertEqual(conversation.entries.count, 1) + XCTAssertEqual(conversation.entries[0].id, "user-1") + XCTAssertEqual(conversation.entries[0].text, "Hello there") + XCTAssertEqual(conversation.entries[0].state, .complete) + } + + func testRealtimeConversationKeepsConcurrentAssistantItemsSeparate() { + var conversation = RealtimeConversation() + + conversation.appendAssistantTranscript("First", itemID: "assistant-1", responseID: "response-1") + conversation.appendAssistantTranscript("Second", itemID: "assistant-2", responseID: "response-1") + conversation.finishAssistantTranscript("First answer", itemID: "assistant-1", responseID: "response-1") + conversation.finishAssistantTranscript("Second answer", itemID: "assistant-2", responseID: "response-1") + + XCTAssertEqual(conversation.entries.map(\.id), ["assistant-1", "assistant-2"]) + XCTAssertEqual(conversation.entries.map(\.text), ["First answer", "Second answer"]) + } + + func testRealtimeConversationUsesServerOrdering() { + var conversation = RealtimeConversation() + conversation.appendUserText("First", itemID: "user-1") + conversation.appendUserText("Third", itemID: "user-3") + + conversation.registerItem( + itemID: "user-2", + type: "message", + role: "user", + previousItemID: "user-1") + conversation.finishUserTranscript("Second", itemID: "user-2") + + XCTAssertEqual(conversation.entries.map(\.id), ["user-1", "user-2", "user-3"]) + } + + func testRealtimeTurnCoordinatorKeepsMicOpenForBargeIn() { + var coordinator = RealtimeTurnCoordinator() + + coordinator.sessionDidBecomeReady() + XCTAssertTrue(coordinator.canStreamMicrophone) + + coordinator.responseDidStart() + XCTAssertTrue(coordinator.canStreamMicrophone) + + _ = coordinator.responseDidFinish(responseID: nil, status: "completed") + coordinator.playbackDrainDidStart() + XCTAssertTrue(coordinator.canStreamMicrophone) + + coordinator.playbackDidFinish() + XCTAssertTrue(coordinator.canStreamMicrophone) + } + + func testRealtimeTurnCoordinatorContinuesOnlyCompletedMatchingToolResponse() { + var coordinator = RealtimeTurnCoordinator() + coordinator.sessionDidBecomeReady() + coordinator.responseDidStart(responseID: "response-1") + coordinator.toolOutputWasSent(responseID: "response-1") + + let unrelated = coordinator.responseDidFinish(responseID: "response-0", status: "cancelled") + XCTAssertFalse(unrelated.didFinishCurrentResponse) + XCTAssertFalse(unrelated.shouldCreateToolContinuation) + + let completed = coordinator.responseDidFinish(responseID: "response-1", status: "completed") + XCTAssertTrue(completed.didFinishCurrentResponse) + XCTAssertTrue(completed.shouldCreateToolContinuation) + } + + func testRealtimeTurnCoordinatorDoesNotContinueCancelledToolResponse() { + var coordinator = RealtimeTurnCoordinator() + coordinator.sessionDidBecomeReady() + coordinator.responseDidStart(responseID: "response-1") + coordinator.toolOutputWasSent(responseID: "response-1") + + let completion = coordinator.responseDidFinish(responseID: "response-1", status: "cancelled") + + XCTAssertTrue(completion.didFinishCurrentResponse) + XCTAssertFalse(completion.shouldCreateToolContinuation) + XCTAssertTrue(coordinator.canStreamMicrophone) + XCTAssertFalse(coordinator.isResponseInProgress) + } + + func testRealtimeTurnCoordinatorIgnoresLateDuplicateCompletion() { + var coordinator = RealtimeTurnCoordinator() + coordinator.sessionDidBecomeReady() + coordinator.responseDidStart(responseID: "response-1") + _ = coordinator.responseDidFinish(responseID: "response-1", status: "completed") + + let duplicate = coordinator.responseDidFinish(responseID: "response-1", status: "completed") + + XCTAssertFalse(duplicate.didFinishCurrentResponse) + XCTAssertFalse(duplicate.shouldCreateToolContinuation) + } + + func testRealtimeMicrophoneGateOpensAndCloses() async { + let gate = RealtimeMicrophoneGate() + + await gate.open() + let isOpen = await gate.isOpen + await gate.close() + let isClosed = await !(gate.isOpen) + + XCTAssertTrue(isOpen) + XCTAssertTrue(isClosed) + } + func testPerformanceExample() throws { // This is an example of a performance test case. measure { diff --git a/README.md b/README.md index 8a6477b0..263c5fef 100644 --- a/README.md +++ b/README.md @@ -399,7 +399,7 @@ playAudio(from: audioObjectData) ### Audio Realtime -The [Realtime API](https://platform.openai.com/docs/api-reference/realtime) enables bidirectional voice conversations with OpenAI's models using WebSockets and low-latency audio streaming. The API supports both audio-to-audio and text-to-audio interactions with built-in voice activity detection, transcription, and function calling. +The [Realtime API](https://platform.openai.com/docs/api-reference/realtime) enables bidirectional voice conversations with OpenAI's models using WebSockets and low-latency audio streaming. For voice agents, the current recommended model is `gpt-realtime-2.1`; the API supports audio-to-audio interaction, voice activity detection, transcription, function tools, and MCP tools. **Platform Requirements:** iOS 15+, macOS 13+, watchOS 9+. Requires AVFoundation (not available on Linux). @@ -412,21 +412,29 @@ Parameters /// Configuration for creating a realtime session public struct OpenAIRealtimeSessionConfiguration: Encodable, Sendable { - /// The input audio format. Options: .pcm16, .g711_ulaw, .g711_alaw. Default is .pcm16 + /// The input audio format. Options: .pcm16, .g711Ulaw, .g711Alaw. Encodes to GA audio format objects. let inputAudioFormat: AudioFormat? - /// Configuration for input audio transcription using Whisper + /// Configuration for input audio transcription. let inputAudioTranscription: InputAudioTranscription? + /// Additional fields to include in server outputs. + let include: [String]? /// System instructions for the model. Recommended default provided let instructions: String? - /// Maximum tokens for response output. Can be .value(Int) or .infinite + /// Maximum tokens for response output. Can be .int(Int) or .infinite let maxResponseOutputTokens: MaxResponseOutputTokens? - /// Output modalities: [.audio, .text] or [.text] only. Default is [.audio, .text] + /// Output modalities: [.audio] or [.text]. GA Realtime does not request both at once. let modalities: [Modality]? - /// The output audio format. Options: .pcm16, .g711_ulaw, .g711_alaw. Default is .pcm16 + /// The model used when creating client secrets. + let model: String? + /// The output audio format. Options: .pcm16, .g711Ulaw, .g711Alaw. let outputAudioFormat: AudioFormat? - /// Audio playback speed. Range: 0.25 to 4.0. Default is 1.0 + /// Whether the model may call multiple tools in parallel. + let parallelToolCalls: Bool? + /// Reasoning effort for reasoning-capable Realtime models. + let reasoning: ReasoningConfiguration? + /// Audio playback speed. Range: 0.25 to 1.5. Default is 1.0 let speed: Double? - /// Sampling temperature for model responses. Range: 0.6 to 1.2. Default is 0.8 + /// Legacy beta field. Not encoded for GA Realtime requests. let temperature: Double? /// Array of tools/functions available for the model to call let tools: [Tool]? @@ -434,14 +442,14 @@ public struct OpenAIRealtimeSessionConfiguration: Encodable, Sendable { let toolChoice: ToolChoice? /// Voice activity detection configuration. Options: .serverVAD or .semanticVAD let turnDetection: TurnDetection? - /// The voice to use. Options: "alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse" + /// The voice to use. Built-ins include "alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse", "marin", and "cedar". let voice: String? /// Available audio formats public enum AudioFormat: String, Encodable, Sendable { case pcm16 - case g711_ulaw = "g711-ulaw" - case g711_alaw = "g711-alaw" + case g711Ulaw = "g711_ulaw" + case g711Alaw = "g711_alaw" } /// Output modalities @@ -474,24 +482,25 @@ Response /// Messages received from the realtime API public enum OpenAIRealtimeMessage: Sendable { case error(String?) // Error occurred + case disconnected(String?) // WebSocket closed unexpectedly case sessionCreated // Session successfully created case sessionUpdated // Configuration updated - case responseCreated // Model started generating response - case responseAudioDelta(String) // Audio chunk (base64 PCM16) - case inputAudioBufferSpeechStarted // User started speaking (VAD detected) - case responseFunctionCallArgumentsDone(name: String, arguments: String, callId: String) - case responseTranscriptDelta(String) // Partial AI transcript - case responseTranscriptDone(String) // Complete AI transcript + case responseCreated(responseID: String?) + case responseAudioDelta(itemID: String?, responseID: String?, delta: String) + case inputAudioBufferSpeechStarted(itemID: String?, audioStartMS: Int?) + case responseFunctionCallArgumentsDone(name: String, arguments: String, callID: String, itemID: String?, responseID: String?) + case responseTranscriptDelta(itemID: String?, responseID: String?, delta: String) + case responseTranscriptDone(itemID: String?, responseID: String?, transcript: String) case inputAudioBufferTranscript(String) // User audio transcript - case inputAudioTranscriptionDelta(String) // Partial user transcription - case inputAudioTranscriptionCompleted(String) // Complete user transcription + case inputAudioTranscriptionDelta(itemID: String?, delta: String) + case inputAudioTranscriptionCompleted(itemID: String?, transcript: String) } ``` Supporting Types ```swift /// Manages microphone input and audio playback for realtime conversations. -/// Audio played through AudioController does not interfere with mic input (the model won't hear itself). +/// Full-duplex mode shares one voice-processing graph for capture, playback, and echo cancellation. @RealtimeActor public final class AudioController { @@ -504,16 +513,19 @@ public final class AudioController { case playback // Enable audio playback } - /// Returns an AsyncStream of microphone audio buffers + /// Installs the microphone tap, starts the shared audio engine, and returns its buffer stream /// - Throws: OpenAIError if .record mode wasn't enabled during initialization public func micStream() throws -> AsyncStream /// Plays base64-encoded PCM16 audio from the model /// - Parameter base64String: Base64-encoded PCM16 audio data - public func playPCM16Audio(base64String: String) + public func playPCM16Audio(base64String: String, itemID: String? = nil) + + /// Interrupts playback and returns the heard duration of the current item + public func interruptPlayback() -> Int? - /// Interrupts current audio playback (useful when user starts speaking) - public func interruptPlayback() + /// Waits until all currently queued audio buffers have played + public func waitUntilPlaybackFinishes() async /// Stops all audio operations public func stop() @@ -533,53 +545,78 @@ Usage ```swift // 1. Create session configuration let configuration = OpenAIRealtimeSessionConfiguration( - voice: "alloy", + inputAudioFormat: .pcm16, + inputAudioTranscription: .init(model: Model.gptRealtimeWhisper.value, delay: .low, language: "en"), instructions: "You are a helpful AI assistant. Be concise and friendly.", - turnDetection: .serverVAD( - prefixPaddingMs: 300, - silenceDurationMs: 500, - threshold: 0.5 - ), - inputAudioTranscription: .init(model: "whisper-1") + maxResponseOutputTokens: .int(4096), + modalities: [.audio], + outputAudioFormat: .pcm16, + parallelToolCalls: true, + reasoning: .init(effort: .low), + turnDetection: .init(type: .semanticVAD(eagerness: .auto, createResponse: true, interruptResponse: true)), + voice: "marin" ) // 2. Create realtime session let session = try await service.realtimeSession( - model: "gpt-4o-mini-realtime-preview-2024-12-17", + model: Model.gptRealtime21.value, configuration: configuration ) // 3. Initialize audio controller for recording and playback let audioController = try await AudioController(modes: [.record, .playback]) +var isSessionReady = false +var currentAudioItemID: String? +var pendingToolResponseID: String? // 4. Handle incoming messages from OpenAI Task { for await message in session.receiver { switch message { - case .responseAudioDelta(let audio): - // Play audio from the model - audioController.playPCM16Audio(base64String: audio) + case .sessionUpdated: + isSessionReady = true + + case .responseAudioDelta(let itemID, _, let audio): + currentAudioItemID = itemID + audioController.playPCM16Audio(base64String: audio, itemID: itemID) case .inputAudioBufferSpeechStarted: - // User started speaking - interrupt model's audio - audioController.interruptPlayback() + if + let currentAudioItemID, + let audioEndMS = audioController.interruptPlayback() + { + await session.sendMessage(OpenAIRealtimeConversationItemTruncate( + itemID: currentAudioItemID, + audioEndMS: audioEndMS)) + } + currentAudioItemID = nil - case .responseTranscriptDelta(let text): + case .responseTranscriptDelta(_, _, let text): // Display partial model transcript print("Model (partial): \(text)") - case .responseTranscriptDone(let text): + case .responseTranscriptDone(_, _, let text): // Display complete model transcript print("Model: \(text)") - case .inputAudioTranscriptionCompleted(let text): + case .inputAudioTranscriptionCompleted(_, let text): // Display user's transcribed speech print("User: \(text)") - case .responseFunctionCallArgumentsDone(let name, let args, let callId): + case .responseFunctionCallArgumentsDone(let name, let args, let callId, _, let responseID): // Handle function call from model print("Function call: \(name) with args: \(args)") - // Execute function and send result back + let result = executeFunction(name: name, arguments: args) + await session.sendMessage(OpenAIRealtimeFunctionCallOutput(callID: callId, output: result)) + pendingToolResponseID = responseID + + case .responseDone(let responseID, let status, _): + if pendingToolResponseID == responseID { + pendingToolResponseID = nil + if status == "completed" { + await session.sendMessage(OpenAIRealtimeResponseCreate()) + } + } case .error(let error): print("Error: \(error ?? "Unknown error")") @@ -594,6 +631,7 @@ Task { Task { do { for try await buffer in audioController.micStream() { + guard isSessionReady else { continue } // Encode audio buffer to base64 guard let base64Audio = AudioUtils.base64EncodeAudioPCMBuffer(from: buffer) else { continue @@ -616,8 +654,8 @@ try await session.sendMessage( // 7. Update session configuration mid-conversation (optional) let newConfig = OpenAIRealtimeSessionConfiguration( - voice: "shimmer", - temperature: 0.9 + instructions: "Be even more concise.", + toolChoice: .auto ) try await session.sendMessage( OpenAIRealtimeSessionUpdate(sessionConfig: newConfig) @@ -631,8 +669,8 @@ session.disconnect() Function Calling ```swift // Define tools in configuration -let tools: [OpenAIRealtimeSessionConfiguration.Tool] = [ - .init( +let tools: [OpenAIRealtimeSessionConfiguration.RealtimeTool] = [ + .function(.init( name: "get_weather", description: "Get the current weather in a location", parameters: [ @@ -645,7 +683,7 @@ let tools: [OpenAIRealtimeSessionConfiguration.Tool] = [ ], "required": ["location"] ] - ) + )) ] let config = OpenAIRealtimeSessionConfiguration( @@ -654,30 +692,35 @@ let config = OpenAIRealtimeSessionConfiguration( toolChoice: .auto ) +var pendingToolResponseID: String? + // Handle function calls in message receiver -case .responseFunctionCallArgumentsDone(let name, let args, let callId): +case .responseFunctionCallArgumentsDone(let name, let args, let callId, _, let responseID): if name == "get_weather" { // Parse arguments and execute function let result = getWeather(arguments: args) // Send result back to model - try await session.sendMessage( - OpenAIRealtimeConversationItemCreate( - item: .functionCallOutput( - callId: callId, - output: result - ) - ) - ) + await session.sendMessage(OpenAIRealtimeFunctionCallOutput(callID: callId, output: result)) + pendingToolResponseID = responseID + } + +case .responseDone(let responseID, let status, _): + if pendingToolResponseID == responseID { + pendingToolResponseID = nil + if status == "completed" { + await session.sendMessage(OpenAIRealtimeResponseCreate()) + } } ``` Advanced Features - **Voice Activity Detection (VAD):** Choose between server-based VAD (with configurable timing) or semantic VAD (with eagerness levels) -- **Transcription:** Enable Whisper transcription for both user input and model output +- **Transcription:** Enable input transcription with `gpt-realtime-whisper` or the supported transcription models - **Session Updates:** Change voice, instructions, or tools mid-conversation without reconnecting - **Response Triggers:** Manually trigger model responses or rely on automatic VAD -- **Platform-Specific Behavior:** Automatically selects optimal audio API based on platform and headphone connection +- **Client Secrets:** Use `createRealtimeClientSecret(parameters:)` for ephemeral credentials in WebRTC integrations +- **Full-Duplex Audio:** Record-and-playback controllers share one `AVAudioEngine` voice-processing graph so the microphone stays live for barge-in without treating model playback as user speech For a complete implementation example, see `Examples/RealtimeExample/RealtimeExample.swift` in the repository. diff --git a/Sources/OpenAI/AIProxy/AIProxyService.swift b/Sources/OpenAI/AIProxy/AIProxyService.swift index 9118063a..fea27e3b 100644 --- a/Sources/OpenAI/AIProxy/AIProxyService.swift +++ b/Sources/OpenAI/AIProxy/AIProxyService.swift @@ -95,6 +95,13 @@ struct AIProxyService: OpenAIService { return AudioSpeechObject(output: data) } + func createRealtimeClientSecret( + parameters _: OpenAIRealtimeClientSecretParameters) + async throws -> OpenAIRealtimeClientSecret + { + fatalError("Realtime API is not yet supported for AIProxy. Please use DefaultOpenAIService instead.") + } + #if canImport(AVFoundation) func realtimeSession( model _: String, diff --git a/Sources/OpenAI/Azure/DefaultOpenAIAzureService.swift b/Sources/OpenAI/Azure/DefaultOpenAIAzureService.swift index d6371043..bea33d26 100644 --- a/Sources/OpenAI/Azure/DefaultOpenAIAzureService.swift +++ b/Sources/OpenAI/Azure/DefaultOpenAIAzureService.swift @@ -50,6 +50,13 @@ public final class DefaultOpenAIAzureService: OpenAIService { "Currently, this API is not supported. We welcome and encourage contributions to our open-source project. Please consider opening an issue or submitting a pull request to add support for this feature.") } + public func createRealtimeClientSecret(parameters _: OpenAIRealtimeClientSecretParameters) async throws + -> OpenAIRealtimeClientSecret + { + fatalError( + "Currently, this API is not supported. We welcome and encourage contributions to our open-source project. Please consider opening an issue or submitting a pull request to add support for this feature.") + } + #if canImport(AVFoundation) public func realtimeSession( model _: String, diff --git a/Sources/OpenAI/LocalModelService/LocalModelService.swift b/Sources/OpenAI/LocalModelService/LocalModelService.swift index 225c1aff..253fb2ea 100644 --- a/Sources/OpenAI/LocalModelService/LocalModelService.swift +++ b/Sources/OpenAI/LocalModelService/LocalModelService.swift @@ -67,6 +67,11 @@ struct LocalModelService: OpenAIService { "Currently, this API is not supported. We welcome and encourage contributions to our open-source project. Please consider opening an issue or submitting a pull request to add support for this feature.") } + func createRealtimeClientSecret(parameters _: OpenAIRealtimeClientSecretParameters) async throws -> OpenAIRealtimeClientSecret { + fatalError( + "Currently, this API is not supported. We welcome and encourage contributions to our open-source project. Please consider opening an issue or submitting a pull request to add support for this feature.") + } + #if canImport(AVFoundation) func realtimeSession( model _: String, diff --git a/Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift b/Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift index 683f0586..22eb53c5 100644 --- a/Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift +++ b/Sources/OpenAI/Private/Audio/AudioPCMPlayer.swift @@ -15,14 +15,8 @@ private let logger = Logger(subsystem: "com.swiftopenai", category: "Audio") // MARK: - AudioPCMPlayer -/// # Warning -/// The order that you initialize `AudioPCMPlayer()` and `MicrophonePCMSampleVendor()` matters, unfortunately. -/// -/// The voice processing audio unit on iOS has a volume bug that is not present on macOS. -/// The volume of playback depends on the initialization order of AVAudioEngine and the `kAudioUnitSubType_VoiceProcessingIO` Audio Unit. -/// We use AudioEngine for playback in this file, and the voice processing audio unit in MicrophonePCMSampleVendor. -/// -/// I find the best result to be initializing `AudioPCMPlayer()` first. Otherwise, the playback volume is too quiet on iOS. +/// Playback shares its `AVAudioEngine` with microphone capture. Keeping both directions in this +/// graph lets the engine's voice-processing I/O node use playback as its echo-cancellation reference. @RealtimeActor final class AudioPCMPlayer { @@ -64,7 +58,7 @@ final class AudioPCMPlayer { logger.debug("AudioPCMPlayer is being freed") } - public func playPCM16Audio(from base64String: String) { + public func playPCM16Audio(from base64String: String, itemID: String?) { guard let audioData = Data(base64Encoded: base64String) else { logger.error("Could not decode base64 string for audio playback") return @@ -109,14 +103,55 @@ final class AudioPCMPlayer { } if audioEngine.isRunning { - playerNode.scheduleBuffer(outPCMBuf, at: nil, options: [], completionHandler: { }) + if !hasActivePlayback || activeItemID != itemID { + hasActivePlayback = true + activeItemID = itemID + playbackStartSampleTime = currentSampleTime + scheduledFrameCount = 0 + } + scheduledFrameCount += AVAudioFramePosition(outPCMBuf.frameLength) + let generation = playbackGeneration + pendingBufferCount += 1 + playerNode.scheduleBuffer( + outPCMBuf, + at: nil, + options: [], + completionCallbackType: .dataPlayedBack) + { [weak self] _ in + Task { @RealtimeActor [weak self] in + self?.didFinishBuffer(generation: generation) + } + } playerNode.play() + if playbackStartSampleTime == nil { + playbackStartSampleTime = currentSampleTime ?? 0 + } } } - public func interruptPlayback() { + public func interruptPlayback() -> Int? { + guard hasActivePlayback else { + playerNode.stop() + return nil + } logger.debug("Interrupting playback") + let playedMilliseconds = Int((Double(playedFrameCount) / playableFormat.sampleRate) * 1000) playerNode.stop() + playbackGeneration += 1 + pendingBufferCount = 0 + resumePlaybackWaiters() + activeItemID = nil + hasActivePlayback = false + playbackStartSampleTime = nil + scheduledFrameCount = 0 + return playedMilliseconds + } + + public func waitUntilPlaybackFinishes() async { + guard pendingBufferCount > 0 else { return } + await withCheckedContinuation { continuation in + playbackWaiters.append(continuation) + } } let audioEngine: AVAudioEngine @@ -124,6 +159,44 @@ final class AudioPCMPlayer { private let inputFormat: AVAudioFormat private let playableFormat: AVAudioFormat private let playerNode: AVAudioPlayerNode + private var activeItemID: String? + private var hasActivePlayback = false + private var playbackStartSampleTime: AVAudioFramePosition? + private var scheduledFrameCount: AVAudioFramePosition = 0 + private var pendingBufferCount = 0 + private var playbackGeneration = 0 + private var playbackWaiters = [CheckedContinuation]() + + private var currentSampleTime: AVAudioFramePosition? { + guard + let renderTime = playerNode.lastRenderTime, + let playerTime = playerNode.playerTime(forNodeTime: renderTime) + else { + return nil + } + return playerTime.sampleTime + } + + private var playedFrameCount: AVAudioFramePosition { + guard let playbackStartSampleTime, let currentSampleTime else { return 0 } + return min(max(0, currentSampleTime - playbackStartSampleTime), scheduledFrameCount) + } + + private func didFinishBuffer(generation: Int) { + guard generation == playbackGeneration, pendingBufferCount > 0 else { return } + pendingBufferCount -= 1 + if pendingBufferCount == 0 { + hasActivePlayback = false + activeItemID = nil + resumePlaybackWaiters() + } + } + + private func resumePlaybackWaiters() { + let waiters = playbackWaiters + playbackWaiters.removeAll(keepingCapacity: true) + for waiter in waiters { waiter.resume() } + } } #endif diff --git a/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift b/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift index c9b59c68..8899eea9 100644 --- a/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift +++ b/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAE.swift @@ -16,6 +16,8 @@ private let logger = Logger(subsystem: "com.swiftopenai", category: "Audio") // MARK: - MicrophonePCMSampleVendorAE /// This is an AVAudioEngine-based implementation that vends PCM16 microphone samples. +/// When the engine also renders playback, voice processing receives the downlink reference it +/// needs for acoustic echo cancellation while keeping microphone capture live for barge-in. /// /// ## Requirements /// @@ -48,6 +50,7 @@ class MicrophonePCMSampleVendorAE: MicrophonePCMSampleVendor { if !AudioUtils.headphonesConnected { try inputNode.setVoiceProcessingEnabled(true) + logger.info("AudioEngine voice processing enabled for full-duplex echo cancellation") } let debugText = """ @@ -63,14 +66,10 @@ class MicrophonePCMSampleVendorAE: MicrophonePCMSampleVendor { } func start() throws -> AsyncStream { - guard - let desiredTapFormat = AVAudioFormat( - commonFormat: .pcmFormatInt16, - sampleRate: inputNode.outputFormat(forBus: 0).sampleRate, - channels: 1, - interleaved: false) - else { - throw OpenAIError.audioConfigurationError("Could not create the desired tap format for realtime") + let tapFormat = inputNode.outputFormat(forBus: 0) + guard tapFormat.sampleRate > 0, tapFormat.channelCount == 1 else { + throw OpenAIError.audioConfigurationError( + "Realtime microphone input must have a valid mono output format") } // The buffer size argument specifies the target number of audio frames. @@ -81,7 +80,7 @@ class MicrophonePCMSampleVendorAE: MicrophonePCMSampleVendor { // // There is a note on the installTap documentation that says AudioEngine may // adjust the bufferSize internally. - let targetBufferSize = UInt32(desiredTapFormat.sampleRate / 20) // 50ms buffers + let targetBufferSize = UInt32(tapFormat.sampleRate / 20) // 50ms buffers logger.info("PCMSampleVendorAE target buffer size is: \(targetBufferSize)") return AsyncStream { [weak self] continuation in @@ -90,7 +89,7 @@ class MicrophonePCMSampleVendorAE: MicrophonePCMSampleVendor { this.installTapNonIsolated( inputNode: this.inputNode, bufferSize: targetBufferSize, - format: desiredTapFormat) + format: tapFormat) } } @@ -106,6 +105,7 @@ class MicrophonePCMSampleVendorAE: MicrophonePCMSampleVendor { private let inputNode: AVAudioInputNode private let microphonePCMSampleVendorCommon = MicrophonePCMSampleVendorCommon() private var continuation: AsyncStream.Continuation? + private var hasLoggedFirstBuffer = false private nonisolated func installTapNonIsolated( inputNode: AVAudioInputNode, @@ -120,6 +120,10 @@ class MicrophonePCMSampleVendorAE: MicrophonePCMSampleVendor { private func processBuffer(_ buffer: AVAudioPCMBuffer) { if let accumulatedBuffer = microphonePCMSampleVendorCommon.resampleAndAccumulate(buffer) { + if !hasLoggedFirstBuffer { + hasLoggedFirstBuffer = true + logger.info("Realtime microphone is producing PCM16 buffers") + } continuation?.yield(accumulatedBuffer) } } diff --git a/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAT.swift b/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAT.swift index c9e63ef3..b9578511 100644 --- a/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAT.swift +++ b/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAT.swift @@ -278,11 +278,11 @@ class MicrophonePCMSampleVendorAT: MicrophonePCMSampleVendor { /// This @RealtimeActor annotation is a lie. @RealtimeActor private let audioRenderCallback: AURenderCallback = { inRefCon, - ioActionFlags, - inTimeStamp, - inBusNumber, - inNumberFrames, - _ in + ioActionFlags, + inTimeStamp, + inBusNumber, + inNumberFrames, + _ in let microphonePCMSampleVendor = Unmanaged .fromOpaque(inRefCon) .takeUnretainedValue() diff --git a/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorCommon.swift b/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorCommon.swift index 6725863b..fd2d6ede 100644 --- a/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorCommon.swift +++ b/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorCommon.swift @@ -20,9 +20,9 @@ nonisolated final class MicrophonePCMSampleVendorCommon { var bufferAccumulator: AVAudioPCMBuffer? var audioConverter: AVAudioConverter? - func resampleAndAccumulate(_ pcm16Buffer: AVAudioPCMBuffer) -> AVAudioPCMBuffer? { + func resampleAndAccumulate(_ inputBuffer: AVAudioPCMBuffer) -> AVAudioPCMBuffer? { if - let resampledBuffer = convertPCM16BufferToExpectedSampleRate(pcm16Buffer), + let resampledBuffer = convertToRealtimeFormat(inputBuffer), let accumulatedBuffer = accummulateAndVendIfFull(resampledBuffer) { return accumulatedBuffer @@ -30,7 +30,7 @@ nonisolated final class MicrophonePCMSampleVendorCommon { return nil } - private func convertPCM16BufferToExpectedSampleRate(_ pcm16Buffer: AVAudioPCMBuffer) -> AVAudioPCMBuffer? { + private func convertToRealtimeFormat(_ inputBuffer: AVAudioPCMBuffer) -> AVAudioPCMBuffer? { guard let audioFormat = AVAudioFormat( commonFormat: .pcmFormatInt16, @@ -43,7 +43,7 @@ nonisolated final class MicrophonePCMSampleVendorCommon { } if audioConverter == nil { - audioConverter = AVAudioConverter(from: pcm16Buffer.format, to: audioFormat) + audioConverter = AVAudioConverter(from: inputBuffer.format, to: audioFormat) } guard let converter = audioConverter else { @@ -66,11 +66,11 @@ nonisolated final class MicrophonePCMSampleVendorCommon { // reached or outStatus.pointee is set to `.noDataNow` or `.endStream`. var error: NSError? nonisolated(unsafe) var ptr: UInt32 = 0 - let targetFrameLength = pcm16Buffer.frameLength + let targetFrameLength = inputBuffer.frameLength let _ = converter.convert(to: outputBuffer, error: &error) { numberOfFrames, outStatus in guard ptr < targetFrameLength, - let workingCopy = advancedPCMBuffer_noCopy(pcm16Buffer, offset: ptr) + let workingCopy = advancedPCMBufferNoCopy(inputBuffer, offset: ptr) else { outStatus.pointee = .noDataNow return nil @@ -92,34 +92,58 @@ nonisolated final class MicrophonePCMSampleVendorCommon { /// The incoming buffer here must be guaranteed at 24kHz in PCM16Int format. private func accummulateAndVendIfFull(_ buf: AVAudioPCMBuffer) -> AVAudioPCMBuffer? { - var returnBuffer: AVAudioPCMBuffer? = nil let targetAccumulatorLength = 2400 - if bufferAccumulator == nil { - bufferAccumulator = AVAudioPCMBuffer(pcmFormat: buf.format, frameCapacity: AVAudioFrameCount(targetAccumulatorLength * 2)) + guard let source = buf.int16ChannelData?[0] else { + logger.error("Converted microphone buffer did not contain PCM16 channel data") + return nil } - guard let accumulator = bufferAccumulator else { return nil } - let copyFrames = min(buf.frameLength, accumulator.frameCapacity - accumulator.frameLength) - let dst = accumulator.int16ChannelData![0].advanced(by: Int(accumulator.frameLength)) - let src = buf.int16ChannelData![0] + var completedBuffer: AVAudioPCMBuffer? + var sourceOffset = 0 + while sourceOffset < Int(buf.frameLength) { + if bufferAccumulator == nil { + bufferAccumulator = AVAudioPCMBuffer( + pcmFormat: buf.format, + frameCapacity: AVAudioFrameCount(targetAccumulatorLength)) + } + guard + let accumulator = bufferAccumulator, + let destination = accumulator.int16ChannelData?[0] + else { + logger.error("Could not create the realtime microphone accumulator") + return completedBuffer + } - dst.update(from: src, count: Int(copyFrames)) - accumulator.frameLength += copyFrames - if accumulator.frameLength >= targetAccumulatorLength { - returnBuffer = accumulator - bufferAccumulator = nil + let remainingTargetFrames = targetAccumulatorLength - Int(accumulator.frameLength) + let copyFrames = min(Int(buf.frameLength) - sourceOffset, remainingTargetFrames) + destination + .advanced(by: Int(accumulator.frameLength)) + .update(from: source.advanced(by: sourceOffset), count: copyFrames) + accumulator.frameLength += AVAudioFrameCount(copyFrames) + sourceOffset += copyFrames + + if accumulator.frameLength == targetAccumulatorLength { + if completedBuffer == nil { + completedBuffer = accumulator + } + bufferAccumulator = nil + } } - return returnBuffer + return completedBuffer } } -nonisolated private func advancedPCMBuffer_noCopy(_ originalBuffer: AVAudioPCMBuffer, offset: UInt32) -> AVAudioPCMBuffer? { +nonisolated private func advancedPCMBufferNoCopy( + _ originalBuffer: AVAudioPCMBuffer, + offset: UInt32) + -> AVAudioPCMBuffer? +{ let audioBufferList = originalBuffer.mutableAudioBufferList guard audioBufferList.pointee.mNumberBuffers == 1, audioBufferList.pointee.mBuffers.mNumberChannels == 1 else { - logger.error("Broken programmer assumption. Audio conversion depends on single channel PCM16 as input") + logger.error("Audio conversion depends on single-channel noninterleaved input") return nil } guard let audioBufferData = audioBufferList.pointee.mBuffers.mData else { @@ -127,8 +151,13 @@ nonisolated private func advancedPCMBuffer_noCopy(_ originalBuffer: AVAudioPCMBu return nil } // advanced(by:) is O(1) + let bytesPerFrame = Int(originalBuffer.format.streamDescription.pointee.mBytesPerFrame) + guard bytesPerFrame > 0 else { + logger.error("Could not determine input audio bytes per frame") + return nil + } audioBufferList.pointee.mBuffers.mData = audioBufferData.advanced( - by: Int(offset) * MemoryLayout.size) + by: Int(offset) * bytesPerFrame) return AVAudioPCMBuffer( pcmFormat: originalBuffer.format, bufferListNoCopy: audioBufferList) diff --git a/Sources/OpenAI/Private/Networking/OpenAIAPI.swift b/Sources/OpenAI/Private/Networking/OpenAIAPI.swift index 5b3eb9ca..753c9075 100644 --- a/Sources/OpenAI/Private/Networking/OpenAIAPI.swift +++ b/Sources/OpenAI/Private/Networking/OpenAIAPI.swift @@ -20,6 +20,7 @@ enum OpenAIAPI { case message(MessageCategory) // https://platform.openai.com/docs/api-reference/messages case model(ModelCategory) // https://platform.openai.com/docs/api-reference/models case moderations // https://platform.openai.com/docs/api-reference/moderations + case realtime(RealtimeCategory) // https://platform.openai.com/docs/api-reference/realtime case run(RunCategory) // https://platform.openai.com/docs/api-reference/runs case runStep(RunStepCategory) // https://platform.openai.com/docs/api-reference/runs/step-object case thread(ThreadCategory) // https://platform.openai.com/docs/api-reference/threads @@ -95,6 +96,10 @@ enum OpenAIAPI { case createThreadAndRun } + enum RealtimeCategory { + case clientSecrets + } + enum RunStepCategory { case retrieve(threadID: String, runID: String, stepID: String) case list(threadID: String, runID: String) @@ -238,6 +243,11 @@ extension OpenAIAPI: Endpoint { case .moderations: return "\(version)/moderations" + case .realtime(let category): + switch category { + case .clientSecrets: return "\(version)/realtime/client_secrets" + } + case .run(let category): switch category { case .create(let threadID), .list(let threadID): return "\(version)/threads/\(threadID)/runs" diff --git a/Sources/OpenAI/Private/Realtime/OpenAIRealtimeSession.swift b/Sources/OpenAI/Private/Realtime/OpenAIRealtimeSession.swift index 0435f6ad..76b14840 100644 --- a/Sources/OpenAI/Private/Realtime/OpenAIRealtimeSession.swift +++ b/Sources/OpenAI/Private/Realtime/OpenAIRealtimeSession.swift @@ -22,8 +22,11 @@ open class OpenAIRealtimeSession { webSocketTask: URLSessionWebSocketTask, sessionConfiguration: OpenAIRealtimeSessionConfiguration) { + let (receiver, continuation) = AsyncStream.makeStream() self.webSocketTask = webSocketTask self.sessionConfiguration = sessionConfiguration + self.receiver = receiver + self.continuation = continuation Task { @RealtimeActor in await self.sendMessage(OpenAIRealtimeSessionUpdate(session: self.sessionConfiguration)) @@ -36,12 +39,9 @@ open class OpenAIRealtimeSession { logger.debug("OpenAIRealtimeSession is being freed") } - /// Messages sent from OpenAI are published on this receiver as they arrive - public var receiver: AsyncStream { - AsyncStream { continuation in - self.continuation = continuation - } - } + /// Messages sent from OpenAI, buffered from the moment the session is created. + /// Consume this stream from one task for the lifetime of the session. + public nonisolated let receiver: AsyncStream /// Sends a message through the websocket connection public func sendMessage(_ encodable: Encodable) async { @@ -67,8 +67,7 @@ open class OpenAIRealtimeSession { /// Close the websocket connection public func disconnect() { isTearingDown = true - continuation?.finish() - continuation = nil + continuation.finish() webSocketTask.cancel() } @@ -76,22 +75,26 @@ open class OpenAIRealtimeSession { private var isTearingDown = false private let webSocketTask: URLSessionWebSocketTask - private var continuation: AsyncStream.Continuation? - private let setupTime = Date() + private nonisolated let continuation: AsyncStream.Continuation + private let setupTime = Date.now private let logger = Logger(subsystem: "com.swiftopenai", category: "Realtime") + nonisolated private static func jsonValues(from dictionary: [String: Any]) -> [String: OpenAIJSONValue] { + dictionary.compactMapValues(OpenAIJSONValue.init(jsonObject:)) + } + /// Tells the websocket task to receive a new message nonisolated private func receiveMessage() { webSocketTask.receive { result in switch result { case .failure(let error as NSError): Task { @RealtimeActor in - await self.didReceiveWebSocketError(error) + self.didReceiveWebSocketError(error) } case .success(let message): Task { @RealtimeActor in - await self.didReceiveWebSocketMessage(message) + self.didReceiveWebSocketMessage(message) } } } @@ -116,6 +119,7 @@ open class OpenAIRealtimeSession { logger.error("Received ws error: \(error.localizedDescription)") } + continuation.yield(.disconnected(error.localizedDescription)) disconnect() } @@ -148,33 +152,54 @@ open class OpenAIRealtimeSession { let messageType = json["type"] as? String else { logger.error("Received websocket data that we don't understand") + continuation.yield(.error("Received an invalid Realtime server event.")) disconnect() return } - logger.debug("Received \(messageType) from OpenAI") + switch messageType { + case "response.output_audio.delta", "response.audio.delta", + "response.output_audio_transcript.delta", "response.audio_transcript.delta", + "conversation.item.input_audio_transcription.delta", + "response.function_call_arguments.delta": + break + default: + logger.debug("Received \(messageType) from OpenAI") + } switch messageType { case "error": - let errorBody = String(describing: json["error"] as? [String: Any]) - logger.error("Received error from OpenAI websocket: \(errorBody)") - continuation?.yield(.error(errorBody)) + let error = json["error"] as? [String: Any] + let errorMessage = error?["message"] as? String ?? "Unknown Realtime error" + logger.error("Received error from OpenAI websocket: \(errorMessage)") + continuation.yield(.error(errorMessage)) case "session.created": - continuation?.yield(.sessionCreated) + continuation.yield(.sessionCreated) case "session.updated": - continuation?.yield(.sessionUpdated) + continuation.yield(.sessionUpdated) - case "response.audio.delta": + case "response.output_audio.delta", "response.audio.delta": if let base64Audio = json["delta"] as? String { - continuation?.yield(.responseAudioDelta(base64Audio)) + continuation.yield(.responseAudioDelta( + itemID: json["item_id"] as? String, + responseID: json["response_id"] as? String, + delta: base64Audio)) } + case "response.output_audio.done", "response.audio.done": + continuation.yield(.responseAudioDone( + itemID: json["item_id"] as? String, + responseID: json["response_id"] as? String)) + case "response.created": - continuation?.yield(.responseCreated) + let response = json["response"] as? [String: Any] + continuation.yield(.responseCreated(responseID: response?["id"] as? String)) case "input_audio_buffer.speech_started": - continuation?.yield(.inputAudioBufferSpeechStarted) + continuation.yield(.inputAudioBufferSpeechStarted( + itemID: json["item_id"] as? String, + audioStartMS: json["audio_start_ms"] as? Int)) case "response.function_call_arguments.done": if @@ -182,46 +207,73 @@ open class OpenAIRealtimeSession { let arguments = json["arguments"] as? String, let callId = json["call_id"] as? String { - continuation?.yield(.responseFunctionCallArgumentsDone(name, arguments, callId)) + continuation.yield(.responseFunctionCallArgumentsDone( + name: name, + arguments: arguments, + callID: callId, + itemID: json["item_id"] as? String, + responseID: json["response_id"] as? String)) + } + + case "response.function_call_arguments.delta": + if let delta = json["delta"] as? String { + continuation.yield(.responseFunctionCallArgumentsDelta( + delta: delta, + callID: json["call_id"] as? String, + itemID: json["item_id"] as? String, + responseID: json["response_id"] as? String)) } // New cases for handling transcription messages - case "response.audio_transcript.delta": + case "response.output_audio_transcript.delta", "response.audio_transcript.delta": if let delta = json["delta"] as? String { - continuation?.yield(.responseTranscriptDelta(delta)) + continuation.yield(.responseTranscriptDelta( + itemID: json["item_id"] as? String, + responseID: json["response_id"] as? String, + delta: delta)) } - case "response.audio_transcript.done": + case "response.output_audio_transcript.done", "response.audio_transcript.done": if let transcript = json["transcript"] as? String { - continuation?.yield(.responseTranscriptDone(transcript)) + continuation.yield(.responseTranscriptDone( + itemID: json["item_id"] as? String, + responseID: json["response_id"] as? String, + transcript: transcript)) } + case "input_audio_buffer.speech_stopped", "input_audio_buffer.committed", "conversation.item.truncated": + break + case "input_audio_buffer.transcript": if let transcript = json["transcript"] as? String { - continuation?.yield(.inputAudioBufferTranscript(transcript)) + continuation.yield(.inputAudioBufferTranscript(transcript)) } case "conversation.item.input_audio_transcription.delta": if let delta = json["delta"] as? String { - continuation?.yield(.inputAudioTranscriptionDelta(delta)) + continuation.yield(.inputAudioTranscriptionDelta( + itemID: json["item_id"] as? String, + delta: delta)) } case "conversation.item.input_audio_transcription.completed": if let transcript = json["transcript"] as? String { - continuation?.yield(.inputAudioTranscriptionCompleted(transcript)) + continuation.yield(.inputAudioTranscriptionCompleted( + itemID: json["item_id"] as? String, + transcript: transcript)) } // MCP (Model Context Protocol) message types case "mcp_list_tools.in_progress": logger.debug("MCP: Tool discovery in progress") - continuation?.yield(.mcpListToolsInProgress) + continuation.yield(.mcpListToolsInProgress) case "mcp_list_tools.completed": logger.debug("MCP: Tool discovery completed") if let tools = json["tools"] as? [String: Any] { - continuation?.yield(.mcpListToolsCompleted(tools)) + continuation.yield(.mcpListToolsCompleted(Self.jsonValues(from: tools))) } else { - continuation?.yield(.mcpListToolsCompleted([:])) + continuation.yield(.mcpListToolsCompleted([:])) } case "mcp_list_tools.failed": @@ -247,16 +299,25 @@ open class OpenAIRealtimeSession { .error( "Top-level fields: message=\(String(describing: topLevelMessage)), code=\(String(describing: topLevelCode)), reason=\(String(describing: topLevelReason))") - continuation?.yield(.mcpListToolsFailed(fullError)) + continuation.yield(.mcpListToolsFailed(fullError)) case "response.mcp_call.completed": let eventId = json["event_id"] as? String let itemId = json["item_id"] as? String let outputIndex = json["output_index"] as? Int - continuation?.yield(.responseMcpCallCompleted(eventId: eventId, itemId: itemId, outputIndex: outputIndex)) + continuation.yield(.responseMcpCallCompleted(eventId: eventId, itemId: itemId, outputIndex: outputIndex)) case "response.mcp_call.in_progress": - continuation?.yield(.responseMcpCallInProgress) + continuation.yield(.responseMcpCallInProgress) + + case "response.mcp_call_arguments.done": + if let arguments = json["arguments"] as? String { + continuation.yield(.responseMcpCallArgumentsDone( + arguments: arguments, + itemId: json["item_id"] as? String, + outputIndex: json["output_index"] as? Int, + responseId: json["response_id"] as? String)) + } case "response.done": // Handle response completion (may contain errors like insufficient_quota) @@ -267,7 +328,10 @@ open class OpenAIRealtimeSession { logger.debug("Response done with status: \(status)") // Pass the full response object for detailed error handling - continuation?.yield(.responseDone(status: status, statusDetails: response)) + continuation.yield(.responseDone( + responseID: response["id"] as? String, + status: status, + statusDetails: Self.jsonValues(from: response))) // Log errors for debugging if @@ -282,14 +346,20 @@ open class OpenAIRealtimeSession { logger.warning("Received response.done with unexpected format") } - case "response.text.delta": + case "response.output_text.delta", "response.text.delta": if let delta = json["delta"] as? String { - continuation?.yield(.responseTextDelta(delta)) + continuation.yield(.responseTextDelta( + itemID: json["item_id"] as? String, + responseID: json["response_id"] as? String, + delta: delta)) } - case "response.text.done": + case "response.output_text.done", "response.text.done": if let text = json["text"] as? String { - continuation?.yield(.responseTextDone(text)) + continuation.yield(.responseTextDone( + itemID: json["item_id"] as? String, + responseID: json["response_id"] as? String, + text: text)) } case "response.output_item.added": @@ -298,7 +368,7 @@ open class OpenAIRealtimeSession { let itemId = item["id"] as? String, let type = item["type"] as? String { - continuation?.yield(.responseOutputItemAdded(itemId: itemId, type: type)) + continuation.yield(.responseOutputItemAdded(itemId: itemId, type: type)) } case "response.output_item.done": @@ -307,8 +377,8 @@ open class OpenAIRealtimeSession { let itemId = item["id"] as? String, let type = item["type"] as? String { - let content = item["content"] as? [[String: Any]] - continuation?.yield(.responseOutputItemDone(itemId: itemId, type: type, content: content)) + let content = (item["content"] as? [[String: Any]])?.map(Self.jsonValues(from:)) + continuation.yield(.responseOutputItemDone(itemId: itemId, type: type, content: content)) } case "response.content_part.added": @@ -316,7 +386,7 @@ open class OpenAIRealtimeSession { let part = json["part"] as? [String: Any], let type = part["type"] as? String { - continuation?.yield(.responseContentPartAdded(type: type)) + continuation.yield(.responseContentPartAdded(type: type)) } case "response.content_part.done": @@ -325,19 +395,54 @@ open class OpenAIRealtimeSession { let type = part["type"] as? String { let text = part["text"] as? String - continuation?.yield(.responseContentPartDone(type: type, text: text)) + continuation.yield(.responseContentPartDone(type: type, text: text)) } - case "conversation.item.created": + case "conversation.item.added", "conversation.item.created": if let item = json["item"] as? [String: Any], let itemId = item["id"] as? String, let type = item["type"] as? String { let role = item["role"] as? String - continuation?.yield(.conversationItemCreated(itemId: itemId, type: type, role: role)) + continuation.yield(.conversationItemCreated( + itemID: itemId, + type: type, + role: role, + previousItemID: json["previous_item_id"] as? String)) + + if + type == "mcp_approval_request", + let name = item["name"] as? String, + let arguments = item["arguments"] as? String, + let serverLabel = item["server_label"] as? String + { + continuation.yield(.mcpApprovalRequest( + id: itemId, + name: name, + arguments: arguments, + serverLabel: serverLabel)) + } } + case "conversation.item.done": + if + let item = json["item"] as? [String: Any], + let itemID = item["id"] as? String, + let type = item["type"] as? String + { + let content = (item["content"] as? [[String: Any]])?.map(Self.jsonValues(from:)) + continuation.yield(.conversationItemDone( + itemID: itemID, + type: type, + role: item["role"] as? String, + previousItemID: json["previous_item_id"] as? String, + content: content)) + } + + case "rate_limits.updated": + continuation.yield(.rateLimitsUpdated) + default: // Log unhandled message types with more detail for debugging logger.warning("⚠️ Unhandled message type: \(messageType)") @@ -345,9 +450,10 @@ open class OpenAIRealtimeSession { break } - if messageType != "error", !isTearingDown { + if !isTearingDown { receiveMessage() } } + } #endif diff --git a/Sources/OpenAI/Public/Parameters/Model.swift b/Sources/OpenAI/Public/Parameters/Model.swift index 38c93141..45e05cbb 100644 --- a/Sources/OpenAI/Public/Parameters/Model.swift +++ b/Sources/OpenAI/Public/Parameters/Model.swift @@ -71,6 +71,23 @@ public enum Model { case gpt5Codex + /// Latest GA Realtime voice-agent model for low-latency speech-to-speech conversations. + case gptRealtime21 + /// Smaller, lower-cost GPT-Realtime 2.1 variant. + case gptRealtime21Mini + /// GPT-Realtime 2 reasoning voice model. + case gptRealtime2 + /// GPT-Realtime 1.5 flagship audio model. + case gptRealtime15 + /// GPT-Realtime alias. + case gptRealtime + /// Cost-efficient GPT-Realtime alias. + case gptRealtimeMini + /// Dedicated Realtime live translation model. + case gptRealtimeTranslate + /// Dedicated Realtime live transcription model. + case gptRealtimeWhisper + /// Images case dalle2 case dalle3 @@ -106,6 +123,14 @@ public enum Model { case .gpt5Mini: "gpt-5-mini" case .gpt5Nano: "gpt-5-nano" case .gpt5Codex: "gpt-5-codex" + case .gptRealtime21: "gpt-realtime-2.1" + case .gptRealtime21Mini: "gpt-realtime-2.1-mini" + case .gptRealtime2: "gpt-realtime-2" + case .gptRealtime15: "gpt-realtime-1.5" + case .gptRealtime: "gpt-realtime" + case .gptRealtimeMini: "gpt-realtime-mini" + case .gptRealtimeTranslate: "gpt-realtime-translate" + case .gptRealtimeWhisper: "gpt-realtime-whisper" case .custom(let model): model } } diff --git a/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeClientSecretParameters.swift b/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeClientSecretParameters.swift new file mode 100644 index 00000000..27a7b25f --- /dev/null +++ b/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeClientSecretParameters.swift @@ -0,0 +1,48 @@ +// +// OpenAIRealtimeClientSecretParameters.swift +// SwiftOpenAI +// + +import Foundation + +// MARK: - OpenAIRealtimeClientSecretParameters + +/// Parameters for creating a Realtime client secret. +public struct OpenAIRealtimeClientSecretParameters: Encodable, Sendable { + public init( + expiresAfter: ExpiresAfter? = nil, + session: OpenAIRealtimeSessionConfiguration? = nil) + { + self.expiresAfter = expiresAfter + self.session = session + } + + /// Configuration for when the generated client secret expires. + public let expiresAfter: ExpiresAfter? + + /// Session configuration to bind to the generated client secret. + public let session: OpenAIRealtimeSessionConfiguration? + + private enum CodingKeys: String, CodingKey { + case expiresAfter = "expires_after" + case session + } +} + +// MARK: OpenAIRealtimeClientSecretParameters.ExpiresAfter + +extension OpenAIRealtimeClientSecretParameters { + public struct ExpiresAfter: Encodable, Sendable { + public init(anchor: Anchor = .createdAt, seconds: Int) { + self.anchor = anchor + self.seconds = seconds + } + + public let anchor: Anchor + public let seconds: Int + + public enum Anchor: String, Encodable, Sendable { + case createdAt = "created_at" + } + } +} diff --git a/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeConversationItemCreate.swift b/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeConversationItemCreate.swift index 5f8ef77b..11b4b351 100644 --- a/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeConversationItemCreate.swift +++ b/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeConversationItemCreate.swift @@ -24,16 +24,19 @@ public struct OpenAIRealtimeConversationItemCreate: Encodable { extension OpenAIRealtimeConversationItemCreate { public struct Item: Encodable { + public let id: String? public let type = "message" public let role: String public let content: [Content] - public init(role: String, text: String) { + public init(id: String? = nil, role: String, text: String) { + self.id = id self.role = role content = [.text(text)] } - public init(role: String, content: [Content]) { + public init(id: String? = nil, role: String, content: [Content]) { + self.id = id self.role = role self.content = content } diff --git a/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeConversationItemTruncate.swift b/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeConversationItemTruncate.swift new file mode 100644 index 00000000..8bd86718 --- /dev/null +++ b/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeConversationItemTruncate.swift @@ -0,0 +1,26 @@ +// +// OpenAIRealtimeConversationItemTruncate.swift +// SwiftOpenAI +// + +/// Truncates unheard assistant audio after local playback is interrupted. +/// https://developers.openai.com/api/reference/resources/realtime/client-events/conversation/item/truncate +public struct OpenAIRealtimeConversationItemTruncate: Encodable, Sendable { + public init(itemID: String, audioEndMS: Int, contentIndex: Int = 0) { + self.itemID = itemID + self.audioEndMS = audioEndMS + self.contentIndex = contentIndex + } + + public let type = "conversation.item.truncate" + public let itemID: String + public let audioEndMS: Int + public let contentIndex: Int + + private enum CodingKeys: String, CodingKey { + case type + case itemID = "item_id" + case audioEndMS = "audio_end_ms" + case contentIndex = "content_index" + } +} diff --git a/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeFunctionCallOutput.swift b/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeFunctionCallOutput.swift new file mode 100644 index 00000000..b3aa1f13 --- /dev/null +++ b/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeFunctionCallOutput.swift @@ -0,0 +1,44 @@ +// +// OpenAIRealtimeFunctionCallOutput.swift +// SwiftOpenAI +// + +import Foundation + +// MARK: - OpenAIRealtimeFunctionCallOutput + +/// Sends a function call result back to a Realtime session. +public struct OpenAIRealtimeFunctionCallOutput: Encodable, Sendable { + public init(callID: String, output: String) { + item = .init(callID: callID, output: output) + } + + public let type = "conversation.item.create" + public let item: Item + + private enum CodingKeys: String, CodingKey { + case item + case type + } +} + +// MARK: OpenAIRealtimeFunctionCallOutput.Item + +extension OpenAIRealtimeFunctionCallOutput { + public struct Item: Encodable, Sendable { + public init(callID: String, output: String) { + self.callID = callID + self.output = output + } + + public let type = "function_call_output" + public let callID: String + public let output: String + + private enum CodingKeys: String, CodingKey { + case callID = "call_id" + case output + case type + } + } +} diff --git a/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeMCPApprovalResponse.swift b/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeMCPApprovalResponse.swift new file mode 100644 index 00000000..35267a7f --- /dev/null +++ b/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeMCPApprovalResponse.swift @@ -0,0 +1,59 @@ +// +// OpenAIRealtimeMCPApprovalResponse.swift +// SwiftOpenAI +// + +import Foundation + +// MARK: - OpenAIRealtimeMCPApprovalResponse + +/// Approves or rejects an MCP tool approval request in a Realtime session. +public struct OpenAIRealtimeMCPApprovalResponse: Encodable, Sendable { + public init( + approvalRequestID: String, + approve: Bool, + id: String, + reason: String? = nil) + { + item = .init( + approvalRequestID: approvalRequestID, + approve: approve, + id: id, + reason: reason) + } + + public let type = "conversation.item.create" + public let item: Item +} + +// MARK: OpenAIRealtimeMCPApprovalResponse.Item + +extension OpenAIRealtimeMCPApprovalResponse { + public struct Item: Encodable, Sendable { + public init( + approvalRequestID: String, + approve: Bool, + id: String, + reason: String? = nil) + { + self.approvalRequestID = approvalRequestID + self.approve = approve + self.id = id + self.reason = reason + } + + public let type = "mcp_approval_response" + public let approvalRequestID: String + public let approve: Bool + public let id: String + public let reason: String? + + private enum CodingKeys: String, CodingKey { + case approvalRequestID = "approval_request_id" + case approve + case id + case reason + case type + } + } +} diff --git a/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeResponseCreate.swift b/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeResponseCreate.swift index a9d5bf44..998529cf 100644 --- a/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeResponseCreate.swift +++ b/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeResponseCreate.swift @@ -24,18 +24,67 @@ public struct OpenAIRealtimeResponseCreate: Encodable { extension OpenAIRealtimeResponseCreate { public struct Response: Encodable { - public let instructions: String? - public let modalities: [String]? - public let tools: [Tool]? - public init( instructions: String? = nil, modalities: [String]? = nil, - tools: [Tool]? = nil) + maxResponseOutputTokens: OpenAIRealtimeSessionConfiguration.MaxResponseOutputTokens? = nil, + parallelToolCalls: Bool? = nil, + reasoning: OpenAIRealtimeSessionConfiguration.ReasoningConfiguration? = nil, + tools: [OpenAIRealtimeSessionConfiguration.RealtimeTool]? = nil, + toolChoice: OpenAIRealtimeSessionConfiguration.ToolChoice? = nil) { self.instructions = instructions self.modalities = modalities + self.maxResponseOutputTokens = maxResponseOutputTokens + self.parallelToolCalls = parallelToolCalls + self.reasoning = reasoning self.tools = tools + self.toolChoice = toolChoice + } + + public init( + instructions: String? = nil, + modalities: [String]? = nil, + tools: [Tool]) + { + self.instructions = instructions + self.modalities = modalities + maxResponseOutputTokens = nil + parallelToolCalls = nil + reasoning = nil + self.tools = tools.map { + .function(.init(name: $0.name, description: $0.description, parameters: $0.parameters)) + } + toolChoice = nil + } + + public let instructions: String? + public let modalities: [String]? + public let maxResponseOutputTokens: OpenAIRealtimeSessionConfiguration.MaxResponseOutputTokens? + public let parallelToolCalls: Bool? + public let reasoning: OpenAIRealtimeSessionConfiguration.ReasoningConfiguration? + public let tools: [OpenAIRealtimeSessionConfiguration.RealtimeTool]? + public let toolChoice: OpenAIRealtimeSessionConfiguration.ToolChoice? + + public func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encodeIfPresent(instructions, forKey: .instructions) + try container.encodeIfPresent(modalities, forKey: .outputModalities) + try container.encodeIfPresent(maxResponseOutputTokens, forKey: .maxOutputTokens) + try container.encodeIfPresent(parallelToolCalls, forKey: .parallelToolCalls) + try container.encodeIfPresent(reasoning, forKey: .reasoning) + try container.encodeIfPresent(tools, forKey: .tools) + try container.encodeIfPresent(toolChoice, forKey: .toolChoice) + } + + private enum CodingKeys: String, CodingKey { + case instructions + case maxOutputTokens = "max_output_tokens" + case outputModalities = "output_modalities" + case parallelToolCalls = "parallel_tool_calls" + case reasoning + case tools + case toolChoice = "tool_choice" } } } diff --git a/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeSessionConfiguration.swift b/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeSessionConfiguration.swift index 6247bf12..9cd38351 100644 --- a/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeSessionConfiguration.swift +++ b/Sources/OpenAI/Public/Parameters/Realtime/OpenAIRealtimeSessionConfiguration.swift @@ -14,10 +14,14 @@ public struct OpenAIRealtimeSessionConfiguration: Encodable, Sendable { public init( inputAudioFormat: OpenAIRealtimeSessionConfiguration.AudioFormat? = nil, inputAudioTranscription: OpenAIRealtimeSessionConfiguration.InputAudioTranscription? = nil, + include: [String]? = nil, instructions: String? = nil, maxResponseOutputTokens: OpenAIRealtimeSessionConfiguration.MaxResponseOutputTokens? = nil, modalities: [OpenAIRealtimeSessionConfiguration.Modality]? = nil, + model: String? = nil, outputAudioFormat: OpenAIRealtimeSessionConfiguration.AudioFormat? = nil, + parallelToolCalls: Bool? = nil, + reasoning: OpenAIRealtimeSessionConfiguration.ReasoningConfiguration? = nil, speed: Float? = 1.0, temperature: Double? = nil, tools: [OpenAIRealtimeSessionConfiguration.RealtimeTool]? = nil, @@ -27,10 +31,14 @@ public struct OpenAIRealtimeSessionConfiguration: Encodable, Sendable { { self.inputAudioFormat = inputAudioFormat self.inputAudioTranscription = inputAudioTranscription + self.include = include self.instructions = instructions self.maxResponseOutputTokens = maxResponseOutputTokens self.modalities = modalities + self.model = model self.outputAudioFormat = outputAudioFormat + self.parallelToolCalls = parallelToolCalls + self.reasoning = reasoning self.speed = speed self.temperature = temperature self.tools = tools @@ -81,12 +89,15 @@ public struct OpenAIRealtimeSessionConfiguration: Encodable, Sendable { } } - /// The format of input audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`. + /// The format of input audio. Options are `.pcm16`, `.g711Ulaw`, or `.g711Alaw`. public let inputAudioFormat: AudioFormat? /// Configuration for input audio transcription. Set to nil to turn off. public let inputAudioTranscription: InputAudioTranscription? + /// Additional fields to include in server outputs. + public let include: [String]? + /// The default system instructions prepended to model calls. /// /// OpenAI recommends the following instructions: @@ -106,18 +117,30 @@ public struct OpenAIRealtimeSessionConfiguration: Encodable, Sendable { /// the maximum available tokens for a given model. Defaults to "inf". public let maxResponseOutputTokens: MaxResponseOutputTokens? - /// The set of modalities the model can respond with. To disable audio, set this to ["text"]. - /// Possible values are `audio` and `text` + /// The set of output modalities the model can respond with. To disable audio, set this to `[.text]`. + /// Realtime GA accepts one output mode per response: `.audio` or `.text`. public let modalities: [Modality]? + /// The Realtime model used for client-secret session creation. + /// WebSocket sessions set the model on the URL, so this can be nil for `realtimeSession`. + public let model: String? + /// The format of output audio. public let outputAudioFormat: AudioFormat? + /// Whether the model may call multiple tools in parallel. + public let parallelToolCalls: Bool? + + /// Reasoning configuration for Realtime reasoning models such as `gpt-realtime-2.1`. + public let reasoning: ReasoningConfiguration? + /// The speed of the generated audio. Select a value from 0.25 to 4.0. /// Default to `1.0` public let speed: Float? - /// Sampling temperature for the model. + /// Sampling temperature for the beta Realtime API. + /// + /// The GA Realtime API no longer documents this field, so SwiftOpenAI does not encode it. public let temperature: Double? /// Tools (functions and MCP servers) available to the model. @@ -133,19 +156,45 @@ public struct OpenAIRealtimeSessionConfiguration: Encodable, Sendable { /// changed once the model has responded with audio at least once. public let voice: String? + public func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + + try container.encode("realtime", forKey: .type) + try container.encodeIfPresent(include, forKey: .include) + try container.encodeIfPresent(instructions, forKey: .instructions) + try container.encodeIfPresent(maxResponseOutputTokens, forKey: .maxOutputTokens) + try container.encodeIfPresent(model, forKey: .model) + try container.encodeIfPresent(modalities, forKey: .outputModalities) + try container.encodeIfPresent(parallelToolCalls, forKey: .parallelToolCalls) + try container.encodeIfPresent(reasoning, forKey: .reasoning) + try container.encodeIfPresent(tools, forKey: .tools) + try container.encodeIfPresent(toolChoice, forKey: .toolChoice) + + let input = AudioInput( + format: inputAudioFormat, + transcription: inputAudioTranscription, + turnDetection: turnDetection) + let output = AudioOutput( + format: outputAudioFormat, + speed: speed, + voice: voice) + if input.hasValues || output.hasValues { + try container.encode(Audio(input: input.hasValues ? input : nil, output: output.hasValues ? output : nil), forKey: .audio) + } + } + private enum CodingKeys: String, CodingKey { - case inputAudioFormat = "input_audio_format" - case inputAudioTranscription = "input_audio_transcription" + case audio + case include case instructions - case maxResponseOutputTokens = "max_response_output_tokens" - case modalities - case outputAudioFormat = "output_audio_format" - case speed - case temperature + case maxOutputTokens = "max_output_tokens" + case model + case outputModalities = "output_modalities" + case parallelToolCalls = "parallel_tool_calls" + case reasoning case tools case toolChoice = "tool_choice" - case turnDetection = "turn_detection" - case voice + case type } } @@ -153,17 +202,39 @@ public struct OpenAIRealtimeSessionConfiguration: Encodable, Sendable { extension OpenAIRealtimeSessionConfiguration { public struct InputAudioTranscription: Encodable, Sendable { + public init( + model: String, + delay: Delay? = nil, + language: String? = nil, + prompt: String? = nil) + { + self.model = model + self.delay = delay + self.language = language + self.prompt = prompt + } + + public enum Delay: String, Encodable, Sendable { + case minimal + case low + case medium + case high + case xhigh + } + /// The model to use for transcription (e.g., "whisper-1"). public let model: String + /// Controls how long the model waits before emitting transcription text. + public let delay: Delay? + /// The language of the input audio in ISO-639-1 format (e.g., "en", "es", "ja"). /// Supplying the input language improves transcription accuracy and latency. public let language: String? - public init(model: String, language: String? = nil) { - self.model = model - self.language = language - } + /// Optional text to guide the transcription model. + public let prompt: String? + } } @@ -186,6 +257,27 @@ extension OpenAIRealtimeSessionConfiguration { } } +// MARK: OpenAIRealtimeSessionConfiguration.ReasoningConfiguration + +extension OpenAIRealtimeSessionConfiguration { + public struct ReasoningConfiguration: Encodable, Sendable { + public init(effort: Effort? = nil) { + self.effort = effort + } + + /// Constrains reasoning effort for reasoning-capable Realtime models. + public let effort: Effort? + + public enum Effort: String, Encodable, Sendable { + case minimal + case low + case medium + case high + case xhigh + } + } +} + // MARK: OpenAIRealtimeSessionConfiguration.FunctionTool extension OpenAIRealtimeSessionConfiguration { @@ -243,15 +335,26 @@ extension OpenAIRealtimeSessionConfiguration { var container = encoder.container(keyedBy: CodingKeys.self) switch type { - case .serverVAD(let prefixPaddingMs, let silenceDurationMs, let threshold): + case .serverVAD( + let prefixPaddingMs, + let silenceDurationMs, + let threshold, + let createResponse, + let idleTimeoutMs, + let interruptResponse): try container.encode("server_vad", forKey: .type) try container.encode(prefixPaddingMs, forKey: .prefixPaddingMs) try container.encode(silenceDurationMs, forKey: .silenceDurationMs) try container.encode(threshold, forKey: .threshold) + try container.encodeIfPresent(createResponse, forKey: .createResponse) + try container.encodeIfPresent(idleTimeoutMs, forKey: .idleTimeoutMs) + try container.encodeIfPresent(interruptResponse, forKey: .interruptResponse) - case .semanticVAD(let eagerness): + case .semanticVAD(let eagerness, let createResponse, let interruptResponse): try container.encode("semantic_vad", forKey: .type) - try container.encode(String(describing: eagerness), forKey: .eagerness) + try container.encode(eagerness.rawValue, forKey: .eagerness) + try container.encodeIfPresent(createResponse, forKey: .createResponse) + try container.encodeIfPresent(interruptResponse, forKey: .interruptResponse) } } @@ -263,24 +366,82 @@ extension OpenAIRealtimeSessionConfiguration { case threshold case type case eagerness + case createResponse = "create_response" + case idleTimeoutMs = "idle_timeout_ms" + case interruptResponse = "interrupt_response" + } + } +} + +// MARK: OpenAIRealtimeSessionConfiguration.Audio + +extension OpenAIRealtimeSessionConfiguration { + private struct Audio: Encodable { + let input: AudioInput? + let output: AudioOutput? + } + + private struct AudioInput: Encodable { + let format: AudioFormat? + let transcription: InputAudioTranscription? + let turnDetection: TurnDetection? + + var hasValues: Bool { + format != nil || transcription != nil || turnDetection != nil + } + + private enum CodingKeys: String, CodingKey { + case format + case transcription + case turnDetection = "turn_detection" + } + } + + private struct AudioOutput: Encodable { + let format: AudioFormat? + let speed: Float? + let voice: String? + + var hasValues: Bool { + format != nil || speed != nil || voice != nil } } } // MARK: OpenAIRealtimeSessionConfiguration.AudioFormat -/// The format of input audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`. +/// The format of input audio. Options are `.pcm16`, `.g711Ulaw`, or `.g711Alaw`. extension OpenAIRealtimeSessionConfiguration { public enum AudioFormat: String, Encodable, Sendable { case pcm16 case g711Ulaw = "g711_ulaw" case g711Alaw = "g711_alaw" + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .pcm16: + try container.encode("audio/pcm", forKey: .type) + try container.encode(24_000, forKey: .rate) + + case .g711Ulaw: + try container.encode("audio/pcmu", forKey: .type) + + case .g711Alaw: + try container.encode("audio/pcma", forKey: .type) + } + } + + private enum CodingKeys: String, CodingKey { + case rate + case type + } } } // MARK: OpenAIRealtimeSessionConfiguration.Modality -/// The format of input audio. Options are `pcm16`, `g711_ulaw`, or `g711_alaw`. +/// Realtime output modalities. extension OpenAIRealtimeSessionConfiguration { public enum Modality: String, Encodable, Sendable { case audio @@ -301,18 +462,28 @@ extension OpenAIRealtimeSessionConfiguration.TurnDetection { /// - threshold: Activation threshold for VAD (0.0 to 1.0). A higher threshold will require louder audio to /// activate the model, and thus might perform better in noisy environments. /// OpenAI's default is 0.5 - case serverVAD(prefixPaddingMs: Int, silenceDurationMs: Int, threshold: Double) + case serverVAD( + prefixPaddingMs: Int, + silenceDurationMs: Int, + threshold: Double, + createResponse: Bool? = nil, + idleTimeoutMs: Int? = nil, + interruptResponse: Bool? = nil) /// - Parameters: /// - eagerness: The eagerness of the model to respond. `low` will wait longer for the user to /// continue speaking, `high` will respond more quickly. /// OpenAI's default is medium - case semanticVAD(eagerness: Eagerness) + case semanticVAD( + eagerness: Eagerness, + createResponse: Bool? = nil, + interruptResponse: Bool? = nil) public enum Eagerness: String, Encodable, Sendable { case low case medium case high + case auto } } } diff --git a/Sources/OpenAI/Public/ResponseModels/Realtime/OpenAIRealtimeClientSecret.swift b/Sources/OpenAI/Public/ResponseModels/Realtime/OpenAIRealtimeClientSecret.swift new file mode 100644 index 00000000..1c7e03b8 --- /dev/null +++ b/Sources/OpenAI/Public/ResponseModels/Realtime/OpenAIRealtimeClientSecret.swift @@ -0,0 +1,25 @@ +// +// OpenAIRealtimeClientSecret.swift +// SwiftOpenAI +// + +import Foundation + +// MARK: - OpenAIRealtimeClientSecret + +public struct OpenAIRealtimeClientSecret: Decodable, Sendable { + /// The generated ephemeral credential value. + public let value: String + + /// Expiration timestamp in seconds since epoch. + public let expiresAt: Int? + + /// The created session object returned by OpenAI. + public let session: OpenAIJSONValue? + + private enum CodingKeys: String, CodingKey { + case value + case expiresAt = "expires_at" + case session + } +} diff --git a/Sources/OpenAI/Public/ResponseModels/Realtime/OpenAIRealtimeMessage.swift b/Sources/OpenAI/Public/ResponseModels/Realtime/OpenAIRealtimeMessage.swift index e46f757e..48d956f8 100644 --- a/Sources/OpenAI/Public/ResponseModels/Realtime/OpenAIRealtimeMessage.swift +++ b/Sources/OpenAI/Public/ResponseModels/Realtime/OpenAIRealtimeMessage.swift @@ -8,34 +8,48 @@ public enum OpenAIRealtimeMessage: Sendable { case error(String?) + case disconnected(String?) case sessionCreated // "session.created" case sessionUpdated // "session.updated" - case responseCreated // "response.created" - case responseAudioDelta(String) // "response.audio.delta" - case inputAudioBufferSpeechStarted // "input_audio_buffer.speech_started" - case responseFunctionCallArgumentsDone(String, String, String) // "response.function_call_arguments.done" + case responseCreated(responseID: String?) // "response.created" + case responseAudioDelta(itemID: String?, responseID: String?, delta: String) // "response.output_audio.delta" + case responseAudioDone(itemID: String?, responseID: String?) // "response.output_audio.done" + case inputAudioBufferSpeechStarted(itemID: String?, audioStartMS: Int?) // "input_audio_buffer.speech_started" + case responseFunctionCallArgumentsDelta( + delta: String, + callID: String?, + itemID: String?, + responseID: String?) // "response.function_call_arguments.delta" + case responseFunctionCallArgumentsDone( + name: String, + arguments: String, + callID: String, + itemID: String?, + responseID: String?) // "response.function_call_arguments.done" // Add new cases for transcription - case responseTranscriptDelta(String) // "response.audio_transcript.delta" - case responseTranscriptDone(String) // "response.audio_transcript.done" + case responseTranscriptDelta(itemID: String?, responseID: String?, delta: String) // "response.output_audio_transcript.delta" + case responseTranscriptDone(itemID: String?, responseID: String?, transcript: String) // "response.output_audio_transcript.done" case inputAudioBufferTranscript(String) // "input_audio_buffer.transcript" - case inputAudioTranscriptionDelta(String) // "conversation.item.input_audio_transcription.delta" - case inputAudioTranscriptionCompleted(String) // "conversation.item.input_audio_transcription.completed" + case inputAudioTranscriptionDelta(itemID: String?, delta: String) // "conversation.item.input_audio_transcription.delta" + case inputAudioTranscriptionCompleted( + itemID: String?, + transcript: String) // "conversation.item.input_audio_transcription.completed" // MCP (Model Context Protocol) messages case mcpListToolsInProgress // "mcp_list_tools.in_progress" - case mcpListToolsCompleted([String: Any]) // "mcp_list_tools.completed" with tools data + case mcpListToolsCompleted([String: OpenAIJSONValue]) // "mcp_list_tools.completed" with tools data case mcpListToolsFailed(String?) // "mcp_list_tools.failed" with error details /// Response completion with potential errors - case responseDone(status: String, statusDetails: [String: Any]?) // "response.done" + case responseDone(responseID: String?, status: String, statusDetails: [String: OpenAIJSONValue]?) // "response.done" // Text streaming (for text-only responses) - case responseTextDelta(String) // "response.text.delta" - case responseTextDone(String) // "response.text.done" + case responseTextDelta(itemID: String?, responseID: String?, delta: String) // "response.output_text.delta" + case responseTextDone(itemID: String?, responseID: String?, text: String) // "response.output_text.done" // Output item lifecycle case responseOutputItemAdded(itemId: String, type: String) // "response.output_item.added" - case responseOutputItemDone(itemId: String, type: String, content: [[String: Any]]?) // "response.output_item.done" + case responseOutputItemDone(itemId: String, type: String, content: [[String: OpenAIJSONValue]]?) // "response.output_item.done" // Content part lifecycle case responseContentPartAdded(type: String) // "response.content_part.added" @@ -44,7 +58,20 @@ public enum OpenAIRealtimeMessage: Sendable { // MCP response case responseMcpCallCompleted(eventId: String?, itemId: String?, outputIndex: Int?) case responseMcpCallInProgress + case responseMcpCallArgumentsDone(arguments: String, itemId: String?, outputIndex: Int?, responseId: String?) + case mcpApprovalRequest(id: String, name: String, arguments: String, serverLabel: String) /// Conversation item - case conversationItemCreated(itemId: String, type: String, role: String?) // "conversation.item.created" + case conversationItemCreated( + itemID: String, + type: String, + role: String?, + previousItemID: String?) // "conversation.item.created" + case conversationItemDone( + itemID: String, + type: String, + role: String?, + previousItemID: String?, + content: [[String: OpenAIJSONValue]]?) // "conversation.item.done" + case rateLimitsUpdated // "rate_limits.updated" } diff --git a/Sources/OpenAI/Public/Service/DefaultOpenAIService.swift b/Sources/OpenAI/Public/Service/DefaultOpenAIService.swift index f2c3f053..659eb9e7 100644 --- a/Sources/OpenAI/Public/Service/DefaultOpenAIService.swift +++ b/Sources/OpenAI/Public/Service/DefaultOpenAIService.swift @@ -78,6 +78,20 @@ struct DefaultOpenAIService: OpenAIService { return AudioSpeechObject(output: data) } + func createRealtimeClientSecret( + parameters: OpenAIRealtimeClientSecretParameters) + async throws -> OpenAIRealtimeClientSecret + { + let request = try OpenAIAPI.realtime(.clientSecrets).request( + apiKey: apiKey, + openAIEnvironment: openAIEnvironment, + organizationID: organizationID, + method: .post, + params: parameters, + extraHeaders: extraHeaders) + return try await fetch(debugEnabled: debugEnabled, type: OpenAIRealtimeClientSecret.self, with: request) + } + #if canImport(AVFoundation) func realtimeSession( model: String, @@ -114,11 +128,6 @@ struct DefaultOpenAIService: OpenAIService { var request = URLRequest(url: url) request.setValue(apiKey.value, forHTTPHeaderField: apiKey.headerField) - // Only add openai-beta header for non-Azure endpoints - if !isAzureEndpoint { - request.setValue("realtime=v1", forHTTPHeaderField: "openai-beta") - } - if let organizationID { request.setValue(organizationID, forHTTPHeaderField: "OpenAI-Organization") } diff --git a/Sources/OpenAI/Public/Service/OpenAIService.swift b/Sources/OpenAI/Public/Service/OpenAIService.swift index a0025ae3..a0fdc6a3 100644 --- a/Sources/OpenAI/Public/Service/OpenAIService.swift +++ b/Sources/OpenAI/Public/Service/OpenAIService.swift @@ -126,10 +126,17 @@ public protocol OpenAIService { parameters: AudioSpeechParameters) async throws -> AudioSpeechObject + /// Creates an ephemeral client secret for a Realtime session. + /// + /// Use this when a browser or mobile WebRTC layer should connect without exposing a standard OpenAI API key. + func createRealtimeClientSecret( + parameters: OpenAIRealtimeClientSecretParameters) + async throws -> OpenAIRealtimeClientSecret + /// Creates a realtime audio session for bidirectional streaming conversation with OpenAI. /// /// - Parameters: - /// - model: The model to use for the realtime session (e.g., "gpt-4o-mini-realtime-preview-2024-12-17") + /// - model: The model to use for the realtime session (e.g., "gpt-realtime-2.1") /// - configuration: Session configuration including voice, turn detection, and other settings /// - Returns: An `OpenAIRealtimeSession` for managing the WebSocket connection /// - Throws: An error if the session creation fails diff --git a/Sources/OpenAI/Public/Shared/AudioController.swift b/Sources/OpenAI/Public/Shared/AudioController.swift index e8bbc5b5..ef4114ab 100644 --- a/Sources/OpenAI/Public/Shared/AudioController.swift +++ b/Sources/OpenAI/Public/Shared/AudioController.swift @@ -15,23 +15,21 @@ private let logger = Logger(subsystem: "com.swiftopenai", category: "Audio") // MARK: - AudioController /// Use this class to control the streaming of mic data and playback of PCM16 data. -/// Audio played using the `playPCM16Audio` method does not interfere with the mic data streaming out of the `micStream` AsyncStream. -/// That is, if you use this to control audio in an OpenAI realtime session, the model will not hear itself. +/// When recording and playback are enabled together, both directions use the same +/// `AVAudioEngine` voice-processing graph so playback is available to acoustic echo cancellation. /// /// ## Implementor's note -/// We use either AVAudioEngine or AudioToolbox for mic data, depending on the platform and whether headphones are attached. +/// We use AVAudioEngine for full-duplex capture and playback. A record-only controller may use +/// AudioToolbox when no headphones are attached because it has no playback reference to share. /// The following arrangement provides for the best user experience: /// -/// +----------+---------------+------------------+ -/// | Platform | Headphones | Audio API | -/// +----------+---------------+------------------+ -/// | macOS | Yes | AudioEngine | -/// | macOS | No | AudioToolbox | -/// | iOS | Yes | AudioEngine | -/// | iOS | No | AudioToolbox | -/// | watchOS | Yes | AudioEngine | -/// | watchOS | No | AudioEngine | -/// +----------+---------------+------------------+ +/// +----------------------+---------------+------------------+ +/// | Modes | Headphones | Capture API | +/// +----------------------+---------------+------------------+ +/// | Record + playback | Any | AudioEngine | +/// | Record only | Yes | AudioEngine | +/// | Record only | No | AudioToolbox | +/// +----------------------+---------------+------------------+ /// @RealtimeActor public final class AudioController { @@ -53,11 +51,16 @@ public final class AudioController { if modes.contains(.record) { #if os(macOS) || os(iOS) - microphonePCMSampleVendor = AudioUtils.headphonesConnected - ? try MicrophonePCMSampleVendorAE(audioEngine: audioEngine) - : MicrophonePCMSampleVendorAT() + let needsSharedPlaybackReference = modes.contains(.playback) + if needsSharedPlaybackReference || AudioUtils.headphonesConnected { + microphonePCMSampleVendor = try MicrophonePCMSampleVendorAE(audioEngine: audioEngine) + usesAudioEngineForCapture = true + } else { + microphonePCMSampleVendor = MicrophonePCMSampleVendorAT() + } #else microphonePCMSampleVendor = try MicrophonePCMSampleVendorAE(audioEngine: audioEngine) + usesAudioEngineForCapture = true #endif } @@ -65,13 +68,11 @@ public final class AudioController { audioPCMPlayer = try await AudioPCMPlayer(audioEngine: audioEngine) } - audioEngine.prepare() - - // Nesting `start` in a Task is necessary on watchOS. - // There is some sort of race, and letting the runloop tick seems to "fix" it. - // If I call `prepare` and `start` in serial succession, then there is no playback on watchOS (sometimes). - Task { - try self.audioEngine.start() + // Capture installs its input tap in `micStream()`. Starting the engine before that tap exists + // can leave voice-processing input silent, particularly in Simulator. Playback-only controllers + // have no capture tap to wait for and can start immediately. + if !modes.contains(.record) { + try startAudioEngineIfNeeded() } } @@ -82,6 +83,8 @@ public final class AudioController { public let modes: [Mode] + /// Installs the microphone tap and starts the shared audio engine. Call this once before expecting + /// playback from a controller configured with both record and playback modes. public func micStream() throws -> AsyncStream { guard modes.contains(.record), @@ -89,15 +92,30 @@ public final class AudioController { else { throw OpenAIError.assertion("Please pass [.record] to the AudioController initializer") } - return try microphonePCMSampleVendor.start() + guard !hasStartedMicrophone else { + throw OpenAIError.assertion("AudioController supports one active microphone stream") + } + + let stream = try microphonePCMSampleVendor.start() + do { + if usesAudioEngineForCapture || modes.contains(.playback) { + try startAudioEngineIfNeeded() + } + hasStartedMicrophone = true + return stream + } catch { + microphonePCMSampleVendor.stop() + throw error + } } public func stop() { + _ = audioPCMPlayer?.interruptPlayback() + audioEngine.stop() microphonePCMSampleVendor?.stop() - audioPCMPlayer?.interruptPlayback() } - public func playPCM16Audio(base64String: String) { + public func playPCM16Audio(base64String: String, itemID: String? = nil) { guard modes.contains(.playback), let audioPCMPlayer @@ -105,23 +123,49 @@ public final class AudioController { logger.error("Please pass [.playback] to the AudioController initializer") return } - audioPCMPlayer.playPCM16Audio(from: base64String) + guard audioEngine.isRunning else { + logger.error("Call micStream() before playing audio on a record-and-playback controller") + return + } + audioPCMPlayer.playPCM16Audio(from: base64String, itemID: itemID) } - public func interruptPlayback() { + /// Stops queued playback and returns how many milliseconds of the current item were heard. + @discardableResult + public func interruptPlayback() -> Int? { guard modes.contains(.playback), let audioPCMPlayer else { logger.error("Please pass [.playback] to the AudioController initializer") + return nil + } + return audioPCMPlayer.interruptPlayback() + } + + /// Suspends until all currently queued audio buffers have played. + public func waitUntilPlaybackFinishes() async { + guard + modes.contains(.playback), + let audioPCMPlayer + else { return } - audioPCMPlayer.interruptPlayback() + await audioPCMPlayer.waitUntilPlaybackFinishes() } private let audioEngine: AVAudioEngine + private var hasStartedMicrophone = false private var microphonePCMSampleVendor: MicrophonePCMSampleVendor? = nil private var audioPCMPlayer: AudioPCMPlayer? = nil + private var usesAudioEngineForCapture = false + + private func startAudioEngineIfNeeded() throws { + guard !audioEngine.isRunning else { return } + audioEngine.prepare() + try audioEngine.start() + logger.info("Audio engine started with its configured capture and playback graph") + } } #endif diff --git a/Sources/OpenAI/Public/Shared/OpenAIJSONValue.swift b/Sources/OpenAI/Public/Shared/OpenAIJSONValue.swift index 9d505f4c..502711f8 100644 --- a/Sources/OpenAI/Public/Shared/OpenAIJSONValue.swift +++ b/Sources/OpenAI/Public/Shared/OpenAIJSONValue.swift @@ -6,6 +6,7 @@ // Original: https://github.com/lzell/AIProxySwift // +import CoreFoundation import Foundation // MARK: - OpenAIJSONValue @@ -41,6 +42,41 @@ public enum OpenAIJSONValue: Codable, Sendable { case array([OpenAIJSONValue]) case object([String: OpenAIJSONValue]) + /// Creates a sendable JSON value from an object produced by `JSONSerialization`. + public init?(jsonObject: Any) { + if let number = jsonObject as? NSNumber { + if CFGetTypeID(number) == CFBooleanGetTypeID() { + self = .bool(number.boolValue) + } else if CFNumberIsFloatType(number) { + self = .double(number.doubleValue) + } else { + self = .int(number.intValue) + } + return + } + + switch jsonObject { + case is NSNull: + self = .null(NSNull()) + + case let value as String: + self = .string(value) + + case let values as [Any]: + let converted = values.compactMap(OpenAIJSONValue.init(jsonObject:)) + guard converted.count == values.count else { return nil } + self = .array(converted) + + case let values as [String: Any]: + let converted = values.compactMapValues(OpenAIJSONValue.init(jsonObject:)) + guard converted.count == values.count else { return nil } + self = .object(converted) + + default: + return nil + } + } + public init(from decoder: Decoder) throws { let container = try decoder.singleValueContainer() if container.decodeNil() { diff --git a/Sources/OpenAI/Public/Shared/Tool.swift b/Sources/OpenAI/Public/Shared/Tool.swift index 3108063b..0ef6a0a6 100644 --- a/Sources/OpenAI/Public/Shared/Tool.swift +++ b/Sources/OpenAI/Public/Shared/Tool.swift @@ -773,7 +773,7 @@ public enum Tool: Codable { } /// Identifier for service connectors - public enum ConnectorId: String, Codable { + public enum ConnectorId: String, Codable, Sendable { /// Dropbox connector case dropbox = "connector_dropbox" @@ -799,8 +799,14 @@ public enum Tool: Codable { case sharePoint = "connector_sharepoint" } + /// Allowed MCP invocation contexts. + public enum MCPAllowedCaller: String, Codable, Sendable { + case direct + case programmatic + } + /// A filter object to specify which tools are allowed - public struct MCPToolFilter: Codable { + public struct MCPToolFilter: Codable, Sendable { public init(readOnly: Bool? = nil, toolNames: [String]? = nil) { self.readOnly = readOnly self.toolNames = toolNames @@ -819,7 +825,7 @@ public enum Tool: Codable { } /// List of allowed tool names or a filter object - public enum AllowedTools: Codable { + public enum AllowedTools: Codable, Sendable { /// A string array of allowed tool names case toolNames([String]) @@ -853,7 +859,7 @@ public enum Tool: Codable { } /// Approval filters for MCP tools - public struct ApprovalFilters: Codable { + public struct ApprovalFilters: Codable, Sendable { public init(always: MCPToolFilter? = nil, never: MCPToolFilter? = nil) { self.always = always self.never = never @@ -867,7 +873,7 @@ public enum Tool: Codable { } /// Specify which of the MCP server's tools require approval - public enum RequireApproval: Codable { + public enum RequireApproval: Codable, Sendable { /// All tools require approval case always @@ -915,25 +921,31 @@ public enum Tool: Codable { } /// Give the model access to additional tools via remote Model Context Protocol (MCP) servers - public struct MCPTool: Codable { + public struct MCPTool: Codable, Sendable { public init( serverLabel: String, + allowedCallers: [MCPAllowedCaller]? = nil, allowedTools: AllowedTools? = nil, authorization: String? = nil, connectorId: ConnectorId? = nil, + deferLoading: Bool? = nil, headers: [String: String]? = nil, requireApproval: RequireApproval? = nil, serverDescription: String? = nil, - serverUrl: String? = nil) + serverUrl: String? = nil, + tunnelId: String? = nil) { self.serverLabel = serverLabel + self.allowedCallers = allowedCallers self.allowedTools = allowedTools self.authorization = authorization self.connectorId = connectorId + self.deferLoading = deferLoading self.headers = headers self.requireApproval = requireApproval self.serverDescription = serverDescription self.serverUrl = serverUrl + self.tunnelId = tunnelId } /// A label for this MCP server, used to identify it in tool calls @@ -945,12 +957,18 @@ public enum Tool: Codable { /// List of allowed tool names or a filter object public let allowedTools: AllowedTools? + /// Allowed invocation contexts for this MCP server. + public let allowedCallers: [MCPAllowedCaller]? + /// An OAuth access token that can be used with a remote MCP server, either with a custom MCP server URL or a service connector public let authorization: String? /// Identifier for service connectors. One of server_url or connector_id must be provided public let connectorId: ConnectorId? + /// Whether this MCP tool is deferred and discovered via tool search. + public let deferLoading: Bool? + /// Optional HTTP headers to send to the MCP server. Use for authentication or other purposes public let headers: [String: String]? @@ -964,16 +982,22 @@ public enum Tool: Codable { /// The URL for the MCP server. One of server_url or connector_id must be provided public let serverUrl: String? + /// The Secure MCP Tunnel ID to use instead of a direct server URL. + public let tunnelId: String? + enum CodingKeys: String, CodingKey { case serverLabel = "server_label" case type + case allowedCallers = "allowed_callers" case allowedTools = "allowed_tools" case authorization case connectorId = "connector_id" + case deferLoading = "defer_loading" case headers case requireApproval = "require_approval" case serverDescription = "server_description" case serverUrl = "server_url" + case tunnelId = "tunnel_id" } } diff --git a/Tests/OpenAITests/RealtimeAudioConversionTests.swift b/Tests/OpenAITests/RealtimeAudioConversionTests.swift new file mode 100644 index 00000000..53e25918 --- /dev/null +++ b/Tests/OpenAITests/RealtimeAudioConversionTests.swift @@ -0,0 +1,34 @@ +#if canImport(AVFoundation) +import AVFoundation +import XCTest + +@testable import SwiftOpenAI + +final class RealtimeAudioConversionTests: XCTestCase { + func testNativeFloatMicrophoneBuffersConvertToRealtimePCM16() throws { + let inputFormat = try XCTUnwrap(AVAudioFormat( + commonFormat: .pcmFormatFloat32, + sampleRate: 48_000, + channels: 1, + interleaved: false)) + let inputBuffer = try XCTUnwrap(AVAudioPCMBuffer( + pcmFormat: inputFormat, + frameCapacity: 2_400)) + inputBuffer.frameLength = 2_400 + let samples = try XCTUnwrap(inputBuffer.floatChannelData?[0]) + samples.update(repeating: 0.25, count: Int(inputBuffer.frameLength)) + + let converter = MicrophonePCMSampleVendorCommon() + var convertedBuffer: AVAudioPCMBuffer? + for _ in 0..<4 where convertedBuffer == nil { + convertedBuffer = converter.resampleAndAccumulate(inputBuffer) + } + + let output = try XCTUnwrap(convertedBuffer) + XCTAssertEqual(output.format.commonFormat, .pcmFormatInt16) + XCTAssertEqual(output.format.sampleRate, 24_000) + XCTAssertEqual(output.format.channelCount, 1) + XCTAssertEqual(output.frameLength, 2_400) + } +} +#endif diff --git a/Tests/OpenAITests/RealtimeParameterTests.swift b/Tests/OpenAITests/RealtimeParameterTests.swift new file mode 100644 index 00000000..9d2df963 --- /dev/null +++ b/Tests/OpenAITests/RealtimeParameterTests.swift @@ -0,0 +1,129 @@ +import XCTest +@testable import SwiftOpenAI + +final class RealtimeParameterTests: XCTestCase { + func testRealtimeSessionConfigurationEncodesGAShape() throws { + let config = OpenAIRealtimeSessionConfiguration( + inputAudioFormat: .pcm16, + inputAudioTranscription: .init(model: Model.gptRealtimeWhisper.value, delay: .low, language: "en"), + instructions: "Be brief.", + maxResponseOutputTokens: .int(1024), + modalities: [.audio], + model: Model.gptRealtime21.value, + outputAudioFormat: .pcm16, + parallelToolCalls: true, + reasoning: .init(effort: .low), + tools: [.function(.init( + name: "get_demo_context", + description: "Get demo context.", + parameters: ["type": "object"]))], + toolChoice: .auto, + turnDetection: .init(type: .semanticVAD(eagerness: .auto, createResponse: true, interruptResponse: true)), + voice: "marin") + + let json = try decodeJSON(config) + + XCTAssertEqual(json["type"] as? String, "realtime") + XCTAssertNil(json["modalities"]) + XCTAssertNil(json["input_audio_format"]) + XCTAssertNil(json["max_response_output_tokens"]) + XCTAssertEqual(json["output_modalities"] as? [String], ["audio"]) + XCTAssertEqual(json["max_output_tokens"] as? Int, 1024) + XCTAssertEqual(json["model"] as? String, "gpt-realtime-2.1") + XCTAssertEqual(json["parallel_tool_calls"] as? Bool, true) + + let audio = try XCTUnwrap(json["audio"] as? [String: Any]) + let input = try XCTUnwrap(audio["input"] as? [String: Any]) + let inputFormat = try XCTUnwrap(input["format"] as? [String: Any]) + XCTAssertEqual(inputFormat["type"] as? String, "audio/pcm") + XCTAssertEqual(inputFormat["rate"] as? Int, 24_000) + + let transcription = try XCTUnwrap(input["transcription"] as? [String: Any]) + XCTAssertEqual(transcription["model"] as? String, "gpt-realtime-whisper") + XCTAssertEqual(transcription["delay"] as? String, "low") + + let output = try XCTUnwrap(audio["output"] as? [String: Any]) + XCTAssertEqual(output["voice"] as? String, "marin") + } + + func testFunctionCallOutputEncodesConversationItem() throws { + let event = OpenAIRealtimeFunctionCallOutput( + callID: "call_123", + output: #"{"status":"ok"}"#) + + let json = try decodeJSON(event) + XCTAssertEqual(json["type"] as? String, "conversation.item.create") + + let item = try XCTUnwrap(json["item"] as? [String: Any]) + XCTAssertEqual(item["type"] as? String, "function_call_output") + XCTAssertEqual(item["call_id"] as? String, "call_123") + XCTAssertEqual(item["output"] as? String, #"{"status":"ok"}"#) + } + + func testConversationItemTruncateEncodesPlaybackPosition() throws { + let event = OpenAIRealtimeConversationItemTruncate( + itemID: "item_assistant_123", + audioEndMS: 1_250) + + let json = try decodeJSON(event) + XCTAssertEqual(json["type"] as? String, "conversation.item.truncate") + XCTAssertEqual(json["item_id"] as? String, "item_assistant_123") + XCTAssertEqual(json["content_index"] as? Int, 0) + XCTAssertEqual(json["audio_end_ms"] as? Int, 1_250) + } + + func testConversationItemPreservesClientGeneratedID() throws { + let event = OpenAIRealtimeConversationItemCreate( + item: .init(id: "item_local_123", role: "user", text: "Hello")) + + let json = try decodeJSON(event) + let item = try XCTUnwrap(json["item"] as? [String: Any]) + XCTAssertEqual(item["id"] as? String, "item_local_123") + } + + func testJSONSerializationConversionPreservesScalarTypes() throws { + let data = Data(#"{"bool":true,"double":1.5,"int":1}"#.utf8) + let object = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any]) + + guard case .bool(true)? = OpenAIJSONValue(jsonObject: object["bool"] as Any) else { + return XCTFail("Expected a boolean JSON value") + } + guard case .double(1.5)? = OpenAIJSONValue(jsonObject: object["double"] as Any) else { + return XCTFail("Expected a double JSON value") + } + guard case .int(1)? = OpenAIJSONValue(jsonObject: object["int"] as Any) else { + return XCTFail("Expected an integer JSON value") + } + } + + func testRealtimeClientSecretParametersEncodeSession() throws { + let parameters = OpenAIRealtimeClientSecretParameters( + expiresAfter: .init(seconds: 600), + session: .init(instructions: "Be brief.", model: Model.gptRealtime21.value)) + + let json = try decodeJSON(parameters) + let expiresAfter = try XCTUnwrap(json["expires_after"] as? [String: Any]) + XCTAssertEqual(expiresAfter["anchor"] as? String, "created_at") + XCTAssertEqual(expiresAfter["seconds"] as? Int, 600) + + let session = try XCTUnwrap(json["session"] as? [String: Any]) + XCTAssertEqual(session["type"] as? String, "realtime") + XCTAssertEqual(session["model"] as? String, "gpt-realtime-2.1") + } + + func testRealtimeModelConstants() { + XCTAssertEqual(Model.gptRealtime21.value, "gpt-realtime-2.1") + XCTAssertEqual(Model.gptRealtime21Mini.value, "gpt-realtime-2.1-mini") + XCTAssertEqual(Model.gptRealtime2.value, "gpt-realtime-2") + XCTAssertEqual(Model.gptRealtime15.value, "gpt-realtime-1.5") + XCTAssertEqual(Model.gptRealtime.value, "gpt-realtime") + XCTAssertEqual(Model.gptRealtimeMini.value, "gpt-realtime-mini") + XCTAssertEqual(Model.gptRealtimeTranslate.value, "gpt-realtime-translate") + XCTAssertEqual(Model.gptRealtimeWhisper.value, "gpt-realtime-whisper") + } + + private func decodeJSON(_ value: Encodable) throws -> [String: Any] { + let data = try JSONEncoder().encode(value) + return try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any]) + } +} From 74c53450ee0c79b1dcc9d094fd9b9e64049e713b Mon Sep 17 00:00:00 2001 From: jamesrochabrun Date: Fri, 10 Jul 2026 11:50:12 -0700 Subject: [PATCH 2/2] Fix CI compatibility issues --- .../Private/Audio/MicrophonePCMSampleVendorAT.swift | 10 +++++----- Sources/OpenAI/Public/Shared/OpenAIJSONValue.swift | 11 +++++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAT.swift b/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAT.swift index b9578511..c9e63ef3 100644 --- a/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAT.swift +++ b/Sources/OpenAI/Private/Audio/MicrophonePCMSampleVendorAT.swift @@ -278,11 +278,11 @@ class MicrophonePCMSampleVendorAT: MicrophonePCMSampleVendor { /// This @RealtimeActor annotation is a lie. @RealtimeActor private let audioRenderCallback: AURenderCallback = { inRefCon, - ioActionFlags, - inTimeStamp, - inBusNumber, - inNumberFrames, - _ in + ioActionFlags, + inTimeStamp, + inBusNumber, + inNumberFrames, + _ in let microphonePCMSampleVendor = Unmanaged .fromOpaque(inRefCon) .takeUnretainedValue() diff --git a/Sources/OpenAI/Public/Shared/OpenAIJSONValue.swift b/Sources/OpenAI/Public/Shared/OpenAIJSONValue.swift index 502711f8..14760b2d 100644 --- a/Sources/OpenAI/Public/Shared/OpenAIJSONValue.swift +++ b/Sources/OpenAI/Public/Shared/OpenAIJSONValue.swift @@ -6,7 +6,6 @@ // Original: https://github.com/lzell/AIProxySwift // -import CoreFoundation import Foundation // MARK: - OpenAIJSONValue @@ -45,17 +44,21 @@ public enum OpenAIJSONValue: Codable, Sendable { /// Creates a sendable JSON value from an object produced by `JSONSerialization`. public init?(jsonObject: Any) { if let number = jsonObject as? NSNumber { - if CFGetTypeID(number) == CFBooleanGetTypeID() { + switch String(cString: number.objCType) { + case "c": self = .bool(number.boolValue) - } else if CFNumberIsFloatType(number) { + case "f", "d": self = .double(number.doubleValue) - } else { + default: self = .int(number.intValue) } return } switch jsonObject { + case let value as Bool: + self = .bool(value) + case is NSNull: self = .null(NSNull())